/**
 * Read through the specified number of bytes.
 * The implementation iterates a byte a time, which may seem inefficient
 * compared to the read(bytes[]) method offered by input streams.
 * However, if you look at the code that implements that method, it comes
 * down to read() one char at a time -only here the return value is discarded.
 *
 *<p/>
 * This is a no-op if the stream is closed
 * @param bytes number of bytes to read.
 * @throws IOException IO problems
 * @throws SwiftException if a read returned -1.
 */
private int chompBytes(long bytes) throws IOException {
  int count = 0;
  if (httpStream != null) {
    int result;
    for (long i = 0; i < bytes; i++) {
      result = httpStream.read();
      if (result < 0) {
        throw new SwiftException("Received error code while chomping input");
      }
      count ++;
      incPos(1);
    }
  }
  return count;
}
 

/**
 * Read through the specified number of bytes.
 * The implementation iterates a byte a time, which may seem inefficient
 * compared to the read(bytes[]) method offered by input streams.
 * However, if you look at the code that implements that method, it comes
 * down to read() one char at a time -only here the return value is discarded.
 *
 *<p/>
 * This is a no-op if the stream is closed
 * @param bytes number of bytes to read.
 * @throws IOException IO problems
 * @throws SwiftException if a read returned -1.
 */
private int chompBytes(long bytes) throws IOException {
  int count = 0;
  if (httpStream != null) {
    int result;
    for (long i = 0; i < bytes; i++) {
      result = httpStream.read();
      if (result < 0) {
        throw new SwiftException("Received error code while chomping input");
      }
      count ++;
      incPos(1);
    }
  }
  return count;
}
 

/**
 * Read through the specified number of bytes.
 * The implementation iterates a byte a time, which may seem inefficient
 * compared to the read(bytes[]) method offered by input streams.
 * However, if you look at the code that implements that method, it comes
 * down to read() one char at a time -only here the return value is discarded.
 *
 *<p/>
 * This is a no-op if the stream is closed
 * @param bytes number of bytes to read.
 * @throws IOException IO problems
 * @throws SwiftException if a read returned -1.
 */
private int chompBytes(long bytes) throws IOException {
  int count = 0;
  if (httpStream != null) {
    int result;
    for (long i = 0; i < bytes; i++) {
      result = httpStream.read();
      if (result < 0) {
        throw new SwiftException("Received error code while chomping input");
      }
      count ++;
      incPos(1);
    }
  }
  return count;
}
 

/**
 * Make an HTTP GET request to Swift to get a range of data in the object.
 *
 * @param path   path to object
 * @param offset offset from file beginning
 * @param length file length
 * @return The input stream -which must be closed afterwards.
 * @throws IOException Problems
 * @throws SwiftException swift specific error
 * @throws FileNotFoundException path is not there
 */
public HttpBodyContent getData(SwiftObjectPath path,
                               long offset,
                               long length) throws IOException {
  if (offset < 0) {
    throw new SwiftException("Invalid offset: " + offset
                          + " in getDataAsInputStream( path=" + path
                          + ", offset=" + offset
                          + ", length =" + length + ")");
  }
  if (length <= 0) {
    throw new SwiftException("Invalid length: " + length
              + " in getDataAsInputStream( path="+ path
                          + ", offset=" + offset
                          + ", length ="+ length + ")");
  }

  final String range = String.format(SWIFT_RANGE_HEADER_FORMAT_PATTERN,
          offset,
          offset + length - 1);
  if (LOG.isDebugEnabled()) {
    LOG.debug("getData:" + range);
  }

  return getData(path,
                 new Header(HEADER_RANGE, range),
                 SwiftRestClient.NEWEST);
}
 

/**
 * Converts Swift path to URI to make request.
 * This is public for unit testing
 *
 * @param path path to object
 * @param endpointURI damain url e.g. http://domain.com
 * @return valid URI for object
 * @throws SwiftException
 */
public static URI pathToURI(SwiftObjectPath path,
                            URI endpointURI) throws SwiftException {
  checkNotNull(endpointURI, "Null Endpoint -client is not authenticated");

  String dataLocationURI = endpointURI.toString();
  try {

    dataLocationURI = SwiftUtils.joinPaths(dataLocationURI, encodeUrl(path.toUriPath()));
    return new URI(dataLocationURI);
  } catch (URISyntaxException e) {
    throw new SwiftException("Failed to create URI from " + dataLocationURI, e);
  }
}
 

/**
 * Encode the URL. This extends {@link URLEncoder#encode(String, String)}
 * with a replacement of + with %20.
 * @param url URL string
 * @return an encoded string
 * @throws SwiftException if the URL cannot be encoded
 */
private static String encodeUrl(String url) throws SwiftException {
  if (url.matches(".*\\s+.*")) {
    try {
      url = URLEncoder.encode(url, "UTF-8");
      url = url.replace("+", "%20");
    } catch (UnsupportedEncodingException e) {
      throw new SwiftException("failed to encode URI", e);
    }
  }

  return url;
}
 

private File newBackupFile() throws IOException {
  File dir = new File(conf.get("hadoop.tmp.dir"));
  if (!dir.mkdirs() && !dir.exists()) {
    throw new SwiftException("Cannot create Swift buffer directory: " + dir);
  }
  File result = File.createTempFile("output-", ".tmp", dir);
  result.deleteOnExit();
  return result;
}
 

/**
 * Copy an object
 * @param srcObject  source object path
 * @param destObject destination object path
 * @throws IOException IO problems
 */
private void copyObject(SwiftObjectPath srcObject,
                                  SwiftObjectPath destObject) throws
        IOException {
  if (srcObject.isEqualToOrParentOf(destObject)) {
    throw new SwiftException(
      "Can't copy " + srcObject + " onto " + destObject);
  }
  //do the copy
  boolean copySucceeded = swiftRestClient.copyObject(srcObject, destObject);
  if (!copySucceeded) {
    throw new SwiftException("Copy of " + srcObject + " to "
            + destObject + "failed");
  }
}
 

/**
 * Take a Hadoop path and return one which uses the URI prefix and authority
 * of this FS. It doesn't make a relative path absolute
 * @param path path in
 * @return path with a URI bound to this FS
 * @throws SwiftException URI cannot be created.
 */
public Path getCorrectSwiftPath(Path path) throws
        SwiftException {
  try {
    final URI fullUri = new URI(uri.getScheme(),
            uri.getAuthority(),
            path.toUri().getPath(),
            null,
            null);

    return new Path(fullUri);
  } catch (URISyntaxException e) {
    throw new SwiftException("Specified path " + path + " is incorrect", e);
  }
}
 

/**
 * Builds a hadoop-Path from a swift path, inserting the URI authority
 * of this FS instance
 * @param path swift object path
 * @return Hadoop path
 * @throws SwiftException if the URI couldn't be created.
 */
private Path getCorrectSwiftPath(SwiftObjectPath path) throws
        SwiftException {
  try {
    final URI fullUri = new URI(uri.getScheme(),
            uri.getAuthority(),
            path.getObject(),
            null,
            null);

    return new Path(fullUri);
  } catch (URISyntaxException e) {
    throw new SwiftException("Specified path " + path + " is incorrect", e);
  }
}
 

/**
 * Make an HTTP GET request to Swift to get a range of data in the object.
 *
 * @param path   path to object
 * @param offset offset from file beginning
 * @param length file length
 * @return The input stream -which must be closed afterwards.
 * @throws IOException Problems
 * @throws SwiftException swift specific error
 * @throws FileNotFoundException path is not there
 */
public HttpBodyContent getData(SwiftObjectPath path,
                               long offset,
                               long length) throws IOException {
  if (offset < 0) {
    throw new SwiftException("Invalid offset: " + offset
                          + " in getDataAsInputStream( path=" + path
                          + ", offset=" + offset
                          + ", length =" + length + ")");
  }
  if (length <= 0) {
    throw new SwiftException("Invalid length: " + length
              + " in getDataAsInputStream( path="+ path
                          + ", offset=" + offset
                          + ", length ="+ length + ")");
  }

  final String range = String.format(SWIFT_RANGE_HEADER_FORMAT_PATTERN,
          offset,
          offset + length - 1);
  if (LOG.isDebugEnabled()) {
    LOG.debug("getData:" + range);
  }

  return getData(path,
                 new Header(HEADER_RANGE, range),
                 SwiftRestClient.NEWEST);
}
 

/**
 * Converts Swift path to URI to make request.
 * This is public for unit testing
 *
 * @param path path to object
 * @param endpointURI damain url e.g. http://domain.com
 * @return valid URI for object
 * @throws SwiftException
 */
public static URI pathToURI(SwiftObjectPath path,
                            URI endpointURI) throws SwiftException {
  checkNotNull(endpointURI, "Null Endpoint -client is not authenticated");

  String dataLocationURI = endpointURI.toString();
  try {

    dataLocationURI = SwiftUtils.joinPaths(dataLocationURI, encodeUrl(path.toUriPath()));
    return new URI(dataLocationURI);
  } catch (URISyntaxException e) {
    throw new SwiftException("Failed to create URI from " + dataLocationURI, e);
  }
}
 

/**
 * Encode the URL. This extends {@link URLEncoder#encode(String, String)}
 * with a replacement of + with %20.
 * @param url URL string
 * @return an encoded string
 * @throws SwiftException if the URL cannot be encoded
 */
private static String encodeUrl(String url) throws SwiftException {
  if (url.matches(".*\\s+.*")) {
    try {
      url = URLEncoder.encode(url, "UTF-8");
      url = url.replace("+", "%20");
    } catch (UnsupportedEncodingException e) {
      throw new SwiftException("failed to encode URI", e);
    }
  }

  return url;
}
 

private File newBackupFile() throws IOException {
  File dir = new File(conf.get("hadoop.tmp.dir"));
  if (!dir.mkdirs() && !dir.exists()) {
    throw new SwiftException("Cannot create Swift buffer directory: " + dir);
  }
  File result = File.createTempFile("output-", ".tmp", dir);
  result.deleteOnExit();
  return result;
}
 

/**
 * Copy an object
 * @param srcObject  source object path
 * @param destObject destination object path
 * @throws IOException IO problems
 */
private void copyObject(SwiftObjectPath srcObject,
                                  SwiftObjectPath destObject) throws
        IOException {
  if (srcObject.isEqualToOrParentOf(destObject)) {
    throw new SwiftException(
      "Can't copy " + srcObject + " onto " + destObject);
  }
  //do the copy
  boolean copySucceeded = swiftRestClient.copyObject(srcObject, destObject);
  if (!copySucceeded) {
    throw new SwiftException("Copy of " + srcObject + " to "
            + destObject + "failed");
  }
}
 

/**
 * Take a Hadoop path and return one which uses the URI prefix and authority
 * of this FS. It doesn't make a relative path absolute
 * @param path path in
 * @return path with a URI bound to this FS
 * @throws SwiftException URI cannot be created.
 */
public Path getCorrectSwiftPath(Path path) throws
        SwiftException {
  try {
    final URI fullUri = new URI(uri.getScheme(),
            uri.getAuthority(),
            path.toUri().getPath(),
            null,
            null);

    return new Path(fullUri);
  } catch (URISyntaxException e) {
    throw new SwiftException("Specified path " + path + " is incorrect", e);
  }
}
 

/**
 * Builds a hadoop-Path from a swift path, inserting the URI authority
 * of this FS instance
 * @param path swift object path
 * @return Hadoop path
 * @throws SwiftException if the URI couldn't be created.
 */
private Path getCorrectSwiftPath(SwiftObjectPath path) throws
        SwiftException {
  try {
    final URI fullUri = new URI(uri.getScheme(),
            uri.getAuthority(),
            path.getObject(),
            null,
            null);

    return new Path(fullUri);
  } catch (URISyntaxException e) {
    throw new SwiftException("Specified path " + path + " is incorrect", e);
  }
}
 

/**
 * Make an HTTP GET request to Swift to get a range of data in the object.
 *
 * @param url   url to object
 * @param offset offset from file beginning
 * @param length file length
 * @return The input stream -which must be closed afterwards.
 * @throws IOException Problems
 * @throws SwiftException swift specific error
 * @throws FileNotFoundException path is not there
 */
public HttpBodyContent getData(URI url,
                               long offset,
                               long length) throws IOException {
  if (offset < 0) {
    throw new SwiftException("Invalid offset: " + offset
                          + " in getDataAsInputStream( url=" + url
                          + ", offset=" + offset
                          + ", length =" + length + ")");
  }
  if (length <= 0) {
    throw new SwiftException("Invalid length: " + length
              + " in getDataAsInputStream( url="+ url
                          + ", offset=" + offset
                          + ", length ="+ length + ")");
  }

  final String range = String.format(SWIFT_RANGE_HEADER_FORMAT_PATTERN,
          offset,
          offset + length - 1);
  if (LOG.isDebugEnabled()) {
    LOG.debug("getData:" + range);
  }

  preRemoteCommand("getData");
  return getData(url,
                 new Header(HEADER_RANGE, range),
                 SwiftRestClient.NEWEST);
}
 

/**
 * Converts Swift path to URI to make request.
 * This is public for unit testing
 *
 * @param path path to object
 * @param endpointURI damain url e.g. http://domain.com
 * @return valid URI for object
 */
public static URI pathToURI(SwiftObjectPath path,
                            URI endpointURI) throws SwiftException {
  checkNotNull(endpointURI, "Null Endpoint -client is not authenticated");

  String dataLocationURI = endpointURI.toString();
  try {

    dataLocationURI = SwiftUtils.joinPaths(dataLocationURI, encodeUrl(path.toUriPath()));
    return new URI(dataLocationURI);
  } catch (URISyntaxException e) {
    throw new SwiftException("Failed to create URI from " + dataLocationURI, e);
  }
}
 

/**
 * Encode the URL. This extends {@link URLEncoder#encode(String, String)}
 * with a replacement of + with %20.
 * @param url URL string
 * @return an encoded string
 * @throws SwiftException if the URL cannot be encoded
 */
private static String encodeUrl(String url) throws SwiftException {
  if (url.matches(".*\\s+.*")) {
    try {
      url = URLEncoder.encode(url, "UTF-8");
      url = url.replace("+", "%20");
    } catch (UnsupportedEncodingException e) {
      throw new SwiftException("failed to encode URI", e);
    }
  }

  return url;
}
 

private File newBackupFile() throws IOException {
  File dir = new File(conf.get("hadoop.tmp.dir"));
  if (!dir.mkdirs() && !dir.exists()) {
    throw new SwiftException("Cannot create Swift buffer directory: " + dir);
  }
  File result = File.createTempFile("output-", ".tmp", dir);
  result.deleteOnExit();
  return result;
}
 

private String getHostRack() throws SwiftException {
  String hostAddress;
  try {
    hostAddress = InetAddress.getLocalHost().getHostAddress();
  } catch (UnknownHostException e) {
    throw new SwiftException("Failed to get localhost address", e);
  }
  return getRack(hostAddress);
}
 

/**
 * Take a Hadoop path and return one which uses the URI prefix and authority
 * of this FS. It doesn't make a relative path absolute
 * @param path path in
 * @return path with a URI bound to this FS
 * @throws SwiftException URI cannot be created.
 */
public Path getCorrectSwiftPath(Path path) throws
        SwiftException {
  try {
    final URI fullUri = new URI(uri.getScheme(),
            uri.getAuthority(),
            path.toUri().getPath(),
            null,
            null);

    return new Path(fullUri);
  } catch (URISyntaxException e) {
    throw new SwiftException("Specified path " + path + " is incorrect", e);
  }
}
 

/**
 * Builds a hadoop-Path from a swift path, inserting the URI authority
 * of this FS instance
 * @param path swift object path
 * @return Hadoop path
 * @throws SwiftException if the URI couldn't be created.
 */
private Path getCorrectSwiftPath(SwiftObjectPath path) throws
        SwiftException {
  try {
    final URI fullUri = new URI(uri.getScheme(),
            uri.getAuthority(),
            path.getObject(),
            null,
            null);

    return new Path(fullUri);
  } catch (URISyntaxException e) {
    throw new SwiftException("Specified path " + path + " is incorrect", e);
  }
}
 

/**
 * check that the output stream is open
 *
 * @throws SwiftException if it is not
 */
private synchronized void verifyOpen() throws SwiftException {
  if (closed) {
    throw new SwiftConnectionClosedException();
  }
}
 

/**
 * check that the output stream is open
 *
 * @throws SwiftException if it is not
 */
private synchronized void verifyOpen() throws SwiftException {
  if (closed) {
    throw new SwiftConnectionClosedException();
  }
}
 

/**
 * check that the output stream is open
 *
 * @throws SwiftException if it is not
 */
private synchronized void verifyOpen() throws SwiftException {
  if (closed) {
    throw new SwiftException("Output stream is closed");
  }
}
 

/**
 * Convert a swift path to a URI relative to the current endpoint.
 *
 * @param path path
 * @return an path off the current endpoint URI.
 * @throws SwiftException
 */
private URI pathToURI(SwiftObjectPath path) throws SwiftException {
  return pathToURI(path, getEndpointURI());
}
 

/**
 * Convert a swift path to a URI relative to the current endpoint.
 *
 * @param path path
 * @return an path off the current endpoint URI.
 * @throws SwiftException
 */
private URI pathToURI(SwiftObjectPath path) throws SwiftException {
  return pathToURI(path, getEndpointURI());
}
 

/**
 * Convert a swift path to a URI relative to the current endpoint.
 *
 * @param path path
 * @return an path off the current endpoint URI.
 * @throws SwiftException
 */
private URI pathToURI(SwiftObjectPath path) throws SwiftException {
  return pathToURI(path, getEndpointURI());
}