private static NativeFileSystemStore createDefaultStore(Configuration conf) {
  NativeFileSystemStore store = new Jets3tNativeFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      conf.getInt("fs.s3.maxRetries", 4),
      conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String, RetryPolicy> methodNameToPolicyMap =
    new HashMap<String, RetryPolicy>();
  methodNameToPolicyMap.put("storeFile", methodPolicy);
  methodNameToPolicyMap.put("rename", methodPolicy);
  
  return (NativeFileSystemStore)
    RetryProxy.create(NativeFileSystemStore.class, store,
        methodNameToPolicyMap);
}
 

private static NativeFileSystemStore createDefaultStore(Configuration conf) {
  NativeFileSystemStore store = new Jets3tNativeFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      conf.getInt("fs.s3.maxRetries", 4),
      conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String, RetryPolicy> methodNameToPolicyMap =
    new HashMap<String, RetryPolicy>();
  methodNameToPolicyMap.put("storeFile", methodPolicy);
  methodNameToPolicyMap.put("rename", methodPolicy);
  
  return (NativeFileSystemStore)
    RetryProxy.create(NativeFileSystemStore.class, store,
        methodNameToPolicyMap);
}
 

private static NativeFileSystemStore createDefaultStore(Configuration conf) {
  NativeFileSystemStore store = new Jets3tNativeFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      conf.getInt("fs.s3.maxRetries", 4),
      conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String, RetryPolicy> methodNameToPolicyMap =
    new HashMap<String, RetryPolicy>();
  methodNameToPolicyMap.put("storeFile", methodPolicy);
  
  return (NativeFileSystemStore)
    RetryProxy.create(NativeFileSystemStore.class, store,
        methodNameToPolicyMap);
}
 

public void initialize(URI uri, Configuration conf) throws IOException {
  S3Credentials s3Credentials = new S3Credentials();
  s3Credentials.initialize(uri, conf);
  try {
    AWSCredentials awsCredentials =
      new AWSCredentials(s3Credentials.getAccessKey(),
          s3Credentials.getSecretAccessKey());
    this.s3Service = new RestS3Service(awsCredentials);
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
  bucket = new S3Bucket(uri.getHost());
}
 

public FileMetadata retrieveMetadata(String key) throws IOException {
  try {
    S3Object object = s3Service.getObjectDetails(bucket, key);
    return new FileMetadata(key, object.getContentLength(),
        object.getLastModifiedDate().getTime());
  } catch (S3ServiceException e) {
    // Following is brittle. Is there a better way?
    if (e.getMessage().contains("ResponseCode=404")) {
      return null;
    }
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}
 

public InputStream retrieve(String key, long byteRangeStart)
  throws IOException {
  try {
    S3Object object = s3Service.getObject(bucket, key, null, null, null,
                                          null, byteRangeStart, null);
    return object.getDataInputStream();
  } catch (S3ServiceException e) {
    if ("NoSuchKey".equals(e.getS3ErrorCode())) {
      return null;
    }
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}
 

private PartialListing list(String prefix, String delimiter,
    int maxListingLength, String priorLastKey) throws IOException {
  try {
    if (prefix.length() > 0 && !prefix.endsWith(PATH_DELIMITER)) {
      prefix += PATH_DELIMITER;
    }
    S3ObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(),
        prefix, delimiter, maxListingLength, priorLastKey);
    
    FileMetadata[] fileMetadata =
      new FileMetadata[chunk.getObjects().length];
    for (int i = 0; i < fileMetadata.length; i++) {
      S3Object object = chunk.getObjects()[i];
      fileMetadata[i] = new FileMetadata(object.getKey(),
          object.getContentLength(), object.getLastModifiedDate().getTime());
    }
    return new PartialListing(chunk.getPriorLastKey(), fileMetadata,
        chunk.getCommonPrefixes());
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}
 

public void dump() throws IOException {
  StringBuilder sb = new StringBuilder("S3 Native Filesystem, ");
  sb.append(bucket.getName()).append("\n");
  try {
    S3Object[] objects = s3Service.listObjects(bucket);
    for (int i = 0; i < objects.length; i++) {
      sb.append(objects[i].getKey()).append("\n");
    }
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
  System.out.println(sb);
}
 

private static NativeFileSystemStore createDefaultStore(Configuration conf) {
  NativeFileSystemStore store = new Jets3tNativeFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      conf.getInt("fs.s3.maxRetries", 4),
      conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String, RetryPolicy> methodNameToPolicyMap =
    new HashMap<String, RetryPolicy>();
  methodNameToPolicyMap.put("storeFile", methodPolicy);
  
  return (NativeFileSystemStore)
    RetryProxy.create(NativeFileSystemStore.class, store,
        methodNameToPolicyMap);
}
 

public void initialize(URI uri, Configuration conf) throws IOException {
  S3Credentials s3Credentials = new S3Credentials();
  s3Credentials.initialize(uri, conf);
  try {
    AWSCredentials awsCredentials =
      new AWSCredentials(s3Credentials.getAccessKey(),
          s3Credentials.getSecretAccessKey());
    this.s3Service = new RestS3Service(awsCredentials);
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
  bucket = new S3Bucket(uri.getHost());
}
 

public FileMetadata retrieveMetadata(String key) throws IOException {
  try {
    S3Object object = s3Service.getObjectDetails(bucket, key);
    return new FileMetadata(key, object.getContentLength(),
        object.getLastModifiedDate().getTime());
  } catch (S3ServiceException e) {
    // Following is brittle. Is there a better way?
    if (e.getMessage().contains("ResponseCode=404")) {
      return null;
    }
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}
 

public InputStream retrieve(String key, long byteRangeStart)
  throws IOException {
  try {
    S3Object object = s3Service.getObject(bucket, key, null, null, null,
                                          null, byteRangeStart, null);
    return object.getDataInputStream();
  } catch (S3ServiceException e) {
    if ("NoSuchKey".equals(e.getS3ErrorCode())) {
      return null;
    }
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}
 

private PartialListing list(String prefix, String delimiter,
    int maxListingLength, String priorLastKey) throws IOException {
  try {
    if (prefix.length() > 0 && !prefix.endsWith(PATH_DELIMITER)) {
      prefix += PATH_DELIMITER;
    }
    S3ObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(),
        prefix, delimiter, maxListingLength, priorLastKey);
    
    FileMetadata[] fileMetadata =
      new FileMetadata[chunk.getObjects().length];
    for (int i = 0; i < fileMetadata.length; i++) {
      S3Object object = chunk.getObjects()[i];
      fileMetadata[i] = new FileMetadata(object.getKey(),
          object.getContentLength(), object.getLastModifiedDate().getTime());
    }
    return new PartialListing(chunk.getPriorLastKey(), fileMetadata,
        chunk.getCommonPrefixes());
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}
 

public void dump() throws IOException {
  StringBuilder sb = new StringBuilder("S3 Native Filesystem, ");
  sb.append(bucket.getName()).append("\n");
  try {
    S3Object[] objects = s3Service.listObjects(bucket);
    for (int i = 0; i < objects.length; i++) {
      sb.append(objects[i].getKey()).append("\n");
    }
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
  System.out.println(sb);
}
 

public void storeEmptyFile(String key) throws IOException {
  try {
    S3Object object = new S3Object(key);
    object.setDataInputStream(new ByteArrayInputStream(new byte[0]));
    object.setContentType("binary/octet-stream");
    object.setContentLength(0);
    s3Service.putObject(bucket, object);
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}
 

public InputStream retrieve(String key) throws IOException {
  try {
    S3Object object = s3Service.getObject(bucket, key);
    return object.getDataInputStream();
  } catch (S3ServiceException e) {
    if ("NoSuchKey".equals(e.getS3ErrorCode())) {
      return null;
    }
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}
 

public void delete(String key) throws IOException {
  try {
    s3Service.deleteObject(bucket, key);
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}
 

public void rename(String srcKey, String dstKey) throws IOException {
  try {
    s3Service.moveObject(bucket.getName(), srcKey, bucket.getName(),
        new S3Object(dstKey), false);
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}
 

public void purge(String prefix) throws IOException {
  try {
    S3Object[] objects = s3Service.listObjects(bucket, prefix, null);
    for (int i = 0; i < objects.length; i++) {
      s3Service.deleteObject(bucket, objects[i].getKey());
    }
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}
 

public void storeEmptyFile(String key) throws IOException {
  try {
    S3Object object = new S3Object(key);
    object.setDataInputStream(new ByteArrayInputStream(new byte[0]));
    object.setContentType("binary/octet-stream");
    object.setContentLength(0);
    s3Service.putObject(bucket, object);
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}
 

public InputStream retrieve(String key) throws IOException {
  try {
    S3Object object = s3Service.getObject(bucket, key);
    return object.getDataInputStream();
  } catch (S3ServiceException e) {
    if ("NoSuchKey".equals(e.getS3ErrorCode())) {
      return null;
    }
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}
 

public void delete(String key) throws IOException {
  try {
    s3Service.deleteObject(bucket, key);
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}
 

public void rename(String srcKey, String dstKey) throws IOException {
  try {
    s3Service.moveObject(bucket.getName(), srcKey, bucket.getName(),
        new S3Object(dstKey), false);
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}
 

public void purge(String prefix) throws IOException {
  try {
    S3Object[] objects = s3Service.listObjects(bucket, prefix, null);
    for (int i = 0; i < objects.length; i++) {
      s3Service.deleteObject(bucket, objects[i].getKey());
    }
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}