org.apache.hadoop.fs.s3.S3Exception#org.jets3t.service.model.S3Object源码实例Demo

public Set<Path> listAllPaths() throws IOException {
  try {
    String prefix = urlEncode(Path.SEPARATOR);
    S3Object[] objects = s3Service.listObjects(bucket, prefix, null);
    Set<Path> prefixes = new TreeSet<Path>();
    for (int i = 0; i < objects.length; i++) {
      prefixes.add(keyToPath(objects[i].getKey()));
    }
    return prefixes;
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }   
}
 

@Override
public HttpResponseOutputStream<VersionId> write(final Path file, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException {
    final S3Object object = new S3WriteFeature(session, new S3DisabledMultipartService())
        .getDetails(file, status);
    // ID for the initiated multipart upload.
    final MultipartUpload multipart;
    try {
        multipart = session.getClient().multipartStartUpload(
            containerService.getContainer(file).getName(), object);
        if(log.isDebugEnabled()) {
            log.debug(String.format("Multipart upload started for %s with ID %s",
                multipart.getObjectKey(), multipart.getUploadId()));
        }
    }
    catch(ServiceException e) {
        throw new S3ExceptionMappingService().map("Upload {0} failed", e, file);
    }
    final MultipartOutputStream proxy = new MultipartOutputStream(multipart, file, status);
    return new HttpResponseOutputStream<VersionId>(new MemorySegementingOutputStream(proxy,
        preferences.getInteger("s3.upload.multipart.size"))) {
        @Override
        public VersionId getStatus() {
            return proxy.getVersionId();
        }
    };
}
 

@Test
public void testSetMetadataFileLeaveOtherFeatures() throws Exception {
    final Path container = new Path("test-us-east-1-cyberduck", EnumSet.of(Path.Type.volume));
    final Path test = new Path(container, UUID.randomUUID().toString(), EnumSet.of(Path.Type.file));
    new S3TouchFeature(session).touch(test, new TransferStatus());
    final String v = UUID.randomUUID().toString();
    final S3StorageClassFeature storage = new S3StorageClassFeature(session);
    storage.setClass(test, S3Object.STORAGE_CLASS_REDUCED_REDUNDANCY);
    assertEquals(S3Object.STORAGE_CLASS_REDUCED_REDUNDANCY, storage.getClass(test));

    final S3EncryptionFeature encryption = new S3EncryptionFeature(session);
    encryption.setEncryption(test, S3EncryptionFeature.SSE_AES256);
    assertEquals("AES256", encryption.getEncryption(test).algorithm);

    final S3MetadataFeature feature = new S3MetadataFeature(session, new S3AccessControlListFeature(session));
    feature.setMetadata(test, Collections.singletonMap("Test", v));
    final Map<String, String> metadata = feature.getMetadata(test);
    assertFalse(metadata.isEmpty());
    assertTrue(metadata.containsKey("test"));
    assertEquals(v, metadata.get("test"));

    assertEquals(S3Object.STORAGE_CLASS_REDUCED_REDUNDANCY, storage.getClass(test));
    assertEquals("AES256", encryption.getEncryption(test).algorithm);

    new S3DefaultDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback());
}
 

@Test
public void test() throws Exception {
    RestS3Service s3Service = mock(RestS3Service.class);
    AccessControlList acl = new AccessControlList();
    doReturn(acl).when(s3Service).getObjectAcl("bucket", "key");
    doNothing().when(s3Service).putObjectAcl("bucket", "key", acl);

    GrantAcl grantAcl = new GrantAcl(s3Service, "1,2,3", 1);
    S3Object obj = new S3Object("key");
    obj.setBucketName("bucket");
    obj.setAcl(GSAccessControlList.REST_CANNED_BUCKET_OWNER_FULL_CONTROL);
    assertTrue(grantAcl.grantAcl(obj));

    Set<GrantAndPermission> grants = new HashSet<GrantAndPermission>(Arrays.asList(acl.getGrantAndPermissions()));
    assertEquals(grants.size(), 3);
    Set<GrantAndPermission> grantSet = new HashSet<GrantAndPermission>();
    for (int i = 1; i <= 3; ++i) {
        grantSet.add(new GrantAndPermission(new CanonicalGrantee(Integer.toString(i)), Permission.PERMISSION_READ));
    }
}
 

public Set<Path> listSubPaths(Path path) throws IOException {
  try {
    String prefix = pathToKey(path);
    if (!prefix.endsWith(PATH_DELIMITER)) {
      prefix += PATH_DELIMITER;
    }
    S3Object[] objects = s3Service.listObjects(bucket, prefix, PATH_DELIMITER);
    Set<Path> prefixes = new TreeSet<Path>();
    for (int i = 0; i < objects.length; i++) {
      prefixes.add(keyToPath(objects[i].getKey()));
    }
    prefixes.remove(path);
    return prefixes;
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}
 

@Action
public void lifecyclePopupClicked(final NSButton sender) {
    if(this.toggleS3Settings(false)) {
        final LifecycleConfiguration configuration = new LifecycleConfiguration(
            lifecycleTransitionCheckbox.state() == NSCell.NSOnState ? Integer.valueOf(lifecycleTransitionPopup.selectedItem().representedObject()) : null,
            S3Object.STORAGE_CLASS_GLACIER,
            lifecycleDeleteCheckbox.state() == NSCell.NSOnState ? Integer.valueOf(lifecycleDeletePopup.selectedItem().representedObject()) : null);
        controller.background(new WorkerBackgroundAction<Boolean>(controller, session, new WriteLifecycleWorker(files, configuration) {
            @Override
            public void cleanup(final Boolean result) {
                toggleS3Settings(true);
                initS3();
            }
        }));
    }
}
 

public void storeLargeFile(String key, File file, byte[] md5Hash)
    throws IOException {
  S3Object object = new S3Object(key);
  object.setDataInputFile(file);
  object.setContentType("binary/octet-stream");
  object.setContentLength(file.length());
  object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
  if (md5Hash != null) {
    object.setMd5Hash(md5Hash);
  }

  List<StorageObject> objectsToUploadAsMultipart =
      new ArrayList<StorageObject>();
  objectsToUploadAsMultipart.add(object);
  MultipartUtils mpUtils = new MultipartUtils(multipartBlockSize);

  try {
    mpUtils.uploadObjects(bucket.getName(), s3Service,
                          objectsToUploadAsMultipart, null);
  } catch (Exception e) {
    handleException(e, key);
  }
}
 

public InputStream retrieve(String key, long byteRangeStart)
  throws IOException {
  try {
    S3Object object = s3Service.getObject(bucket, key, null, null, null,
                                          null, byteRangeStart, null);
    return object.getDataInputStream();
  } catch (S3ServiceException e) {
    if ("NoSuchKey".equals(e.getS3ErrorCode())) {
      return null;
    }
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}
 

/**
 *
 * @param key
 * The key is the object name that is being retrieved from the S3 bucket
 * @return
 * This method returns null if the key is not found
 * @throws IOException
 */

@Override
public InputStream retrieve(String key, long byteRangeStart)
        throws IOException {
  try {
    LOG.debug("Getting key: {} from bucket: {} with byteRangeStart: {}",
        key, bucket.getName(), byteRangeStart);
    S3Object object = s3Service.getObject(bucket, key, null, null, null,
                                          null, byteRangeStart, null);
    return object.getDataInputStream();
  } catch (ServiceException e) {
    handleException(e, key);
    return null;
  }
}
 

@Override
public void copy(String srcKey, String dstKey) throws IOException {
  try {
    if(LOG.isDebugEnabled()) {
      LOG.debug("Copying srcKey: " + srcKey + "to dstKey: " + dstKey + "in bucket: " + bucket.getName());
    }
    if (multipartEnabled) {
      S3Object object = s3Service.getObjectDetails(bucket, srcKey, null,
                                                   null, null, null);
      if (multipartCopyBlockSize > 0 &&
          object.getContentLength() > multipartCopyBlockSize) {
        copyLargeFile(object, dstKey);
        return;
      }
    }

    S3Object dstObject = new S3Object(dstKey);
    dstObject.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
    s3Service.copyObject(bucket.getName(), srcKey, bucket.getName(),
        dstObject, false);
  } catch (ServiceException e) {
    handleException(e, srcKey);
  }
}
 

public boolean grantAcl(S3Object object) throws ServiceException, InterruptedException {
    if(Strings.isNullOrEmpty(s3Acl)){
        return true;
    }

    for (int i = 0; i < s3AclRetries; ++i) {
        try {
            AccessControlList acl = s3Service.getObjectAcl(object.getBucketName(), object.getKey());
            for (String id : s3Acl.split(",")) {
                acl.grantPermission(new CanonicalGrantee(id), Permission.PERMISSION_READ);
            }
            s3Service.putObjectAcl(object.getBucketName(), object.getKey(), acl);
            return true;
        } catch (Exception e) {
            log.error("Exception while granting ACL: " + e.getMessage(), e);
            Thread.sleep(1000 * (i + 1));
        }
    }

    return false;

}
 

private PartialListing list(String prefix, String delimiter,
    int maxListingLength, String priorLastKey) throws IOException {
  try {
    if (prefix.length() > 0 && !prefix.endsWith(PATH_DELIMITER)) {
      prefix += PATH_DELIMITER;
    }
    S3ObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(),
        prefix, delimiter, maxListingLength, priorLastKey);
    
    FileMetadata[] fileMetadata =
      new FileMetadata[chunk.getObjects().length];
    for (int i = 0; i < fileMetadata.length; i++) {
      S3Object object = chunk.getObjects()[i];
      fileMetadata[i] = new FileMetadata(object.getKey(),
          object.getContentLength(), object.getLastModifiedDate().getTime());
    }
    return new PartialListing(chunk.getPriorLastKey(), fileMetadata,
        chunk.getCommonPrefixes());
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}
 

public Set<Path> listDeepSubPaths(Path path) throws IOException {
  try {
    String prefix = pathToKey(path);
    if (!prefix.endsWith(PATH_DELIMITER)) {
      prefix += PATH_DELIMITER;
    }
    S3Object[] objects = s3Service.listObjects(bucket, prefix, null);
    Set<Path> prefixes = new TreeSet<Path>();
    for (int i = 0; i < objects.length; i++) {
      prefixes.add(keyToPath(objects[i].getKey()));
    }
    prefixes.remove(path);
    return prefixes;
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }    
}
 

private void put(String key, InputStream in, long length, boolean storeMetadata)
    throws IOException {
  
  try {
    S3Object object = new S3Object(key);
    object.setDataInputStream(in);
    object.setContentType("binary/octet-stream");
    object.setContentLength(length);
    if (storeMetadata) {
      object.addAllMetadata(METADATA);
    }
    s3Service.putObject(bucket, object);
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}
 

@Test
public void givenStringUploaded_StringIsDownloaded() throws Exception {

    // Get a bucket
    S3Bucket bucket = createBucket();
    assertNotNull(bucket);

    uploadStringData();

    // Download
    S3Object stringObject = s3Service.getObject(BucketName, TestStringName);

    // Process stream into a string
    String downloadedString = new BufferedReader(new InputStreamReader(stringObject.getDataInputStream())).lines().collect(Collectors.joining("\n"));

    // Verify
    assertTrue(TestString.equals(downloadedString));


    // Clean up for next test
    deleteObject(TestStringName);
    deleteBucket();
}
 

public void dump() throws IOException {
  StringBuilder sb = new StringBuilder("S3 Native Filesystem, ");
  sb.append(bucket.getName()).append("\n");
  try {
    S3Object[] objects = s3Service.listObjects(bucket);
    for (int i = 0; i < objects.length; i++) {
      sb.append(objects[i].getKey()).append("\n");
    }
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
  System.out.println(sb);
}
 

@Override
public Set<Path> listAllPaths() throws IOException {
  try {
    String prefix = urlEncode(Path.SEPARATOR);
    S3Object[] objects = s3Service.listObjects(bucket.getName(), prefix, null);
    Set<Path> prefixes = new TreeSet<Path>();
    for (int i = 0; i < objects.length; i++) {
      prefixes.add(keyToPath(objects[i].getKey()));
    }
    return prefixes;
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }   
}
 

public void storeLargeFile(String key, File file, byte[] md5Hash)
    throws IOException {
  S3Object object = new S3Object(key);
  object.setDataInputFile(file);
  object.setContentType("binary/octet-stream");
  object.setContentLength(file.length());
  object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
  if (md5Hash != null) {
    object.setMd5Hash(md5Hash);
  }

  List<StorageObject> objectsToUploadAsMultipart =
      new ArrayList<StorageObject>();
  objectsToUploadAsMultipart.add(object);
  MultipartUtils mpUtils = new MultipartUtils(multipartBlockSize);

  try {
    mpUtils.uploadObjects(bucket.getName(), s3Service,
                          objectsToUploadAsMultipart, null);
  } catch (Exception e) {
    handleException(e, key);
  }
}
 

/**
 *
 * @param key
 * The key is the object name that is being retrieved from the S3 bucket
 * @return
 * This method returns null if the key is not found
 * @throws IOException
 */

@Override
public InputStream retrieve(String key, long byteRangeStart)
        throws IOException {
  try {
    LOG.debug("Getting key: {} from bucket: {} with byteRangeStart: {}",
        key, bucket.getName(), byteRangeStart);
    S3Object object = s3Service.getObject(bucket, key, null, null, null,
                                          null, byteRangeStart, null);
    return object.getDataInputStream();
  } catch (ServiceException e) {
    handleException(e, key);
    return null;
  }
}
 

@Override
public void copy(String srcKey, String dstKey) throws IOException {
  try {
    if(LOG.isDebugEnabled()) {
      LOG.debug("Copying srcKey: " + srcKey + "to dstKey: " + dstKey + "in bucket: " + bucket.getName());
    }
    if (multipartEnabled) {
      S3Object object = s3Service.getObjectDetails(bucket, srcKey, null,
                                                   null, null, null);
      if (multipartCopyBlockSize > 0 &&
          object.getContentLength() > multipartCopyBlockSize) {
        copyLargeFile(object, dstKey);
        return;
      }
    }

    S3Object dstObject = new S3Object(dstKey);
    dstObject.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
    s3Service.copyObject(bucket.getName(), srcKey, bucket.getName(),
        dstObject, false);
  } catch (ServiceException e) {
    handleException(e, srcKey);
  }
}
 

public void dump() throws IOException {
  StringBuilder sb = new StringBuilder("S3 Filesystem, ");
  sb.append(bucket.getName()).append("\n");
  try {
    S3Object[] objects = s3Service.listObjects(bucket, PATH_DELIMITER, null);
    for (int i = 0; i < objects.length; i++) {
      Path path = keyToPath(objects[i].getKey());
      sb.append(path).append("\n");
      INode m = retrieveINode(path);
      sb.append("\t").append(m.getFileType()).append("\n");
      if (m.getFileType() == FileType.DIRECTORY) {
        continue;
      }
      for (int j = 0; j < m.getBlocks().length; j++) {
        sb.append("\t").append(m.getBlocks()[j]).append("\n");
      }
    }
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
  System.out.println(sb);
}
 

private void checkMetadata(S3Object object) throws S3FileSystemException,
    S3ServiceException {
  
  String name = (String) object.getMetadata(FILE_SYSTEM_NAME);
  if (!FILE_SYSTEM_VALUE.equals(name)) {
    throw new S3FileSystemException("Not a Hadoop S3 file.");
  }
  String type = (String) object.getMetadata(FILE_SYSTEM_TYPE_NAME);
  if (!FILE_SYSTEM_TYPE_VALUE.equals(type)) {
    throw new S3FileSystemException("Not a block file.");
  }
  String dataVersion = (String) object.getMetadata(FILE_SYSTEM_VERSION_NAME);
  if (!FILE_SYSTEM_VERSION_VALUE.equals(dataVersion)) {
    throw new VersionMismatchException(FILE_SYSTEM_VERSION_VALUE,
        dataVersion);
  }
}
 

@Override
public Set<Path> listSubPaths(Path path) throws IOException {
  try {
    String prefix = pathToKey(path);
    if (!prefix.endsWith(PATH_DELIMITER)) {
      prefix += PATH_DELIMITER;
    }
    S3Object[] objects = s3Service.listObjects(bucket.getName(), prefix, PATH_DELIMITER);
    Set<Path> prefixes = new TreeSet<Path>();
    for (int i = 0; i < objects.length; i++) {
      prefixes.add(keyToPath(objects[i].getKey()));
    }
    prefixes.remove(path);
    return prefixes;
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}
 

@Override
public Set<Path> listDeepSubPaths(Path path) throws IOException {
  try {
    String prefix = pathToKey(path);
    if (!prefix.endsWith(PATH_DELIMITER)) {
      prefix += PATH_DELIMITER;
    }
    S3Object[] objects = s3Service.listObjects(bucket.getName(), prefix, null);
    Set<Path> prefixes = new TreeSet<Path>();
    for (int i = 0; i < objects.length; i++) {
      prefixes.add(keyToPath(objects[i].getKey()));
    }
    prefixes.remove(path);
    return prefixes;
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }    
}
 

private void put(String key, InputStream in, long length, boolean storeMetadata)
    throws IOException {
  
  try {
    S3Object object = new S3Object(key);
    object.setDataInputStream(in);
    object.setContentType("binary/octet-stream");
    object.setContentLength(length);
    if (storeMetadata) {
      object.addAllMetadata(METADATA);
    }
    s3Service.putObject(bucket, object);
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}
 

@Override
public void dump() throws IOException {
  StringBuilder sb = new StringBuilder("S3 Filesystem, ");
  sb.append(bucket.getName()).append("\n");
  try {
    S3Object[] objects = s3Service.listObjects(bucket.getName(), PATH_DELIMITER, null);
    for (int i = 0; i < objects.length; i++) {
      Path path = keyToPath(objects[i].getKey());
      sb.append(path).append("\n");
      INode m = retrieveINode(path);
      sb.append("\t").append(m.getFileType()).append("\n");
      if (m.getFileType() == FileType.DIRECTORY) {
        continue;
      }
      for (int j = 0; j < m.getBlocks().length; j++) {
        sb.append("\t").append(m.getBlocks()[j]).append("\n");
      }
    }
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
  System.out.println(sb);
}
 

@Override
public Set<Path> listAllPaths() throws IOException {
  try {
    String prefix = urlEncode(Path.SEPARATOR);
    S3Object[] objects = s3Service.listObjects(bucket.getName(), prefix, null);
    Set<Path> prefixes = new TreeSet<Path>();
    for (int i = 0; i < objects.length; i++) {
      prefixes.add(keyToPath(objects[i].getKey()));
    }
    return prefixes;
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }   
}
 

private InputStream get(String key, boolean checkMetadata)
    throws IOException {
  
  try {
    S3Object object = s3Service.getObject(bucket, key);
    if (checkMetadata) {
      checkMetadata(object);
    }
    return object.getDataInputStream();
  } catch (S3ServiceException e) {
    if ("NoSuchKey".equals(e.getS3ErrorCode())) {
      return null;
    }
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}
 

public byte[] get(String path) {
    try {
        S3Object obj = m_s3service.getObject(BUCKET, path);
        m_logger.trace("GetObject: {}", path);
        inc();
        if(obj == null) return null;
        int len = (int)obj.getContentLength();
        byte[] data = new byte[len];
        int start = 0;
        InputStream is = obj.getDataInputStream();
        while(start < len) {
            start += is.read(data, start, len - start);
        }
        is.close();
        return data;
    } catch (Exception e) {
        return null;
    }
}
 

public FileMetadata retrieveMetadata(String key) throws IOException {
  try {
    S3Object object = s3Service.getObjectDetails(bucket, key);
    return new FileMetadata(key, object.getContentLength(),
        object.getLastModifiedDate().getTime());
  } catch (S3ServiceException e) {
    // Following is brittle. Is there a better way?
    if (e.getMessage().contains("ResponseCode=404")) {
      return null;
    }
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
}