类com.amazonaws.services.s3.model.S3ObjectSummary源码实例Demo

下面列出了怎么用com.amazonaws.services.s3.model.S3ObjectSummary的API类实例代码及写法,或者点击链接到github查看源代码。

源代码1 项目: herd   文件: S3DaoTest.java
@Test
public void testTagObjectsAmazonServiceException()
{
    // Create an S3 file transfer request parameters DTO to access S3 objects with a mocked S3 bucket name that would trigger an AWS exception.
    S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
    params.setS3BucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_INTERNAL_ERROR);

    // Create an S3 object summary.
    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setKey(TARGET_S3_KEY);

    // Create an S3 object tag.
    Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);

    try
    {
        s3Dao.tagObjects(params, new S3FileTransferRequestParamsDto(), Collections.singletonList(s3ObjectSummary), tag);
        fail();
    }
    catch (IllegalStateException e)
    {
        assertEquals(String.format("Failed to tag S3 object with \"%s\" key and \"null\" version id in \"%s\" bucket. " +
                "Reason: InternalError (Service: null; Status Code: 0; Error Code: InternalError; Request ID: null)", TARGET_S3_KEY,
            MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_INTERNAL_ERROR), e.getMessage());
    }
}
 
源代码2 项目: s3proxy   文件: AwsSdkTest.java
@Test
public void testUnicodeObject() throws Exception {
    String blobName = "ŪņЇЌœđЗ/☺ unicode € rocks ™";
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    client.putObject(containerName, blobName, BYTE_SOURCE.openStream(),
            metadata);

    metadata = client.getObjectMetadata(containerName, blobName);
    assertThat(metadata).isNotNull();

    ObjectListing listing = client.listObjects(containerName);
    List<S3ObjectSummary> summaries = listing.getObjectSummaries();
    assertThat(summaries).hasSize(1);
    S3ObjectSummary summary = summaries.iterator().next();
    assertThat(summary.getKey()).isEqualTo(blobName);
}
 
源代码3 项目: herd   文件: StorageFileHelperTest.java
@Test
public void testValidateRegisteredS3FilesUnexpectedNonEmptyS3FileFound() throws IOException
{
    // Create two lists of expected and actual storage files, with an actual file not being added to the list of expected files.
    List<StorageFile> testExpectedFiles = new ArrayList<>();
    List<S3ObjectSummary> testActualFiles = Collections.singletonList(createS3ObjectSummary(TARGET_S3_KEY, FILE_SIZE_1_KB));

    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey =
        new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES,
            DATA_VERSION);

    // Try to validate S3 files when unexpected non-empty S3 file exists.
    // The validation is expected to fail when detecting an unregistered non-empty S3 file.
    try
    {
        storageFileHelper.validateRegisteredS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME, businessObjectDataKey);
        fail();
    }
    catch (IllegalStateException e)
    {
        assertEquals(String.format("Found unregistered non-empty S3 file \"%s\" in \"%s\" storage. Business object data {%s}", TARGET_S3_KEY, STORAGE_NAME,
            businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataKeyAsString(businessObjectDataKey)), e.getMessage());
    }
}
 
源代码4 项目: nifi   文件: ListS3.java
@Override
public VersionListing listVersions() {
    VersionListing versionListing = new VersionListing();
    this.objectListing = client.listObjects(listObjectsRequest);
    for(S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        S3VersionSummary versionSummary = new S3VersionSummary();
        versionSummary.setBucketName(objectSummary.getBucketName());
        versionSummary.setETag(objectSummary.getETag());
        versionSummary.setKey(objectSummary.getKey());
        versionSummary.setLastModified(objectSummary.getLastModified());
        versionSummary.setOwner(objectSummary.getOwner());
        versionSummary.setSize(objectSummary.getSize());
        versionSummary.setStorageClass(objectSummary.getStorageClass());
        versionSummary.setIsLatest(true);

        versionListing.getVersionSummaries().add(versionSummary);
    }

    return versionListing;
}
 
源代码5 项目: crate   文件: MockAmazonS3.java
@Override
public ObjectListing listObjects(final ListObjectsRequest request) throws AmazonClientException {
    assertThat(request.getBucketName(), equalTo(bucket));

    final ObjectListing listing = new ObjectListing();
    listing.setBucketName(request.getBucketName());
    listing.setPrefix(request.getPrefix());

    for (Map.Entry<String, byte[]> blob : blobs.entrySet()) {
        if (Strings.isEmpty(request.getPrefix()) || blob.getKey().startsWith(request.getPrefix())) {
            S3ObjectSummary summary = new S3ObjectSummary();
            summary.setBucketName(request.getBucketName());
            summary.setKey(blob.getKey());
            summary.setSize(blob.getValue().length);
            listing.getObjectSummaries().add(summary);
        }
    }
    return listing;
}
 
源代码6 项目: aws-ant-tasks   文件: AWSTestUtils.java
public static void emptyAndDeleteBucket(AmazonS3Client client,
        String bucketName) {
    ObjectListing objectListing = client.listObjects(bucketName);

    while (true) {
        for (Iterator<?> iterator = objectListing.getObjectSummaries()
                .iterator(); iterator.hasNext();) {
            S3ObjectSummary objectSummary = (S3ObjectSummary) iterator
                    .next();
            client.deleteObject(bucketName, objectSummary.getKey());
        }

        if (objectListing.isTruncated()) {
            objectListing = client.listNextBatchOfObjects(objectListing);
        } else {
            break;
        }
    }
    client.deleteBucket(bucketName);
}
 
源代码7 项目: cantor   文件: ObjectsOnS3.java
private Collection<String> getKeys(final String bucket, final int start, final int count) {
    final Set<String> keys = new HashSet<>();
    int index = 0;
    ObjectListing listing = this.s3Client.listObjects(bucket);
    do {
        for (final S3ObjectSummary summary : listing.getObjectSummaries()) {
            if (index < start) {
                logger.debug("skipping {} at index={} start={}", summary.getKey(), index++, start);
                continue;
            }
            keys.add(summary.getKey());

            if (keys.size() == count) {
                logger.debug("retrieved {}/{} keys, returning early", keys.size(), count);
                return keys;
            }
        }

        logger.debug("got {} keys from {}", listing.getObjectSummaries().size(), listing);
        listing = this.s3Client.listNextBatchOfObjects(listing);
    } while (listing.isTruncated());

    return keys;
}
 
源代码8 项目: localization_nifi   文件: ListS3.java
@Override
public VersionListing listVersions() {
    VersionListing versionListing = new VersionListing();
    this.objectListing = client.listObjects(listObjectsRequest);
    for(S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        S3VersionSummary versionSummary = new S3VersionSummary();
        versionSummary.setBucketName(objectSummary.getBucketName());
        versionSummary.setETag(objectSummary.getETag());
        versionSummary.setKey(objectSummary.getKey());
        versionSummary.setLastModified(objectSummary.getLastModified());
        versionSummary.setOwner(objectSummary.getOwner());
        versionSummary.setSize(objectSummary.getSize());
        versionSummary.setStorageClass(objectSummary.getStorageClass());
        versionSummary.setIsLatest(true);

        versionListing.getVersionSummaries().add(versionSummary);
    }

    return versionListing;
}
 
源代码9 项目: herd   文件: S3DaoTest.java
@Test
public void testTagObjects()
{
    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
    params.setS3BucketName(S3_BUCKET_NAME);

    // Create an S3 object summary.
    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setKey(TARGET_S3_KEY);

    // Create an S3 object tag.
    Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);

    // Put a file in S3.
    s3Operations.putObject(new PutObjectRequest(S3_BUCKET_NAME, TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), new ObjectMetadata()), null);

    // Tag the file with an S3 object tag.
    s3Dao.tagObjects(params, new S3FileTransferRequestParamsDto(), Collections.singletonList(s3ObjectSummary), tag);

    // Validate that the object got tagged.
    GetObjectTaggingResult getObjectTaggingResult = s3Operations.getObjectTagging(new GetObjectTaggingRequest(S3_BUCKET_NAME, TARGET_S3_KEY), null);
    assertEquals(Collections.singletonList(tag), getObjectTaggingResult.getTagSet());
}
 
源代码10 项目: jobcacher-plugin   文件: S3Profile.java
public void delete(String bucketName, String pathPrefix) {
    ObjectListing listing = null;
    do {
        listing = listing == null ? helper.client().listObjects(bucketName, pathPrefix) : helper.client().listNextBatchOfObjects(listing);

        DeleteObjectsRequest req = new DeleteObjectsRequest(bucketName);

        List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<>(listing.getObjectSummaries().size());
        for (S3ObjectSummary summary : listing.getObjectSummaries()) {
            keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
        }
        req.withKeys(keys);

        helper.client().deleteObjects(req);
    } while (listing.isTruncated());
}
 
源代码11 项目: herd   文件: S3ServiceTest.java
@Test
public void testTagObjects()
{
    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();

    // Create an S3 file transfer request parameters DTO to tag S3 objects.
    S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    s3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    s3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    s3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);

    // Create an S3 object summary.
    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setKey(S3_KEY);

    // Create an S3 object tag.
    Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);

    // Call the method under test.
    s3Service.tagObjects(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, Collections.singletonList(s3ObjectSummary), tag);

    // Verify the external calls.
    verify(s3Dao).tagObjects(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, Collections.singletonList(s3ObjectSummary), tag);
    verifyNoMoreInteractions(s3Dao);
}
 
源代码12 项目: pipeline-aws-plugin   文件: S3FindFilesStep.java
/**
 * This creates a new FileWrapper instance based on the S3ObjectSummary information.
 *
 * @param pathComponentCount The root path component count.
 * @param javaPath           The Path instance for the file.
 * @param entry              The S3ObjectSummary for the file.
 * @return A new FileWrapper instance.
 */
public static FileWrapper createFileWrapperFromFile(int pathComponentCount, Path javaPath, S3ObjectSummary entry) {
	Path pathFileName = javaPath.getFileName();
	if (pathFileName == null) {
		return null;
	}
	return new FileWrapper(
			// Name:
			convertPathToAwsFormat(pathFileName),
			// Path (relative to the `path` parameter):
			convertPathToAwsFormat(javaPath.subpath(pathComponentCount, javaPath.getNameCount())),
			// Directory?
			false,
			// Size:
			entry.getSize(),
			// Last modified (milliseconds):
			entry.getLastModified().getTime()
	);
}
 
源代码13 项目: ignite   文件: S3CheckpointSpiSelfTest.java
/**
 * @throws Exception If error.
 */
@Override protected void afterSpiStopped() throws Exception {
    AWSCredentials cred = new BasicAWSCredentials(IgniteS3TestSuite.getAccessKey(),
        IgniteS3TestSuite.getSecretKey());

    AmazonS3 s3 = new AmazonS3Client(cred);

    String bucketName = S3CheckpointSpi.BUCKET_NAME_PREFIX + "unit-test-bucket";

    try {
        ObjectListing list = s3.listObjects(bucketName);

        while (true) {
            for (S3ObjectSummary sum : list.getObjectSummaries())
                s3.deleteObject(bucketName, sum.getKey());

            if (list.isTruncated())
                list = s3.listNextBatchOfObjects(list);
            else
                break;
        }
    }
    catch (AmazonClientException e) {
        throw new IgniteSpiException("Failed to read checkpoint bucket: " + bucketName, e);
    }
}
 
源代码14 项目: herd   文件: S3ServiceTest.java
@Test
public void testListDirectoryIgnoreZeroByteDirectoryMarkers()
{
    // Create an S3 file transfer request parameters DTO.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();

    // Create a list of S3 object summaries.
    List<S3ObjectSummary> s3ObjectSummaries = Collections.singletonList(new S3ObjectSummary());

    // Mock the external calls.
    when(s3Dao.listDirectory(s3FileTransferRequestParamsDto, true)).thenReturn(s3ObjectSummaries);

    // Call the method under test.
    List<S3ObjectSummary> result = s3Service.listDirectory(s3FileTransferRequestParamsDto, true);

    // Verify the external calls.
    verify(s3Dao).listDirectory(s3FileTransferRequestParamsDto, true);
    verifyNoMoreInteractions(s3Dao);

    // Validate the returned object.
    assertEquals(s3ObjectSummaries, result);
}
 
源代码15 项目: incubator-gobblin   文件: AWSSdkClient.java
/***
 * Download a S3 object to local directory
 *
 * @param s3ObjectSummary S3 object summary for the object to download
 * @param targetDirectory Local target directory to download the object to
 * @throws IOException If any errors were encountered in downloading the object
 */
public void downloadS3Object(S3ObjectSummary s3ObjectSummary,
    String targetDirectory)
    throws IOException {

  final AmazonS3 amazonS3 = getS3Client();

  final GetObjectRequest getObjectRequest = new GetObjectRequest(
      s3ObjectSummary.getBucketName(),
      s3ObjectSummary.getKey());

  final S3Object s3Object = amazonS3.getObject(getObjectRequest);

  final String targetFile = StringUtils.removeEnd(targetDirectory, File.separator) + File.separator + s3Object.getKey();
  FileUtils.copyInputStreamToFile(s3Object.getObjectContent(), new File(targetFile));

  LOGGER.info("S3 object downloaded to file: " + targetFile);
}
 
源代码16 项目: emodb   文件: StashReaderTest.java
private Answer<ObjectListing> objectListingAnswer(@Nullable final String marker, final String... fileNames) {
    return new Answer<ObjectListing>() {
        @Override
        public ObjectListing answer(InvocationOnMock invocation)
                throws Throwable {
            ListObjectsRequest request = (ListObjectsRequest) invocation.getArguments()[0];

            ObjectListing objectListing = new ObjectListing();
            objectListing.setBucketName(request.getBucketName());
            objectListing.setPrefix(request.getPrefix());

            objectListing.setTruncated(marker != null);
            objectListing.setNextMarker(marker);

            for (String fileName : fileNames) {
                S3ObjectSummary objectSummary = new S3ObjectSummary();
                objectSummary.setKey(request.getPrefix() + fileName);
                objectSummary.setSize(100);
                objectListing.getObjectSummaries().add(objectSummary);
            }

            return objectListing;
        }
    };
}
 
源代码17 项目: hawkbit-extensions   文件: S3Repository.java
@Override
public void deleteByTenant(final String tenant) {
    final String folder = sanitizeTenant(tenant);

    LOG.info("Deleting S3 object folder (tenant) from bucket {} and key {}", s3Properties.getBucketName(), folder);

    // Delete artifacts
    ObjectListing objects = amazonS3.listObjects(s3Properties.getBucketName(), folder + "/");
    do {
        for (final S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
            amazonS3.deleteObject(s3Properties.getBucketName(), objectSummary.getKey());
        }
        objects = amazonS3.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());

}
 
源代码18 项目: netcdf-java   文件: CrawlableDatasetAmazonS3.java
/**
 * Returns the size of the dataset, in bytes. Will be zero if this dataset is a collection or non-existent.
 *
 * @return the size of the dataset
 */
@Override
public long length() {
  // If the summary is already in the cache, return it.
  // It'll have been added by a listDatasets() call on the parent directory.
  S3ObjectSummary objectSummary = objectSummaryCache.getIfPresent(s3uri);
  if (objectSummary != null) {
    return objectSummary.getSize();
  }

  /*
   * Get the metadata directly from S3. This will be expensive.
   * We get punished hard if length() and/or lastModified() is called on a bunch of datasets without
   * listDatasets() first being called on their parent directory.
   *
   * So, is the right thing to do here "getParentDataset().listDatasets()" and then query the cache again?
   * Perhaps, but listDatasets() throws an IOException, and length() and lastModified() do not.
   * We would have to change their signatures and the upstream client code to make it work.
   */
  ObjectMetadata metadata = threddsS3Client.getObjectMetadata(s3uri);
  if (metadata != null) {
    return metadata.getContentLength();
  } else {
    // "this" may be a collection or non-existent. In both cases, we return 0.
    return 0;
  }
}
 
源代码19 项目: datacollector   文件: S3FileRef.java
@SuppressWarnings("unchecked")
public S3FileRef(
    AmazonS3 s3Client,
    S3ObjectSummary s3ObjectSummary,
    boolean useSSE,
    CredentialValue customerKey,
    CredentialValue customerKeyMd5,
    int bufferSize,
    boolean createMetrics,
    long totalSizeInBytes,
    double rateLimit,
    boolean verifyChecksum,
    String checksum,
    HashingUtil.HashType checksumAlgorithm) {
  super(
      ImmutableSet.<Class<? extends AutoCloseable>>of(InputStream.class),
      bufferSize,
      createMetrics,
      totalSizeInBytes,
      rateLimit,
      verifyChecksum,
      checksum,
      checksumAlgorithm
  );
  this.s3Client = s3Client;
  this.s3ObjectSummary = s3ObjectSummary;
  this.useSSE = useSSE;
  this.customerKey = customerKey;
  this.customerKeyMd5 = customerKeyMd5;
}
 
源代码20 项目: s3proxy   文件: AwsSdkTest.java
@Test
public void testBlobListRecursive() throws Exception {
    ObjectListing listing = client.listObjects(containerName);
    assertThat(listing.getObjectSummaries()).isEmpty();

    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    client.putObject(containerName, "prefix/blob1",
            BYTE_SOURCE.openStream(), metadata);
    client.putObject(containerName, "prefix/blob2",
            BYTE_SOURCE.openStream(), metadata);

    ImmutableList.Builder<String> builder = ImmutableList.builder();
    listing = client.listObjects(new ListObjectsRequest()
            .withBucketName(containerName)
            .withDelimiter("/"));
    assertThat(listing.getObjectSummaries()).isEmpty();
    for (String prefix : listing.getCommonPrefixes()) {
        builder.add(prefix);
    }
    assertThat(builder.build()).containsOnly("prefix/");

    builder = ImmutableList.builder();
    listing = client.listObjects(containerName);
    for (S3ObjectSummary summary : listing.getObjectSummaries()) {
        builder.add(summary.getKey());
    }
    assertThat(builder.build()).containsOnly("prefix/blob1",
            "prefix/blob2");
    assertThat(listing.getCommonPrefixes()).isEmpty();
}
 
源代码21 项目: presto   文件: S3TableConfigClient.java
/**
 * Connect to S3 directory to look for new or updated table definitions and then
 * update the map.
 */
private void updateTablesFromS3()
{
    long now = System.currentTimeMillis();

    AmazonS3Client s3client = clientManager.getS3Client();

    for (S3ObjectSummary summary : getObjectSummaries()) {
        if (!descriptors.containsKey(summary.getKey()) || summary.getLastModified().getTime() >= lastCheck) {
            // New or updated file, so we must read from AWS
            if (summary.getKey().endsWith("/")) {
                continue;
            }

            log.info("Getting : %s - %s", summary.getBucketName(), summary.getKey());
            S3Object object = s3client.getObject(new GetObjectRequest(summary.getBucketName(), summary.getKey()));

            try (BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent(), UTF_8))) {
                KinesisStreamDescription table = streamDescriptionCodec.fromJson(CharStreams.toString(reader));
                descriptors.put(summary.getKey(), table);
                log.info("Put table description into the map from %s", summary.getKey());
            }
            catch (IOException iox) {
                log.error("Problem reading input stream from object.", iox);
                throwIfUnchecked(iox);
                throw new RuntimeException(iox);
            }
        }
    }

    log.info("Completed updating table definitions from S3.");
    lastCheck = now;
}
 
源代码22 项目: athenz   文件: S3ChangeLogStoreTest.java
@Test
public void testGetAllSignedDomainsException() throws FileNotFoundException {
    MockS3ChangeLogStore store = new MockS3ChangeLogStore(null, 1);

    InputStream is1 = new FileInputStream("src/test/resources/iaas.json");
    MockS3ObjectInputStream s3Is1 = new MockS3ObjectInputStream(is1, null);

    InputStream is2 = new FileInputStream("src/test/resources/iaas.json");
    MockS3ObjectInputStream s3Is2 = new MockS3ObjectInputStream(is2, null);

    S3Object object = mock(S3Object.class);
    when(object.getObjectContent()).thenReturn(s3Is1).thenReturn(s3Is2);

    when(store.awsS3Client.getObject("s3-unit-test-bucket-name", "iaas")).thenReturn(object);
    ObjectListing mockObjectListing = mock(ObjectListing.class);
    when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(mockObjectListing);
    List<S3ObjectSummary> tempList = new ArrayList<>();
    S3ObjectSummary s3ObjectSummary = mock(S3ObjectSummary.class);
    when(s3ObjectSummary.getKey()).thenReturn("iaas");
    tempList.add(s3ObjectSummary);
    when(mockObjectListing.getObjectSummaries()).thenReturn(tempList);


    List<String> temp = new LinkedList<>();
    temp.add("iaas");

    try {
        when(store.executorService.awaitTermination(defaultTimeoutSeconds, TimeUnit.SECONDS)).thenThrow(new InterruptedException());
        assertFalse(store.getAllSignedDomains(temp));
        assertTrue(store.getLocalDomainList().size() > 0);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
}
 
源代码23 项目: tutorials   文件: SpringCloudS3LiveTest.java
@AfterClass
public static void cleanUpResources() {
    AmazonS3 amazonS3 = SpringCloudAwsTestUtil.amazonS3();
    ListObjectsV2Result listObjectsV2Result = amazonS3.listObjectsV2(bucketName);
    for (S3ObjectSummary objectSummary : listObjectsV2Result.getObjectSummaries()) {
        amazonS3.deleteObject(bucketName, objectSummary.getKey());
    }
    amazonS3.deleteBucket(bucketName);

    new File(testFileToDownload).delete();
    new File(testFileToUpload).delete();
    similarNameFiles.forEach(File::delete);
}
 
源代码24 项目: terrapin   文件: TerrapinUtilTest.java
@Test
@PrepareForTest(TerrapinUtil.class)
public void testGetS3FileList() throws Exception {
  AmazonS3Client s3Client = mock(AmazonS3Client.class);
  ObjectListing objectListing = mock(ObjectListing.class);
  S3ObjectSummary summary1 = new S3ObjectSummary();
  S3ObjectSummary summary2 = new S3ObjectSummary();
  S3ObjectSummary summary3 = new S3ObjectSummary();
  summary1.setKey("/abc/123");
  summary2.setKey("/abc/456");
  summary3.setKey("/def/123");
  summary1.setSize(32432);
  summary2.setSize(213423);
  summary3.setSize(2334);
  List<S3ObjectSummary> summaries = ImmutableList.of(summary1, summary2, summary3);
  whenNew(AmazonS3Client.class).withAnyArguments().thenReturn(s3Client);
  when(s3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
  when(objectListing.getObjectSummaries()).thenReturn(summaries);

  List<Pair<Path, Long>> results = TerrapinUtil.getS3FileList(new AnonymousAWSCredentials(),
      "bucket", "/abc");

  assertEquals(2, results.size());
  assertTrue(results.get(0).getLeft().toString().endsWith(summary1.getKey()));
  assertEquals(new Long(summary1.getSize()), results.get(0).getRight());
  assertTrue(results.get(1).getLeft().toString().endsWith(summary2.getKey()));
  assertEquals(new Long(summary2.getSize()), results.get(1).getRight());
}
 
源代码25 项目: crate   文件: S3FileInput.java
private void addKeyUris(List<URI> uris, ObjectListing list, URI uri, Predicate<URI> uriPredicate) {
    List<S3ObjectSummary> summaries = list.getObjectSummaries();
    for (S3ObjectSummary summary : summaries) {
        String key = summary.getKey();
        if (!key.endsWith("/")) {
            URI keyUri = uri.resolve("/" + key);
            if (uriPredicate.test(keyUri)) {
                uris.add(keyUri);
                if (LOGGER.isDebugEnabled()) {
                    LOGGER.debug("{}", keyUri);
                }
            }
        }
    }
}
 
源代码26 项目: singleton   文件: S3ProductDaoImpl.java
/**
 * get locale list from S3 server
 */
@Override
public List<String> getLocaleList(String productName, String version) throws DataException {
   List<String> localeList = new ArrayList<String>();
   String filePathPrefix = S3Utils.genProductVersionS3Path(productName, version);
   ListObjectsV2Result result =
         s3Client.getS3Client().listObjectsV2(config.getBucketName(), S3Utils.S3_L10N_BUNDLES_PATH);
   if (result == null) {
      throw new DataException("Can't find S3 resource from " + productName + "\\" + version);
   }
   List<S3ObjectSummary> objects = result.getObjectSummaries();
   if (objects == null || objects.size() < 1) {
      throw new DataException("S3 Component list is empty.");
   }
   for (S3ObjectSummary s3os : objects) {
      String s3obKey = s3os.getKey().replace(filePathPrefix, "");
      if((!s3obKey.startsWith(ConstantsFile.CREATION_INFO)) && (!s3obKey.startsWith(ConstantsFile.VERSION_FILE))) {
         String resultKey =s3obKey.split(ConstantsChar.BACKSLASH)[1];
         String localeKey = S3Utils.getLocaleByFileName(resultKey);
         if (localeKey != null && !localeList.contains(localeKey)) {
            localeList.add(localeKey);
         }
         
      }
      
      
   }
   return localeList;
}
 
源代码27 项目: herd   文件: UploaderController.java
/**
 * Logs all files found in the specified S3 location.
 *
 * @param params the S3 file transfer request parameters
 */
private void logS3KeyPrefixContents(S3FileTransferRequestParamsDto params)
{
    List<S3ObjectSummary> s3ObjectSummaries = s3Service.listDirectory(params);
    LOGGER.info(
        String.format("Found %d keys with prefix \"%s\" in bucket \"%s\":", s3ObjectSummaries.size(), params.getS3KeyPrefix(), params.getS3BucketName()));

    for (S3ObjectSummary s3ObjectSummary : s3ObjectSummaries)
    {
        LOGGER.info(String.format("    s3://%s/%s", params.getS3BucketName(), s3ObjectSummary.getKey()));
    }
}
 
源代码28 项目: iaf   文件: AmazonS3FileSystem.java
@Override
public boolean folderExists(String folder) throws FileSystemException {
	ObjectListing objectListing = s3Client.listObjects(bucketName);
	Iterator<S3ObjectSummary> objIter = objectListing.getObjectSummaries().iterator();
	while (objIter.hasNext()) {
		S3ObjectSummary s3ObjectSummary = objIter.next();
		String key = s3ObjectSummary.getKey();
		if(key.endsWith("/") && key.equals(folder+"/")){
			return true;
		}
	}
	return false;
}
 
源代码29 项目: datacollector   文件: TestUtil.java
public static void createBucket(AmazonS3 s3client, String bucketName) {
  if(s3client.doesBucketExist(bucketName)) {
    for(S3ObjectSummary s : S3Objects.inBucket(s3client, bucketName)) {
      s3client.deleteObject(bucketName, s.getKey());
    }
    s3client.deleteBucket(bucketName);
  }
  Assert.assertFalse(s3client.doesBucketExist(bucketName));
  // Note that CreateBucketRequest does not specify region. So bucket is
  // bucketName
  s3client.createBucket(new CreateBucketRequest(bucketName));
}
 
源代码30 项目: Scribengin   文件: S3Folder.java
public List<String> getChildrenNames() {
  List<String> holder = new ArrayList<String>() ;
  for(S3ObjectSummary sel : getChildren()) {
    String name = sel.getKey().substring(folderPath.length() + 1);
    holder.add(name);
  }
  return holder;
}
 
 同包方法