类com.amazonaws.services.s3.model.ListObjectsRequest源码实例Demo

下面列出了怎么用com.amazonaws.services.s3.model.ListObjectsRequest的API类实例代码及写法,或者点击链接到github查看源代码。

源代码1 项目: sequenceiq-samples   文件: S3Source.java
@Override
protected void doStart() {
    AWSCredentials myCredentials = new BasicAWSCredentials(accessKey, secretKey);
    AmazonS3 s3Client = new AmazonS3Client(myCredentials);
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket);
    ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);
    ChannelProcessor channelProcessor = getChannelProcessor();
    for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) {
        String file = s3ObjectSummary.getKey();
        LOGGER.info("Read the content of {}", file);
        GetObjectRequest objectRequest = new GetObjectRequest(bucket, file);
        S3Object objectPortion = s3Client.getObject(objectRequest);
        try {
            long startTime = System.currentTimeMillis();
            processLines(channelProcessor, objectPortion.getObjectContent());
            LOGGER.info("Processing of {} took {} ms", file, System.currentTimeMillis() - startTime);
        } catch (IOException e) {
            LOGGER.warn("Cannot process the {}, skipping", file, e);
        }
    }
}
 
源代码2 项目: hop   文件: S3CommonFileObject.java
private void handleAttachExceptionFallback( String bucket, String keyWithDelimiter, AmazonS3Exception exception )
  throws FileSystemException {
  ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
    .withBucketName( bucket )
    .withPrefix( keyWithDelimiter )
    .withDelimiter( DELIMITER );
  ObjectListing ol = fileSystem.getS3Client().listObjects( listObjectsRequest );

  if ( !( ol.getCommonPrefixes().isEmpty() && ol.getObjectSummaries().isEmpty() ) ) {
    injectType( FileType.FOLDER );
  } else {
    //Folders don't really exist - they will generate a "NoSuchKey" exception
    // confirms key doesn't exist but connection okay
    String errorCode = exception.getErrorCode();
    if ( !errorCode.equals( "NoSuchKey" ) ) {
      // bubbling up other connection errors
      logger.error( "Could not get information on " + getQualifiedName(),
        exception ); // make sure this gets printed for the user
      throw new FileSystemException( "vfs.provider/get-type.error", getQualifiedName(), exception );
    }
  }
}
 
源代码3 项目: hop   文件: S3NFileObjectTest.java
@Test
public void testHandleAttachExceptionEmptyFolder() throws FileSystemException {
  AmazonS3Exception exception = new AmazonS3Exception( "NoSuchKey" );
  exception.setErrorCode( "NoSuchKey" );

  //test the case where the folder exists and contains things; no exception should be thrown
  when( s3ServiceMock.getObject( BUCKET_NAME, origKey ) ).thenThrow( exception );
  when( s3ServiceMock.getObject( BUCKET_NAME, origKey + "/" ) ).thenThrow( exception );
  childObjectListing = mock( ObjectListing.class );
  when( childObjectListing.getObjectSummaries() ).thenReturn( new ArrayList<>() );
  when( childObjectListing.getCommonPrefixes() ).thenReturn( new ArrayList<>() );
  when( s3ServiceMock.listObjects( any( ListObjectsRequest.class ) ) ).thenReturn( childObjectListing );
  try {
    s3FileObjectFileSpy.doAttach();
  } catch ( Exception e ) {
    fail( "Caught exception " + e.getMessage() );
  }
  assertEquals( FileType.IMAGINARY, s3FileObjectFileSpy.getType() );
}
 
源代码4 项目: hop   文件: S3NFileObjectTest.java
@Test
public void testHandleAttachExceptionFileNotFound() throws FileSystemException {
  AmazonS3Exception notFoundException = new AmazonS3Exception( "404 Not Found" );
  notFoundException.setErrorCode( "404 Not Found" );
  AmazonS3Exception noSuchKeyException = new AmazonS3Exception( "NoSuchKey" );
  noSuchKeyException.setErrorCode( "NoSuchKey" );

  //test the case where the file is not found; no exception should be thrown
  when( s3ServiceMock.getObject( BUCKET_NAME, origKey ) ).thenThrow( notFoundException );
  when( s3ServiceMock.getObject( BUCKET_NAME, origKey + "/" ) ).thenThrow( noSuchKeyException );
  childObjectListing = mock( ObjectListing.class );
  when( childObjectListing.getObjectSummaries() ).thenReturn( new ArrayList<>() );
  when( childObjectListing.getCommonPrefixes() ).thenReturn( new ArrayList<>() );
  when( s3ServiceMock.listObjects( any( ListObjectsRequest.class ) ) ).thenReturn( childObjectListing );
  try {
    s3FileObjectFileSpy.doAttach();
  } catch ( Exception e ) {
    fail( "Caught exception " + e.getMessage() );
  }
  assertEquals( FileType.IMAGINARY, s3FileObjectFileSpy.getType() );
}
 
源代码5 项目: circus-train   文件: S3S3CopierTest.java
@Test
public void copyMultipleObjects() throws Exception {
  // Making sure we only request 1 file at the time so we need to loop
  ListObjectsRequestFactory mockListObjectRequestFactory = Mockito.mock(ListObjectsRequestFactory.class);
  when(mockListObjectRequestFactory.newInstance()).thenReturn(new ListObjectsRequest().withMaxKeys(1));

  client.putObject("source", "bar/data1", inputData);
  client.putObject("source", "bar/data2", inputData);

  Path sourceBaseLocation = new Path("s3://source/bar/");
  Path replicaLocation = new Path("s3://target/foo/");
  List<Path> sourceSubLocations = new ArrayList<>();
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      transferManagerFactory, mockListObjectRequestFactory, registry, s3S3CopierOptions);
  Metrics metrics = s3s3Copier.copy();
  assertThat(metrics.getBytesReplicated(), is(14L));

  S3Object object1 = client.getObject("target", "foo/data1");
  String data1 = IOUtils.toString(object1.getObjectContent());
  assertThat(data1, is("bar foo"));
  S3Object object2 = client.getObject("target", "foo/data2");
  String data2 = IOUtils.toString(object2.getObjectContent());
  assertThat(data2, is("bar foo"));
}
 
源代码6 项目: emodb   文件: StashReaderTest.java
private Matcher<ListObjectsRequest> listObjectRequest(final String bucket, final String prefix, @Nullable final String marker) {
    return new BaseMatcher<ListObjectsRequest>() {
        @Override
        public boolean matches(Object item) {
            ListObjectsRequest request = (ListObjectsRequest) item;
            return request != null &&
                    request.getBucketName().equals(bucket) &&
                    request.getPrefix().equals(prefix) &&
                    Objects.equals(request.getMarker(), marker);
        }

        @Override
        public void describeTo(Description description) {
            description.appendText("ListObjectRequest[s3://").appendText(bucket).appendText("/").appendText(prefix);
            if (marker != null) {
                description.appendText(", marker=").appendText(marker);
            }
            description.appendText("]");
        }
    };
}
 
源代码7 项目: emodb   文件: StashReaderTest.java
private Answer<ObjectListing> objectListingAnswer(@Nullable final String marker, final String... fileNames) {
    return new Answer<ObjectListing>() {
        @Override
        public ObjectListing answer(InvocationOnMock invocation)
                throws Throwable {
            ListObjectsRequest request = (ListObjectsRequest) invocation.getArguments()[0];

            ObjectListing objectListing = new ObjectListing();
            objectListing.setBucketName(request.getBucketName());
            objectListing.setPrefix(request.getPrefix());

            objectListing.setTruncated(marker != null);
            objectListing.setNextMarker(marker);

            for (String fileName : fileNames) {
                S3ObjectSummary objectSummary = new S3ObjectSummary();
                objectSummary.setKey(request.getPrefix() + fileName);
                objectSummary.setSize(100);
                objectListing.getObjectSummaries().add(objectSummary);
            }

            return objectListing;
        }
    };
}
 
源代码8 项目: nifi   文件: TestListS3.java
@Test
public void testWriteObjectTags() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.WRITE_OBJECT_TAGS, "true");

    Date lastModified = new Date();
    ObjectListing objectListing = new ObjectListing();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);

    Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<GetObjectTaggingRequest> captureRequest = ArgumentCaptor.forClass(GetObjectTaggingRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).getObjectTagging(captureRequest.capture());
    GetObjectTaggingRequest request = captureRequest.getValue();

    assertEquals("test-bucket", request.getBucketName());
    assertEquals("a", request.getKey());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());
}
 
源代码9 项目: exhibitor   文件: S3PseudoLock.java
@Override
protected List<String> getFileNames(String lockPrefix) throws Exception
{
    ListObjectsRequest  request = new ListObjectsRequest();
    request.setBucketName(bucket);
    request.setPrefix(lockPrefix);
    ObjectListing objectListing = client.listObjects(request);

    return Lists.transform
    (
        objectListing.getObjectSummaries(),
        new Function<S3ObjectSummary, String>()
        {
            @Override
            public String apply(S3ObjectSummary summary)
            {
                return summary.getKey();
            }
        }
    );
}
 
源代码10 项目: nexus-public   文件: BucketManager.java
@Override
public void deleteStorageLocation(final BlobStoreConfiguration blobStoreConfiguration) {
  String bucket = getConfiguredBucket(blobStoreConfiguration);
  ObjectListing listing = s3.listObjects(new ListObjectsRequest().withBucketName(bucket).withMaxKeys(1));
  if (listing.getObjectSummaries().isEmpty()) {
    s3.deleteBucket(bucket);
  }
  else {
    log.info("Not removing S3 bucket {} because it is not empty", bucket);
    BucketLifecycleConfiguration lifecycleConfiguration = s3.getBucketLifecycleConfiguration(bucket);
    List<Rule> nonBlobstoreRules = nonBlobstoreRules(lifecycleConfiguration, blobStoreConfiguration.getName());
    if(!isEmpty(nonBlobstoreRules)) {
      lifecycleConfiguration.setRules(nonBlobstoreRules);
      s3.setBucketLifecycleConfiguration(bucket, lifecycleConfiguration);
    } else {
      s3.deleteBucketLifecycleConfiguration(bucket);
    }
  }
}
 
源代码11 项目: zeppelin   文件: OldS3NotebookRepo.java
@Override
public void remove(String noteId, AuthenticationInfo subject) throws IOException {
  String key = user + "/" + "notebook" + "/" + noteId;
  final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
      .withBucketName(bucketName).withPrefix(key);

  try {
    ObjectListing objects = s3client.listObjects(listObjectsRequest);
    do {
      for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
        s3client.deleteObject(bucketName, objectSummary.getKey());
      }
      objects = s3client.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());
  }
  catch (AmazonClientException ace) {
    throw new IOException("Unable to remove note in S3: " + ace, ace);
  }
}
 
源代码12 项目: fullstop   文件: S3Service.java
public List<String> listCommonPrefixesS3Objects(final String bucketName, final String prefix) {
    final List<String> commonPrefixes = Lists.newArrayList();

    try {
        log.debug("Listing objects in bucket '{}' with prefix '{}'", bucketName, prefix);

        final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
                .withDelimiter("/")
                .withBucketName(bucketName)
                .withPrefix(prefix);

        ObjectListing objectListing;

        do {
            objectListing = s3client.listObjects(listObjectsRequest);
            objectListing.getCommonPrefixes().stream().map(S3Service::urlDecode).forEach(commonPrefixes::add);
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());

    } catch (final AmazonServiceException e) {
        log.error("Could not list common prefixes in S3", e);
    }

    return commonPrefixes;
}
 
源代码13 项目: digdag   文件: TestUtils.java
public static void s3DeleteRecursively(AmazonS3 s3, String bucket, String prefix)
        throws Exception
{
    ListObjectsRequest request = new ListObjectsRequest()
            .withBucketName(bucket)
            .withPrefix(prefix);

    while (true) {
        ObjectListing listing = s3.listObjects(request);
        String[] keys = listing.getObjectSummaries().stream().map(S3ObjectSummary::getKey).toArray(String[]::new);
        for (String key : keys) {
            logger.info("delete s3://{}/{}", bucket, key);
        }
        retryExecutor()
                .retryIf(e -> e instanceof AmazonServiceException)
                .run(() -> s3.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keys)));
        if (listing.getNextMarker() == null) {
            break;
        }
    }
}
 
源代码14 项目: digdag   文件: EmrIT.java
protected void validateTdSparkQueryOutput()
{
    AmazonS3URI resultUri = new AmazonS3URI(tmpS3FolderUri.toString() + "/result/");
    ObjectListing resultListing = s3.listObjects(new ListObjectsRequest().withBucketName(resultUri.getBucket()).withPrefix(resultUri.getKey()));
    List<String> resultLines = resultListing.getObjectSummaries().stream().flatMap(o -> {
        try (S3Object object = s3.getObject(o.getBucketName(), o.getKey())) {
            return CharStreams.readLines(new InputStreamReader(object.getObjectContent())).stream();
        }
        catch (IOException e) {
            throw Throwables.propagate(e);
        }
    }).collect(toList());
    // FIXME
    // we need to specify the version of td-spark that we can use for acceptance tests.
    // In the meantime the assertion is commented out.
    //assertThat(resultLines, Matchers.hasItem(",164.54.104.106,/item/games/4663,/category/electronics,404,Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0),121,GET,1412383598"));
}
 
源代码15 项目: datacollector   文件: S3ConnectionSourceConfig.java
private void validateConnection(
    Stage.Context context,
    String configPrefix,
    List<Stage.ConfigIssue> issues
) {
  try {
    //check if the credentials are right by trying to list an object in the common prefix
    getS3Client().listObjects(new ListObjectsRequest(bucket, commonPrefix, null, delimiter, 1).withEncodingType("url"));
  } catch (AmazonS3Exception e) {
    LOG.debug(Errors.S3_SPOOLDIR_20.getMessage(), e.toString(), e);
    issues.add(
        context.createConfigIssue(
            Groups.S3.name(),
            configPrefix + S3ConnectionBaseConfig.AWS_CONFIG_PREFIX + "awsAccessKeyId",
            Errors.S3_SPOOLDIR_20,
            e.toString()
        )
    );
  }
}
 
源代码16 项目: crate   文件: MockAmazonS3.java
@Override
public ObjectListing listObjects(final ListObjectsRequest request) throws AmazonClientException {
    assertThat(request.getBucketName(), equalTo(bucket));

    final ObjectListing listing = new ObjectListing();
    listing.setBucketName(request.getBucketName());
    listing.setPrefix(request.getPrefix());

    for (Map.Entry<String, byte[]> blob : blobs.entrySet()) {
        if (Strings.isEmpty(request.getPrefix()) || blob.getKey().startsWith(request.getPrefix())) {
            S3ObjectSummary summary = new S3ObjectSummary();
            summary.setBucketName(request.getBucketName());
            summary.setKey(blob.getKey());
            summary.setSize(blob.getValue().length);
            listing.getObjectSummaries().add(summary);
        }
    }
    return listing;
}
 
源代码17 项目: StormCV   文件: S3Connector.java
@Override
public List<String> list() {
	List<String> files = new ArrayList<String>();
	if(s3URI == null) return files;
	String[] buckAndKey = this.getBucketAndKey();
	String bucket = buckAndKey[0];
	String key = buckAndKey[1];
	ObjectListing objectListing = s3. listObjects(new ListObjectsRequest().withBucketName(bucket).withPrefix(key));
	s3list: for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
		// check for valid extensions
		if(extensions == null){
			files.add("s3://"+bucket+"/"+objectSummary.getKey());
		} else for(String ext : extensions) if(objectSummary.getKey().endsWith(ext)){
			files.add("s3://"+bucket+"/"+objectSummary.getKey());
			continue s3list;
		}
	 }
	return files;
}
 
源代码18 项目: presto   文件: S3TableConfigClient.java
/**
 * Call S3 to get the most recent object list.
 * <p>
 * This is an object list request to AWS in the given "directory".
 */
private List<S3ObjectSummary> getObjectSummaries()
{
    AmazonS3Client s3client = clientManager.getS3Client();
    AmazonS3URI directoryURI = new AmazonS3URI(bucketUrl.get());

    List<S3ObjectSummary> result = new ArrayList<>();
    try {
        log.info("Getting the listing of objects in the S3 table config directory: bucket %s prefix %s :", directoryURI.getBucket(), directoryURI.getKey());
        ListObjectsRequest request = new ListObjectsRequest()
                .withBucketName(directoryURI.getBucket())
                .withPrefix(directoryURI.getKey() + "/")
                .withDelimiter("/")
                .withMaxKeys(25);
        ObjectListing response;

        do {
            response = s3client.listObjects(request);

            result.addAll(response.getObjectSummaries());
            request.setMarker(response.getNextMarker());
        }
        while (response.isTruncated());

        log.info("Completed getting S3 object listing.");
    }
    catch (AmazonClientException e) {
        log.error("Skipping update as faced error fetching table descriptions from S3 " + e.toString());
    }
    return result;
}
 
源代码19 项目: hop   文件: S3FileObject.java
@Override
public void handleAttachException( String key, String bucket ) throws FileSystemException {
  SimpleEntry<String, String> newPath = fixFilePath( key, bucket );
  String keyWithDelimiter = newPath.getKey() + DELIMITER;
  try {
    s3Object = getS3Object( keyWithDelimiter, newPath.getValue() );
    s3ObjectMetadata = s3Object.getObjectMetadata();
    injectType( FileType.FOLDER );
  } catch ( AmazonS3Exception e2 ) {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
            .withBucketName( newPath.getValue() )
            .withPrefix( keyWithDelimiter )
            .withDelimiter( DELIMITER );
    ObjectListing ol = fileSystem.getS3Client().listObjects( listObjectsRequest );

    if ( !( ol.getCommonPrefixes().isEmpty() && ol.getObjectSummaries().isEmpty() ) ) {
      injectType( FileType.FOLDER );
    } else {
      //Folders don't really exist - they will generate a "NoSuchKey" exception
      String errorCode = e2.getErrorCode();
      // confirms key doesn't exist but connection okay
      if ( !errorCode.equals( "NoSuchKey" ) ) {
        // bubbling up other connection errors
        logger.error( "Could not get information on " + getQualifiedName(),
                e2 ); // make sure this gets printed for the user
        throw new FileSystemException( "vfs.provider/get-type.error", getQualifiedName(), e2 );
      }
    }
  }
}
 
private void findAllResourcesThatMatches(String bucketName, Set<Resource> resources,
		String prefix, String keyPattern) {
	ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
			.withBucketName(bucketName).withPrefix(prefix);
	ObjectListing objectListing = null;

	do {
		try {
			if (objectListing == null) {
				objectListing = this.amazonS3.listObjects(listObjectsRequest);
			}
			else {
				objectListing = this.amazonS3.listNextBatchOfObjects(objectListing);
			}
			Set<Resource> newResources = getResourcesFromObjectSummaries(bucketName,
					keyPattern, objectListing.getObjectSummaries());
			if (!newResources.isEmpty()) {
				resources.addAll(newResources);
			}
		}
		catch (AmazonS3Exception e) {
			if (301 != e.getStatusCode()) {
				throw e;
			}
		}
	}
	while (objectListing != null && objectListing.isTruncated());
}
 
源代码21 项目: circus-train   文件: S3S3Copier.java
private void initialiseCopyJobsFromListing(
    AmazonS3URI sourceS3Uri,
    final AmazonS3URI targetS3Uri,
    ListObjectsRequest request,
    ObjectListing listing) {
  LOG
      .debug("Found objects to copy {}, for request {}/{}", listing.getObjectSummaries(), request.getBucketName(),
          request.getPrefix());
  List<S3ObjectSummary> objectSummaries = listing.getObjectSummaries();
  for (final S3ObjectSummary s3ObjectSummary : objectSummaries) {
    totalBytesToReplicate += s3ObjectSummary.getSize();
    String fileName = StringUtils.removeStart(s3ObjectSummary.getKey(), sourceS3Uri.getKey());
    final String targetKey = Strings.nullToEmpty(targetS3Uri.getKey()) + fileName;
    CopyObjectRequest copyObjectRequest = new CopyObjectRequest(s3ObjectSummary.getBucketName(),
        s3ObjectSummary.getKey(), targetS3Uri.getBucket(), targetKey);

    if (s3s3CopierOptions.getCannedAcl() != null) {
      copyObjectRequest.withCannedAccessControlList(s3s3CopierOptions.getCannedAcl());
    }

    applyObjectMetadata(copyObjectRequest);

    TransferStateChangeListener stateChangeListener = new BytesTransferStateChangeListener(s3ObjectSummary,
        targetS3Uri, targetKey);
    copyJobRequests.add(new CopyJobRequest(copyObjectRequest, stateChangeListener));
  }
}
 
源代码22 项目: nifi   文件: TestListS3.java
@Test
public void testWriteUserMetadata() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.WRITE_USER_METADATA, "true");

    Date lastModified = new Date();
    ObjectListing objectListing = new ObjectListing();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);

    Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<GetObjectMetadataRequest> captureRequest = ArgumentCaptor.forClass(GetObjectMetadataRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).getObjectMetadata(captureRequest.capture());
    GetObjectMetadataRequest request = captureRequest.getValue();

    assertEquals("test-bucket", request.getBucketName());
    assertEquals("a", request.getKey());

    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());
}
 
源代码23 项目: jobcacher-plugin   文件: S3DownloadAllCallable.java
/**
 * Download to executor
 */
@Override
public Integer invoke(TransferManager transferManager, File base, VirtualChannel channel) throws IOException, InterruptedException {
    if(!base.exists()) {
        if (!base.mkdirs()) {
            throw new IOException("Failed to create directory : " + base);
        }
    }

    int totalCount;
    Downloads downloads = new Downloads();
    ObjectListing objectListing = null;

    do {
        objectListing = transferManager.getAmazonS3Client().listObjects(new ListObjectsRequest()
                .withBucketName(bucketName)
                .withPrefix(pathPrefix)
                .withMarker(objectListing != null ? objectListing.getNextMarker() : null));

        for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
            downloads.startDownload(transferManager, base, pathPrefix, summary);
        }

    } while (objectListing.getNextMarker() != null);

    // Grab # of files copied
    totalCount = downloads.count();

    // Finish the asynchronous downloading process
    downloads.finishDownloading();

    return totalCount;
}
 
源代码24 项目: emodb   文件: StashReader.java
private static String determineEndpointForBucket(String bucket, AWSCredentialsProvider credentialsProvider,
                                                 @Nullable ClientConfiguration s3Config, String rootPath) {

    // Guess us-east-1.  If wrong AWS will return a redirect with the correct endpoint
    AmazonS3 s3 = createS3ClientForRegion(Regions.US_EAST_1.getName(), credentialsProvider, s3Config);
    if (rootPath.startsWith("/")) {
        rootPath = rootPath.substring(1);
    }
    if (!rootPath.endsWith("/")) {
        rootPath = rootPath + "/";
    }

    try {
        // Any request will work but presumably the client has list access for stash so perform a list.
        s3.listObjects(new ListObjectsRequest()
                .withBucketName(bucket)
                .withPrefix(rootPath)
                .withDelimiter("/")
                .withMaxKeys(1));

        // If this didn't error out then the presumed us-east-1 region was correct
        return  "s3.us-east-1.amazonaws.com";
    } catch (AmazonS3Exception e) {
        if (e.getStatusCode() == 301 /* MOVED_PERMANENTLY */) {
            String endPoint = e.getAdditionalDetails().get("Endpoint");
            // The end point is prefixed with the bucket name, so strip it
            return endPoint.substring(bucket.length() + 1);
        }

        throw e;
    }
}
 
源代码25 项目: emodb   文件: StashReader.java
private Iterator<S3ObjectSummary> getS3ObjectSummaries(final String prefix) {
    final int prefixLength = prefix.length();

    Iterator<S3ObjectSummary> allSummaries = Iterators.concat(new AbstractIterator<Iterator<S3ObjectSummary>>() {
        String marker = null;
        ObjectListing response;
        protected Iterator<S3ObjectSummary> computeNext() {
            if (response == null || response.isTruncated()) {
                response = _s3.listObjects(new ListObjectsRequest()
                        .withBucketName(_bucket)
                        .withPrefix(prefix)
                        .withDelimiter("/")
                        .withMarker(marker)
                        .withMaxKeys(1000));
                marker = response.getNextMarker();
                return response.getObjectSummaries().iterator();
            }
            return endOfData();
        }
    });

    // Sometimes the prefix itself can come back as a result.  Filter that entry out.
    return Iterators.filter(allSummaries, new Predicate<S3ObjectSummary>() {
        @Override
        public boolean apply(S3ObjectSummary summary) {
            return summary.getKey().length() > prefixLength;
        }
    });
}
 
源代码26 项目: alexa-skills-kit-tester-java   文件: Lambda.java
@Override
public void handleRequest(final InputStream input, final OutputStream output, final Context context) throws IOException {
    final String inputS = IOUtils.toString(Optional.ofNullable(input).orElse(new ByteArrayInputStream("{}".getBytes())));
    final JsonNode root = om.readTree(inputS);

    final String bucket = Optional.ofNullable(root.get(S3_BUCKET_PROPERTY)).map(JsonNode::textValue).filter(StringUtils::isNotBlank).orElse(System.getenv(S3_BUCKET_PROPERTY));
    Validate.notBlank(bucket, S3_BUCKET_PROPERTY + " hasn't been set in the request payload nor as an environment variable.");

    final String key = Optional.ofNullable(root.get(S3_KEY_PROPERTY)).map(JsonNode::textValue).filter(StringUtils::isNotBlank)
            .orElse(System.getenv(S3_KEY_PROPERTY));
    final String region = Optional.ofNullable(root.get(S3_REGION_PROPERTY)).map(JsonNode::textValue).filter(StringUtils::isNotBlank)
            .orElse(System.getenv(S3_REGION_PROPERTY));

    final AmazonS3 s3client = StringUtils.isNotBlank(region) ? AmazonS3ClientBuilder.standard().withRegion(region).build() : AmazonS3ClientBuilder.defaultClient();

    final ListObjectsRequest listRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(Optional.ofNullable(key).map(k -> k + (k.endsWith("/") ? "" : "/")).orElse(""));

    log.info("[INFO] Reading out *.yml conversation script files in folder '" + listRequest.getPrefix() + "' in bucket '" + listRequest.getBucketName() + "'");

    final List<S3ObjectSummary> conversationScripts = s3client.listObjects(listRequest).getObjectSummaries().stream()
            .filter(os -> os.getKey().toLowerCase().endsWith(".yml")).collect(Collectors.toList());

    log.info("[INFO] Found " + conversationScripts.size() + " conversation script files in bucket '" + bucket + "'");

    for (final S3ObjectSummary conversationScript : conversationScripts) {
        log.info("[INFO] Load conversation script file " + conversationScript.getKey() + " from S3 bucket " + bucket);

        AlexaClient.create(s3client.getObject(bucket, conversationScript.getKey()).getObjectContent())
                .build()
                .startScript();
    }
    output.write("{ \"OK\" }".getBytes());
}
 
源代码27 项目: athenz   文件: S3ChangeLogStoreTest.java
@Test
public void testGetLocalDomainList() throws FileNotFoundException {

    MockS3ChangeLogStore store = new MockS3ChangeLogStore(null, 0);

    InputStream is1 = new FileInputStream("src/test/resources/iaas.json");
    MockS3ObjectInputStream s3Is1 = new MockS3ObjectInputStream(is1, null);

    InputStream is2 = new FileInputStream("src/test/resources/iaas.json");
    MockS3ObjectInputStream s3Is2 = new MockS3ObjectInputStream(is2, null);

    S3Object object = mock(S3Object.class);
    when(object.getObjectContent()).thenReturn(s3Is1).thenReturn(s3Is2);

    when(store.awsS3Client.getObject("s3-unit-test-bucket-name", "iaas")).thenReturn(object);
    ObjectListing mockObjectListing = mock(ObjectListing.class);
    when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(mockObjectListing);
    List<S3ObjectSummary> tempList = new ArrayList<>();
    S3ObjectSummary s3ObjectSummary = mock(S3ObjectSummary.class);
    when(s3ObjectSummary.getKey()).thenReturn("iaas");
    tempList.add(s3ObjectSummary);
    when(mockObjectListing.getObjectSummaries()).thenReturn(tempList);

    List<String> temp = store.getLocalDomainList();
    assertNotNull(temp);
    store.getLocalSignedDomain("iaas");
}
 
源代码28 项目: herd   文件: MockS3OperationsImpl.java
/**
 * {@inheritDoc}
 * <p/>
 * If the bucket does not exist, returns a listing with an empty list. If a prefix is specified in listObjectsRequest, only keys starting with the prefix
 * will be returned.
 */
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest, AmazonS3 s3Client)
{
    LOGGER.debug("listObjects(): listObjectsRequest.getBucketName() = " + listObjectsRequest.getBucketName());

    String bucketName = listObjectsRequest.getBucketName();

    if (MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION.equals(bucketName))
    {
        AmazonS3Exception amazonS3Exception = new AmazonS3Exception(MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION);
        amazonS3Exception.setErrorCode("NoSuchBucket");
        throw amazonS3Exception;
    }

    ObjectListing objectListing = new ObjectListing();
    objectListing.setBucketName(bucketName);

    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucketName);
    if (mockS3Bucket != null)
    {
        for (MockS3Object mockS3Object : mockS3Bucket.getObjects().values())
        {
            String s3ObjectKey = mockS3Object.getKey();
            if (listObjectsRequest.getPrefix() == null || s3ObjectKey.startsWith(listObjectsRequest.getPrefix()))
            {
                S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
                s3ObjectSummary.setBucketName(bucketName);
                s3ObjectSummary.setKey(s3ObjectKey);
                s3ObjectSummary.setSize(mockS3Object.getData().length);
                s3ObjectSummary.setStorageClass(mockS3Object.getObjectMetadata() != null ? mockS3Object.getObjectMetadata().getStorageClass() : null);

                objectListing.getObjectSummaries().add(s3ObjectSummary);
            }
        }
    }

    return objectListing;
}
 
源代码29 项目: terrapin   文件: TerrapinUtilTest.java
@Test
@PrepareForTest(TerrapinUtil.class)
public void testGetS3FileList() throws Exception {
  AmazonS3Client s3Client = mock(AmazonS3Client.class);
  ObjectListing objectListing = mock(ObjectListing.class);
  S3ObjectSummary summary1 = new S3ObjectSummary();
  S3ObjectSummary summary2 = new S3ObjectSummary();
  S3ObjectSummary summary3 = new S3ObjectSummary();
  summary1.setKey("/abc/123");
  summary2.setKey("/abc/456");
  summary3.setKey("/def/123");
  summary1.setSize(32432);
  summary2.setSize(213423);
  summary3.setSize(2334);
  List<S3ObjectSummary> summaries = ImmutableList.of(summary1, summary2, summary3);
  whenNew(AmazonS3Client.class).withAnyArguments().thenReturn(s3Client);
  when(s3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
  when(objectListing.getObjectSummaries()).thenReturn(summaries);

  List<Pair<Path, Long>> results = TerrapinUtil.getS3FileList(new AnonymousAWSCredentials(),
      "bucket", "/abc");

  assertEquals(2, results.size());
  assertTrue(results.get(0).getLeft().toString().endsWith(summary1.getKey()));
  assertEquals(new Long(summary1.getSize()), results.get(0).getRight());
  assertTrue(results.get(1).getLeft().toString().endsWith(summary2.getKey()));
  assertEquals(new Long(summary2.getSize()), results.get(1).getRight());
}
 
源代码30 项目: micro-server   文件: S3Utils.java
/**
 * Method returns list of all <b>S3ObjectSummary</b> objects, subject to
 * <i>req</i> parameters. Multiple S3 calls will be performed if there are
 * more than 1000 elements there
 * 
 * @param req
 *            - ListObjectRequest to be used.
 * @return List of S3ObjectSummary from bucket,
 */
public List<S3ObjectSummary> getAllSummaries(ListObjectsRequest req) {
    List<S3ObjectSummary> result = new ArrayList<>();
    String marker = null;
    ListObjectsRequest req2 = (ListObjectsRequest) req.clone();
    ObjectListing listing;
    do {
        listing = client.listObjects(req2.withMarker(marker));
        marker = listing.getNextMarker();
        result.addAll(listing.getObjectSummaries());
    } while (listing.isTruncated());

    return result;
}
 
 同包方法