下面列出了com.amazonaws.services.s3.model.GetObjectMetadataRequest#com.amazonaws.services.s3.model.ListObjectsRequest 实例代码，或者点击链接到github查看源代码，也可以在右侧发表评论。
@Override
protected void doStart() {
AWSCredentials myCredentials = new BasicAWSCredentials(accessKey, secretKey);
AmazonS3 s3Client = new AmazonS3Client(myCredentials);
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket);
ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);
ChannelProcessor channelProcessor = getChannelProcessor();
for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) {
String file = s3ObjectSummary.getKey();
LOGGER.info("Read the content of {}", file);
GetObjectRequest objectRequest = new GetObjectRequest(bucket, file);
S3Object objectPortion = s3Client.getObject(objectRequest);
try {
long startTime = System.currentTimeMillis();
processLines(channelProcessor, objectPortion.getObjectContent());
LOGGER.info("Processing of {} took {} ms", file, System.currentTimeMillis() - startTime);
} catch (IOException e) {
LOGGER.warn("Cannot process the {}, skipping", file, e);
}
}
}
private void handleAttachExceptionFallback( String bucket, String keyWithDelimiter, AmazonS3Exception exception )
throws FileSystemException {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
.withBucketName( bucket )
.withPrefix( keyWithDelimiter )
.withDelimiter( DELIMITER );
ObjectListing ol = fileSystem.getS3Client().listObjects( listObjectsRequest );
if ( !( ol.getCommonPrefixes().isEmpty() && ol.getObjectSummaries().isEmpty() ) ) {
injectType( FileType.FOLDER );
} else {
//Folders don't really exist - they will generate a "NoSuchKey" exception
// confirms key doesn't exist but connection okay
String errorCode = exception.getErrorCode();
if ( !errorCode.equals( "NoSuchKey" ) ) {
// bubbling up other connection errors
logger.error( "Could not get information on " + getQualifiedName(),
exception ); // make sure this gets printed for the user
throw new FileSystemException( "vfs.provider/get-type.error", getQualifiedName(), exception );
}
}
}
@Test
public void testHandleAttachExceptionEmptyFolder() throws FileSystemException {
AmazonS3Exception exception = new AmazonS3Exception( "NoSuchKey" );
exception.setErrorCode( "NoSuchKey" );
//test the case where the folder exists and contains things; no exception should be thrown
when( s3ServiceMock.getObject( BUCKET_NAME, origKey ) ).thenThrow( exception );
when( s3ServiceMock.getObject( BUCKET_NAME, origKey + "/" ) ).thenThrow( exception );
childObjectListing = mock( ObjectListing.class );
when( childObjectListing.getObjectSummaries() ).thenReturn( new ArrayList<>() );
when( childObjectListing.getCommonPrefixes() ).thenReturn( new ArrayList<>() );
when( s3ServiceMock.listObjects( any( ListObjectsRequest.class ) ) ).thenReturn( childObjectListing );
try {
s3FileObjectFileSpy.doAttach();
} catch ( Exception e ) {
fail( "Caught exception " + e.getMessage() );
}
assertEquals( FileType.IMAGINARY, s3FileObjectFileSpy.getType() );
}
@Test
public void testHandleAttachExceptionFileNotFound() throws FileSystemException {
AmazonS3Exception notFoundException = new AmazonS3Exception( "404 Not Found" );
notFoundException.setErrorCode( "404 Not Found" );
AmazonS3Exception noSuchKeyException = new AmazonS3Exception( "NoSuchKey" );
noSuchKeyException.setErrorCode( "NoSuchKey" );
//test the case where the file is not found; no exception should be thrown
when( s3ServiceMock.getObject( BUCKET_NAME, origKey ) ).thenThrow( notFoundException );
when( s3ServiceMock.getObject( BUCKET_NAME, origKey + "/" ) ).thenThrow( noSuchKeyException );
childObjectListing = mock( ObjectListing.class );
when( childObjectListing.getObjectSummaries() ).thenReturn( new ArrayList<>() );
when( childObjectListing.getCommonPrefixes() ).thenReturn( new ArrayList<>() );
when( s3ServiceMock.listObjects( any( ListObjectsRequest.class ) ) ).thenReturn( childObjectListing );
try {
s3FileObjectFileSpy.doAttach();
} catch ( Exception e ) {
fail( "Caught exception " + e.getMessage() );
}
assertEquals( FileType.IMAGINARY, s3FileObjectFileSpy.getType() );
}
@Test
public void copyMultipleObjects() throws Exception {
// Making sure we only request 1 file at the time so we need to loop
ListObjectsRequestFactory mockListObjectRequestFactory = Mockito.mock(ListObjectsRequestFactory.class);
when(mockListObjectRequestFactory.newInstance()).thenReturn(new ListObjectsRequest().withMaxKeys(1));
client.putObject("source", "bar/data1", inputData);
client.putObject("source", "bar/data2", inputData);
Path sourceBaseLocation = new Path("s3://source/bar/");
Path replicaLocation = new Path("s3://target/foo/");
List<Path> sourceSubLocations = new ArrayList<>();
S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
transferManagerFactory, mockListObjectRequestFactory, registry, s3S3CopierOptions);
Metrics metrics = s3s3Copier.copy();
assertThat(metrics.getBytesReplicated(), is(14L));
S3Object object1 = client.getObject("target", "foo/data1");
String data1 = IOUtils.toString(object1.getObjectContent());
assertThat(data1, is("bar foo"));
S3Object object2 = client.getObject("target", "foo/data2");
String data2 = IOUtils.toString(object2.getObjectContent());
assertThat(data2, is("bar foo"));
}
private Matcher<ListObjectsRequest> listObjectRequest(final String bucket, final String prefix, @Nullable final String marker) {
return new BaseMatcher<ListObjectsRequest>() {
@Override
public boolean matches(Object item) {
ListObjectsRequest request = (ListObjectsRequest) item;
return request != null &&
request.getBucketName().equals(bucket) &&
request.getPrefix().equals(prefix) &&
Objects.equals(request.getMarker(), marker);
}
@Override
public void describeTo(Description description) {
description.appendText("ListObjectRequest[s3://").appendText(bucket).appendText("/").appendText(prefix);
if (marker != null) {
description.appendText(", marker=").appendText(marker);
}
description.appendText("]");
}
};
}
private Answer<ObjectListing> objectListingAnswer(@Nullable final String marker, final String... fileNames) {
return new Answer<ObjectListing>() {
@Override
public ObjectListing answer(InvocationOnMock invocation)
throws Throwable {
ListObjectsRequest request = (ListObjectsRequest) invocation.getArguments()[0];
ObjectListing objectListing = new ObjectListing();
objectListing.setBucketName(request.getBucketName());
objectListing.setPrefix(request.getPrefix());
objectListing.setTruncated(marker != null);
objectListing.setNextMarker(marker);
for (String fileName : fileNames) {
S3ObjectSummary objectSummary = new S3ObjectSummary();
objectSummary.setKey(request.getPrefix() + fileName);
objectSummary.setSize(100);
objectListing.getObjectSummaries().add(objectSummary);
}
return objectListing;
}
};
}
@Test
public void testWriteObjectTags() {
runner.setProperty(ListS3.REGION, "eu-west-1");
runner.setProperty(ListS3.BUCKET, "test-bucket");
runner.setProperty(ListS3.WRITE_OBJECT_TAGS, "true");
Date lastModified = new Date();
ObjectListing objectListing = new ObjectListing();
S3ObjectSummary objectSummary1 = new S3ObjectSummary();
objectSummary1.setBucketName("test-bucket");
objectSummary1.setKey("a");
objectSummary1.setLastModified(lastModified);
objectListing.getObjectSummaries().add(objectSummary1);
Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);
runner.run();
ArgumentCaptor<GetObjectTaggingRequest> captureRequest = ArgumentCaptor.forClass(GetObjectTaggingRequest.class);
Mockito.verify(mockS3Client, Mockito.times(1)).getObjectTagging(captureRequest.capture());
GetObjectTaggingRequest request = captureRequest.getValue();
assertEquals("test-bucket", request.getBucketName());
assertEquals("a", request.getKey());
Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());
}
@Override
protected List<String> getFileNames(String lockPrefix) throws Exception
{
ListObjectsRequest request = new ListObjectsRequest();
request.setBucketName(bucket);
request.setPrefix(lockPrefix);
ObjectListing objectListing = client.listObjects(request);
return Lists.transform
(
objectListing.getObjectSummaries(),
new Function<S3ObjectSummary, String>()
{
@Override
public String apply(S3ObjectSummary summary)
{
return summary.getKey();
}
}
);
}
@Override
public void deleteStorageLocation(final BlobStoreConfiguration blobStoreConfiguration) {
String bucket = getConfiguredBucket(blobStoreConfiguration);
ObjectListing listing = s3.listObjects(new ListObjectsRequest().withBucketName(bucket).withMaxKeys(1));
if (listing.getObjectSummaries().isEmpty()) {
s3.deleteBucket(bucket);
}
else {
log.info("Not removing S3 bucket {} because it is not empty", bucket);
BucketLifecycleConfiguration lifecycleConfiguration = s3.getBucketLifecycleConfiguration(bucket);
List<Rule> nonBlobstoreRules = nonBlobstoreRules(lifecycleConfiguration, blobStoreConfiguration.getName());
if(!isEmpty(nonBlobstoreRules)) {
lifecycleConfiguration.setRules(nonBlobstoreRules);
s3.setBucketLifecycleConfiguration(bucket, lifecycleConfiguration);
} else {
s3.deleteBucketLifecycleConfiguration(bucket);
}
}
}
@Override
public void remove(String noteId, AuthenticationInfo subject) throws IOException {
String key = user + "/" + "notebook" + "/" + noteId;
final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
.withBucketName(bucketName).withPrefix(key);
try {
ObjectListing objects = s3client.listObjects(listObjectsRequest);
do {
for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
s3client.deleteObject(bucketName, objectSummary.getKey());
}
objects = s3client.listNextBatchOfObjects(objects);
} while (objects.isTruncated());
}
catch (AmazonClientException ace) {
throw new IOException("Unable to remove note in S3: " + ace, ace);
}
}
public List<String> listCommonPrefixesS3Objects(final String bucketName, final String prefix) {
final List<String> commonPrefixes = Lists.newArrayList();
try {
log.debug("Listing objects in bucket '{}' with prefix '{}'", bucketName, prefix);
final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
.withDelimiter("/")
.withBucketName(bucketName)
.withPrefix(prefix);
ObjectListing objectListing;
do {
objectListing = s3client.listObjects(listObjectsRequest);
objectListing.getCommonPrefixes().stream().map(S3Service::urlDecode).forEach(commonPrefixes::add);
listObjectsRequest.setMarker(objectListing.getNextMarker());
} while (objectListing.isTruncated());
} catch (final AmazonServiceException e) {
log.error("Could not list common prefixes in S3", e);
}
return commonPrefixes;
}
public static void s3DeleteRecursively(AmazonS3 s3, String bucket, String prefix)
throws Exception
{
ListObjectsRequest request = new ListObjectsRequest()
.withBucketName(bucket)
.withPrefix(prefix);
while (true) {
ObjectListing listing = s3.listObjects(request);
String[] keys = listing.getObjectSummaries().stream().map(S3ObjectSummary::getKey).toArray(String[]::new);
for (String key : keys) {
logger.info("delete s3://{}/{}", bucket, key);
}
retryExecutor()
.retryIf(e -> e instanceof AmazonServiceException)
.run(() -> s3.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keys)));
if (listing.getNextMarker() == null) {
break;
}
}
}
protected void validateTdSparkQueryOutput()
{
AmazonS3URI resultUri = new AmazonS3URI(tmpS3FolderUri.toString() + "/result/");
ObjectListing resultListing = s3.listObjects(new ListObjectsRequest().withBucketName(resultUri.getBucket()).withPrefix(resultUri.getKey()));
List<String> resultLines = resultListing.getObjectSummaries().stream().flatMap(o -> {
try (S3Object object = s3.getObject(o.getBucketName(), o.getKey())) {
return CharStreams.readLines(new InputStreamReader(object.getObjectContent())).stream();
}
catch (IOException e) {
throw Throwables.propagate(e);
}
}).collect(toList());
// FIXME
// we need to specify the version of td-spark that we can use for acceptance tests.
// In the meantime the assertion is commented out.
//assertThat(resultLines, Matchers.hasItem(",164.54.104.106,/item/games/4663,/category/electronics,404,Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0),121,GET,1412383598"));
}
private void validateConnection(
Stage.Context context,
String configPrefix,
List<Stage.ConfigIssue> issues
) {
try {
//check if the credentials are right by trying to list an object in the common prefix
getS3Client().listObjects(new ListObjectsRequest(bucket, commonPrefix, null, delimiter, 1).withEncodingType("url"));
} catch (AmazonS3Exception e) {
LOG.debug(Errors.S3_SPOOLDIR_20.getMessage(), e.toString(), e);
issues.add(
context.createConfigIssue(
Groups.S3.name(),
configPrefix + S3ConnectionBaseConfig.AWS_CONFIG_PREFIX + "awsAccessKeyId",
Errors.S3_SPOOLDIR_20,
e.toString()
)
);
}
}
@Override
public ObjectListing listObjects(final ListObjectsRequest request) throws AmazonClientException {
assertThat(request.getBucketName(), equalTo(bucket));
final ObjectListing listing = new ObjectListing();
listing.setBucketName(request.getBucketName());
listing.setPrefix(request.getPrefix());
for (Map.Entry<String, byte[]> blob : blobs.entrySet()) {
if (Strings.isEmpty(request.getPrefix()) || blob.getKey().startsWith(request.getPrefix())) {
S3ObjectSummary summary = new S3ObjectSummary();
summary.setBucketName(request.getBucketName());
summary.setKey(blob.getKey());
summary.setSize(blob.getValue().length);
listing.getObjectSummaries().add(summary);
}
}
return listing;
}
@Override
public List<String> list() {
List<String> files = new ArrayList<String>();
if(s3URI == null) return files;
String[] buckAndKey = this.getBucketAndKey();
String bucket = buckAndKey[0];
String key = buckAndKey[1];
ObjectListing objectListing = s3. listObjects(new ListObjectsRequest().withBucketName(bucket).withPrefix(key));
s3list: for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
// check for valid extensions
if(extensions == null){
files.add("s3://"+bucket+"/"+objectSummary.getKey());
} else for(String ext : extensions) if(objectSummary.getKey().endsWith(ext)){
files.add("s3://"+bucket+"/"+objectSummary.getKey());
continue s3list;
}
}
return files;
}
/**
* Call S3 to get the most recent object list.
* <p>
* This is an object list request to AWS in the given "directory".
*/
private List<S3ObjectSummary> getObjectSummaries()
{
AmazonS3Client s3client = clientManager.getS3Client();
AmazonS3URI directoryURI = new AmazonS3URI(bucketUrl.get());
List<S3ObjectSummary> result = new ArrayList<>();
try {
log.info("Getting the listing of objects in the S3 table config directory: bucket %s prefix %s :", directoryURI.getBucket(), directoryURI.getKey());
ListObjectsRequest request = new ListObjectsRequest()
.withBucketName(directoryURI.getBucket())
.withPrefix(directoryURI.getKey() + "/")
.withDelimiter("/")
.withMaxKeys(25);
ObjectListing response;
do {
response = s3client.listObjects(request);
result.addAll(response.getObjectSummaries());
request.setMarker(response.getNextMarker());
}
while (response.isTruncated());
log.info("Completed getting S3 object listing.");
}
catch (AmazonClientException e) {
log.error("Skipping update as faced error fetching table descriptions from S3 " + e.toString());
}
return result;
}
@Override
public void handleAttachException( String key, String bucket ) throws FileSystemException {
SimpleEntry<String, String> newPath = fixFilePath( key, bucket );
String keyWithDelimiter = newPath.getKey() + DELIMITER;
try {
s3Object = getS3Object( keyWithDelimiter, newPath.getValue() );
s3ObjectMetadata = s3Object.getObjectMetadata();
injectType( FileType.FOLDER );
} catch ( AmazonS3Exception e2 ) {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
.withBucketName( newPath.getValue() )
.withPrefix( keyWithDelimiter )
.withDelimiter( DELIMITER );
ObjectListing ol = fileSystem.getS3Client().listObjects( listObjectsRequest );
if ( !( ol.getCommonPrefixes().isEmpty() && ol.getObjectSummaries().isEmpty() ) ) {
injectType( FileType.FOLDER );
} else {
//Folders don't really exist - they will generate a "NoSuchKey" exception
String errorCode = e2.getErrorCode();
// confirms key doesn't exist but connection okay
if ( !errorCode.equals( "NoSuchKey" ) ) {
// bubbling up other connection errors
logger.error( "Could not get information on " + getQualifiedName(),
e2 ); // make sure this gets printed for the user
throw new FileSystemException( "vfs.provider/get-type.error", getQualifiedName(), e2 );
}
}
}
}
private void findAllResourcesThatMatches(String bucketName, Set<Resource> resources,
String prefix, String keyPattern) {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
.withBucketName(bucketName).withPrefix(prefix);
ObjectListing objectListing = null;
do {
try {
if (objectListing == null) {
objectListing = this.amazonS3.listObjects(listObjectsRequest);
}
else {
objectListing = this.amazonS3.listNextBatchOfObjects(objectListing);
}
Set<Resource> newResources = getResourcesFromObjectSummaries(bucketName,
keyPattern, objectListing.getObjectSummaries());
if (!newResources.isEmpty()) {
resources.addAll(newResources);
}
}
catch (AmazonS3Exception e) {
if (301 != e.getStatusCode()) {
throw e;
}
}
}
while (objectListing != null && objectListing.isTruncated());
}
private void initialiseCopyJobsFromListing(
AmazonS3URI sourceS3Uri,
final AmazonS3URI targetS3Uri,
ListObjectsRequest request,
ObjectListing listing) {
LOG
.debug("Found objects to copy {}, for request {}/{}", listing.getObjectSummaries(), request.getBucketName(),
request.getPrefix());
List<S3ObjectSummary> objectSummaries = listing.getObjectSummaries();
for (final S3ObjectSummary s3ObjectSummary : objectSummaries) {
totalBytesToReplicate += s3ObjectSummary.getSize();
String fileName = StringUtils.removeStart(s3ObjectSummary.getKey(), sourceS3Uri.getKey());
final String targetKey = Strings.nullToEmpty(targetS3Uri.getKey()) + fileName;
CopyObjectRequest copyObjectRequest = new CopyObjectRequest(s3ObjectSummary.getBucketName(),
s3ObjectSummary.getKey(), targetS3Uri.getBucket(), targetKey);
if (s3s3CopierOptions.getCannedAcl() != null) {
copyObjectRequest.withCannedAccessControlList(s3s3CopierOptions.getCannedAcl());
}
applyObjectMetadata(copyObjectRequest);
TransferStateChangeListener stateChangeListener = new BytesTransferStateChangeListener(s3ObjectSummary,
targetS3Uri, targetKey);
copyJobRequests.add(new CopyJobRequest(copyObjectRequest, stateChangeListener));
}
}
@Test
public void testWriteUserMetadata() {
runner.setProperty(ListS3.REGION, "eu-west-1");
runner.setProperty(ListS3.BUCKET, "test-bucket");
runner.setProperty(ListS3.WRITE_USER_METADATA, "true");
Date lastModified = new Date();
ObjectListing objectListing = new ObjectListing();
S3ObjectSummary objectSummary1 = new S3ObjectSummary();
objectSummary1.setBucketName("test-bucket");
objectSummary1.setKey("a");
objectSummary1.setLastModified(lastModified);
objectListing.getObjectSummaries().add(objectSummary1);
Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);
runner.run();
ArgumentCaptor<GetObjectMetadataRequest> captureRequest = ArgumentCaptor.forClass(GetObjectMetadataRequest.class);
Mockito.verify(mockS3Client, Mockito.times(1)).getObjectMetadata(captureRequest.capture());
GetObjectMetadataRequest request = captureRequest.getValue();
assertEquals("test-bucket", request.getBucketName());
assertEquals("a", request.getKey());
Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());
}
/**
* Download to executor
*/
@Override
public Integer invoke(TransferManager transferManager, File base, VirtualChannel channel) throws IOException, InterruptedException {
if(!base.exists()) {
if (!base.mkdirs()) {
throw new IOException("Failed to create directory : " + base);
}
}
int totalCount;
Downloads downloads = new Downloads();
ObjectListing objectListing = null;
do {
objectListing = transferManager.getAmazonS3Client().listObjects(new ListObjectsRequest()
.withBucketName(bucketName)
.withPrefix(pathPrefix)
.withMarker(objectListing != null ? objectListing.getNextMarker() : null));
for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
downloads.startDownload(transferManager, base, pathPrefix, summary);
}
} while (objectListing.getNextMarker() != null);
// Grab # of files copied
totalCount = downloads.count();
// Finish the asynchronous downloading process
downloads.finishDownloading();
return totalCount;
}
private static String determineEndpointForBucket(String bucket, AWSCredentialsProvider credentialsProvider,
@Nullable ClientConfiguration s3Config, String rootPath) {
// Guess us-east-1. If wrong AWS will return a redirect with the correct endpoint
AmazonS3 s3 = createS3ClientForRegion(Regions.US_EAST_1.getName(), credentialsProvider, s3Config);
if (rootPath.startsWith("/")) {
rootPath = rootPath.substring(1);
}
if (!rootPath.endsWith("/")) {
rootPath = rootPath + "/";
}
try {
// Any request will work but presumably the client has list access for stash so perform a list.
s3.listObjects(new ListObjectsRequest()
.withBucketName(bucket)
.withPrefix(rootPath)
.withDelimiter("/")
.withMaxKeys(1));
// If this didn't error out then the presumed us-east-1 region was correct
return "s3.us-east-1.amazonaws.com";
} catch (AmazonS3Exception e) {
if (e.getStatusCode() == 301 /* MOVED_PERMANENTLY */) {
String endPoint = e.getAdditionalDetails().get("Endpoint");
// The end point is prefixed with the bucket name, so strip it
return endPoint.substring(bucket.length() + 1);
}
throw e;
}
}
private Iterator<S3ObjectSummary> getS3ObjectSummaries(final String prefix) {
final int prefixLength = prefix.length();
Iterator<S3ObjectSummary> allSummaries = Iterators.concat(new AbstractIterator<Iterator<S3ObjectSummary>>() {
String marker = null;
ObjectListing response;
protected Iterator<S3ObjectSummary> computeNext() {
if (response == null || response.isTruncated()) {
response = _s3.listObjects(new ListObjectsRequest()
.withBucketName(_bucket)
.withPrefix(prefix)
.withDelimiter("/")
.withMarker(marker)
.withMaxKeys(1000));
marker = response.getNextMarker();
return response.getObjectSummaries().iterator();
}
return endOfData();
}
});
// Sometimes the prefix itself can come back as a result. Filter that entry out.
return Iterators.filter(allSummaries, new Predicate<S3ObjectSummary>() {
@Override
public boolean apply(S3ObjectSummary summary) {
return summary.getKey().length() > prefixLength;
}
});
}
@Override
public void handleRequest(final InputStream input, final OutputStream output, final Context context) throws IOException {
final String inputS = IOUtils.toString(Optional.ofNullable(input).orElse(new ByteArrayInputStream("{}".getBytes())));
final JsonNode root = om.readTree(inputS);
final String bucket = Optional.ofNullable(root.get(S3_BUCKET_PROPERTY)).map(JsonNode::textValue).filter(StringUtils::isNotBlank).orElse(System.getenv(S3_BUCKET_PROPERTY));
Validate.notBlank(bucket, S3_BUCKET_PROPERTY + " hasn't been set in the request payload nor as an environment variable.");
final String key = Optional.ofNullable(root.get(S3_KEY_PROPERTY)).map(JsonNode::textValue).filter(StringUtils::isNotBlank)
.orElse(System.getenv(S3_KEY_PROPERTY));
final String region = Optional.ofNullable(root.get(S3_REGION_PROPERTY)).map(JsonNode::textValue).filter(StringUtils::isNotBlank)
.orElse(System.getenv(S3_REGION_PROPERTY));
final AmazonS3 s3client = StringUtils.isNotBlank(region) ? AmazonS3ClientBuilder.standard().withRegion(region).build() : AmazonS3ClientBuilder.defaultClient();
final ListObjectsRequest listRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(Optional.ofNullable(key).map(k -> k + (k.endsWith("/") ? "" : "/")).orElse(""));
log.info("[INFO] Reading out *.yml conversation script files in folder '" + listRequest.getPrefix() + "' in bucket '" + listRequest.getBucketName() + "'");
final List<S3ObjectSummary> conversationScripts = s3client.listObjects(listRequest).getObjectSummaries().stream()
.filter(os -> os.getKey().toLowerCase().endsWith(".yml")).collect(Collectors.toList());
log.info("[INFO] Found " + conversationScripts.size() + " conversation script files in bucket '" + bucket + "'");
for (final S3ObjectSummary conversationScript : conversationScripts) {
log.info("[INFO] Load conversation script file " + conversationScript.getKey() + " from S3 bucket " + bucket);
AlexaClient.create(s3client.getObject(bucket, conversationScript.getKey()).getObjectContent())
.build()
.startScript();
}
output.write("{ \"OK\" }".getBytes());
}
@Test
public void testGetLocalDomainList() throws FileNotFoundException {
MockS3ChangeLogStore store = new MockS3ChangeLogStore(null, 0);
InputStream is1 = new FileInputStream("src/test/resources/iaas.json");
MockS3ObjectInputStream s3Is1 = new MockS3ObjectInputStream(is1, null);
InputStream is2 = new FileInputStream("src/test/resources/iaas.json");
MockS3ObjectInputStream s3Is2 = new MockS3ObjectInputStream(is2, null);
S3Object object = mock(S3Object.class);
when(object.getObjectContent()).thenReturn(s3Is1).thenReturn(s3Is2);
when(store.awsS3Client.getObject("s3-unit-test-bucket-name", "iaas")).thenReturn(object);
ObjectListing mockObjectListing = mock(ObjectListing.class);
when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(mockObjectListing);
List<S3ObjectSummary> tempList = new ArrayList<>();
S3ObjectSummary s3ObjectSummary = mock(S3ObjectSummary.class);
when(s3ObjectSummary.getKey()).thenReturn("iaas");
tempList.add(s3ObjectSummary);
when(mockObjectListing.getObjectSummaries()).thenReturn(tempList);
List<String> temp = store.getLocalDomainList();
assertNotNull(temp);
store.getLocalSignedDomain("iaas");
}
/**
* {@inheritDoc}
* <p/>
* If the bucket does not exist, returns a listing with an empty list. If a prefix is specified in listObjectsRequest, only keys starting with the prefix
* will be returned.
*/
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest, AmazonS3 s3Client)
{
LOGGER.debug("listObjects(): listObjectsRequest.getBucketName() = " + listObjectsRequest.getBucketName());
String bucketName = listObjectsRequest.getBucketName();
if (MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION.equals(bucketName))
{
AmazonS3Exception amazonS3Exception = new AmazonS3Exception(MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION);
amazonS3Exception.setErrorCode("NoSuchBucket");
throw amazonS3Exception;
}
ObjectListing objectListing = new ObjectListing();
objectListing.setBucketName(bucketName);
MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucketName);
if (mockS3Bucket != null)
{
for (MockS3Object mockS3Object : mockS3Bucket.getObjects().values())
{
String s3ObjectKey = mockS3Object.getKey();
if (listObjectsRequest.getPrefix() == null || s3ObjectKey.startsWith(listObjectsRequest.getPrefix()))
{
S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
s3ObjectSummary.setBucketName(bucketName);
s3ObjectSummary.setKey(s3ObjectKey);
s3ObjectSummary.setSize(mockS3Object.getData().length);
s3ObjectSummary.setStorageClass(mockS3Object.getObjectMetadata() != null ? mockS3Object.getObjectMetadata().getStorageClass() : null);
objectListing.getObjectSummaries().add(s3ObjectSummary);
}
}
}
return objectListing;
}
@Test
@PrepareForTest(TerrapinUtil.class)
public void testGetS3FileList() throws Exception {
AmazonS3Client s3Client = mock(AmazonS3Client.class);
ObjectListing objectListing = mock(ObjectListing.class);
S3ObjectSummary summary1 = new S3ObjectSummary();
S3ObjectSummary summary2 = new S3ObjectSummary();
S3ObjectSummary summary3 = new S3ObjectSummary();
summary1.setKey("/abc/123");
summary2.setKey("/abc/456");
summary3.setKey("/def/123");
summary1.setSize(32432);
summary2.setSize(213423);
summary3.setSize(2334);
List<S3ObjectSummary> summaries = ImmutableList.of(summary1, summary2, summary3);
whenNew(AmazonS3Client.class).withAnyArguments().thenReturn(s3Client);
when(s3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
when(objectListing.getObjectSummaries()).thenReturn(summaries);
List<Pair<Path, Long>> results = TerrapinUtil.getS3FileList(new AnonymousAWSCredentials(),
"bucket", "/abc");
assertEquals(2, results.size());
assertTrue(results.get(0).getLeft().toString().endsWith(summary1.getKey()));
assertEquals(new Long(summary1.getSize()), results.get(0).getRight());
assertTrue(results.get(1).getLeft().toString().endsWith(summary2.getKey()));
assertEquals(new Long(summary2.getSize()), results.get(1).getRight());
}
/**
* Method returns list of all <b>S3ObjectSummary</b> objects, subject to
* <i>req</i> parameters. Multiple S3 calls will be performed if there are
* more than 1000 elements there
*
* @param req
* - ListObjectRequest to be used.
* @return List of S3ObjectSummary from bucket,
*/
public List<S3ObjectSummary> getAllSummaries(ListObjectsRequest req) {
List<S3ObjectSummary> result = new ArrayList<>();
String marker = null;
ListObjectsRequest req2 = (ListObjectsRequest) req.clone();
ObjectListing listing;
do {
listing = client.listObjects(req2.withMarker(marker));
marker = listing.getNextMarker();
result.addAll(listing.getObjectSummaries());
} while (listing.isTruncated());
return result;
}