下面列出了com.amazonaws.services.s3.model.S3Object#setObjectMetadata ( ) 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
private void setupS3(String fileName, String version, String propertyContent)
throws UnsupportedEncodingException {
final S3ObjectId s3ObjectId = new S3ObjectId("bucket1", fileName);
final GetObjectRequest request = new GetObjectRequest(s3ObjectId);
final S3Object s3Object = new S3Object();
s3Object.setObjectContent(new StringInputStream(propertyContent));
if (version != null) {
final ObjectMetadata metadata = new ObjectMetadata();
metadata.setHeader("x-amz-version-id", version);
s3Object.setObjectMetadata(metadata);
}
when(s3Client.getObject(argThat(new GetObjectRequestMatcher(request))))
.thenReturn(s3Object);
}
@Test
void getInputStream_existingObject_returnsInputStreamWithContent() throws Exception {
// Arrange
AmazonS3 amazonS3 = mock(AmazonS3.class);
ObjectMetadata objectMetadata = mock(ObjectMetadata.class);
when(amazonS3.getObjectMetadata(any(GetObjectMetadataRequest.class)))
.thenReturn(objectMetadata);
S3Object s3Object = new S3Object();
s3Object.setObjectMetadata(objectMetadata);
s3Object.setObjectContent(new ByteArrayInputStream(new byte[] { 42 }));
when(amazonS3.getObject(any(GetObjectRequest.class))).thenReturn(s3Object);
// Act
SimpleStorageResource simpleStorageResource = new SimpleStorageResource(amazonS3,
"bucket", "object", new SyncTaskExecutor());
// Assert
assertThat(simpleStorageResource.exists()).isTrue();
assertThat(simpleStorageResource.getInputStream().read()).isEqualTo(42);
}
@Override
public S3Object getObject(final GetObjectRequest request) throws AmazonClientException {
assertThat(request.getBucketName(), equalTo(bucket));
final String blobName = request.getKey();
final byte[] content = blobs.get(blobName);
if (content == null) {
AmazonS3Exception exception = new AmazonS3Exception("[" + blobName + "] does not exist.");
exception.setStatusCode(404);
throw exception;
}
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(content.length);
S3Object s3Object = new S3Object();
s3Object.setObjectContent(new S3ObjectInputStream(new ByteArrayInputStream(content), null, false));
s3Object.setKey(blobName);
s3Object.setObjectMetadata(metadata);
return s3Object;
}
@Override
public S3Object getS3Object(GetObjectRequest getObjectRequest, AmazonS3 s3)
{
MockS3Object mockS3Object = getMockS3Object(getObjectRequest.getBucketName(), getObjectRequest.getKey());
S3Object s3Object = new S3Object();
s3Object.setBucketName(getObjectRequest.getBucketName());
s3Object.setKey(getObjectRequest.getKey());
s3Object.setObjectContent(new ByteArrayInputStream(mockS3Object.getData()));
s3Object.setObjectMetadata(mockS3Object.getObjectMetadata());
return s3Object;
}
@Test
public void testGetObjectVersion() throws IOException {
runner.setProperty(FetchS3Object.REGION, "us-east-1");
runner.setProperty(FetchS3Object.BUCKET, "request-bucket");
runner.setProperty(FetchS3Object.VERSION_ID, "${s3.version}");
final Map<String, String> attrs = new HashMap<>();
attrs.put("filename", "request-key");
attrs.put("s3.version", "request-version");
runner.enqueue(new byte[0], attrs);
S3Object s3ObjectResponse = new S3Object();
s3ObjectResponse.setBucketName("response-bucket-name");
s3ObjectResponse.setObjectContent(new StringInputStream("Some Content"));
ObjectMetadata metadata = Mockito.spy(ObjectMetadata.class);
metadata.setContentDisposition("key/path/to/file.txt");
Mockito.when(metadata.getVersionId()).thenReturn("response-version");
s3ObjectResponse.setObjectMetadata(metadata);
Mockito.when(mockS3Client.getObject(Mockito.any())).thenReturn(s3ObjectResponse);
runner.run(1);
ArgumentCaptor<GetObjectRequest> captureRequest = ArgumentCaptor.forClass(GetObjectRequest.class);
Mockito.verify(mockS3Client, Mockito.times(1)).getObject(captureRequest.capture());
GetObjectRequest request = captureRequest.getValue();
assertEquals("request-bucket", request.getBucketName());
assertEquals("request-key", request.getKey());
assertEquals("request-version", request.getVersionId());
runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
MockFlowFile ff = ffs.get(0);
ff.assertAttributeEquals("s3.bucket", "response-bucket-name");
ff.assertAttributeEquals(CoreAttributes.FILENAME.key(), "file.txt");
ff.assertAttributeEquals(CoreAttributes.PATH.key(), "key/path/to");
ff.assertAttributeEquals(CoreAttributes.ABSOLUTE_PATH.key(), "key/path/to/file.txt");
ff.assertAttributeEquals("s3.version", "response-version");
ff.assertContentEquals("Some Content");
}
@Override
public Iterator<S3Object> listFiles(String folder) throws FileSystemException {
List<S3ObjectSummary> summaries = null;
String prefix = folder != null ? folder + "/" : "";
try {
ObjectListing listing = s3Client.listObjects(bucketName, prefix);
summaries = listing.getObjectSummaries();
while (listing.isTruncated()) {
listing = s3Client.listNextBatchOfObjects(listing);
summaries.addAll(listing.getObjectSummaries());
}
} catch (AmazonServiceException e) {
throw new FileSystemException("Cannot process requested action", e);
}
List<S3Object> list = new ArrayList<S3Object>();
for (S3ObjectSummary summary : summaries) {
S3Object object = new S3Object();
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(summary.getSize());
object.setBucketName(summary.getBucketName());
object.setKey(summary.getKey());
object.setObjectMetadata(metadata);
if(!object.getKey().endsWith("/") && !(prefix.isEmpty() && object.getKey().contains("/"))) {
list.add(object);
}
}
return list.iterator();
}
@Test
public void getSplit() throws Exception {
// Intentionally create the gzip file as concatenated gzip files to verify the Java bug involving reading
// concatenated gzip files is resolved.
List<Map<String, Object>> expected = Lists.newArrayListWithCapacity(4);
ByteArrayOutputStream splitOut = new ByteArrayOutputStream();
for (int p=0; p < 2; p++) {
for (int i=0; i < 2; i++) {
ByteArrayOutputStream partialOut = new ByteArrayOutputStream();
try (BufferedWriter out = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(partialOut)))) {
Map<String, Object> value = ImmutableMap.<String, Object>builder()
.put("~id", "row" + (p * 2 + i))
.put("~table", "test:table")
.put("~version", 1)
.put("~signature", "3a0da59fabf298d389b7b0b59728e887")
.put("~lastUpdateAt", "2014-08-28T21:24:36.440Z")
.put("~firstUpdateAt", "2014-06-07T09:51:40.077Z")
.build();
String json = JsonHelper.asJson(value);
out.write(json);
out.write("\n");
expected.add(value);
}
partialOut.writeTo(splitOut);
}
}
splitOut.close();
S3Object s3Object = new S3Object();
s3Object.setObjectContent(new ByteArrayInputStream(splitOut.toByteArray(), 0, splitOut.size()));
ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.setContentLength(splitOut.size());
s3Object.setObjectMetadata(objectMetadata);
AmazonS3 s3 = mock(AmazonS3.class);
when(s3.getObject(argThat(getsObject("stash-bucket", "stash/test/2015-01-01-00-00-00/test-table/split0.gz"))))
.thenReturn(s3Object);
StashSplit stashSplit = new StashSplit("test:table", "2015-01-01-00-00-00/test-table/split0.gz", splitOut.size());
StandardStashReader reader = new StandardStashReader(URI.create("s3://stash-bucket/stash/test"), s3, 0);
StashRowIterator contentIter = reader.getSplit(stashSplit);
List<Map<String, Object>> content = ImmutableList.copyOf(contentIter);
assertEquals(content, expected);
}
/**
* Note: This test depends on the default character set to be US-ASCII
*/
@Test
public void testUTF8ClientDecoding() throws Exception {
if (!Charset.defaultCharset().equals(Charsets.US_ASCII)) {
throw new SkipException("testUTF8ClientDecoding() requires default charset to be US-ASCII");
}
String utf8Text = "ชิงรางวัลกันจ้า";
ByteArrayOutputStream splitOut = new ByteArrayOutputStream();
try (BufferedWriter out = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(splitOut), Charsets.UTF_8))) {
Map<String, Object> value = ImmutableMap.<String, Object>builder()
.put("~id", "1")
.put("~table", "test:table")
.put("~version", 1)
.put("~signature", "3a0da59fabf298d389b7b0b59728e887")
.put("~lastUpdateAt", "2014-08-28T21:24:36.440Z")
.put("~firstUpdateAt", "2014-06-07T09:51:40.077Z")
.put("text", utf8Text)
.build();
String json = JsonHelper.asJson(value);
out.write(json);
out.write("\n");
}
splitOut.close();
S3Object s3Object = new S3Object();
s3Object.setObjectContent(new ByteArrayInputStream(splitOut.toByteArray(), 0, splitOut.size()));
ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.setContentLength(splitOut.size());
s3Object.setObjectMetadata(objectMetadata);
AmazonS3 s3 = mock(AmazonS3.class);
when(s3.getObject(argThat(getsObject("stash-bucket", "stash/test/2015-01-01-00-00-00/test-table/split0.gz"))))
.thenReturn(s3Object);
StashSplit stashSplit = new StashSplit("test:table", "2015-01-01-00-00-00/test-table/split0.gz", splitOut.size());
StandardStashReader reader = new StandardStashReader(URI.create("s3://stash-bucket/stash/test"), s3, 0);
try (StashRowIterator rowIter = reader.getSplit(stashSplit)) {
assertTrue(rowIter.hasNext());
Map<String, Object> row = rowIter.next();
assertEquals(row.get("text"), utf8Text);
}
}
@Test
public void testGetObject() throws IOException {
runner.setProperty(FetchS3Object.REGION, "us-east-1");
runner.setProperty(FetchS3Object.BUCKET, "request-bucket");
final Map<String, String> attrs = new HashMap<>();
attrs.put("filename", "request-key");
runner.enqueue(new byte[0], attrs);
S3Object s3ObjectResponse = new S3Object();
s3ObjectResponse.setBucketName("response-bucket-name");
s3ObjectResponse.setKey("response-key");
s3ObjectResponse.setObjectContent(new StringInputStream("Some Content"));
ObjectMetadata metadata = Mockito.spy(ObjectMetadata.class);
metadata.setContentDisposition("key/path/to/file.txt");
metadata.setContentType("text/plain");
metadata.setContentMD5("testMD5hash");
Date expiration = new Date();
metadata.setExpirationTime(expiration);
metadata.setExpirationTimeRuleId("testExpirationRuleId");
Map<String, String> userMetadata = new HashMap<>();
userMetadata.put("userKey1", "userValue1");
userMetadata.put("userKey2", "userValue2");
metadata.setUserMetadata(userMetadata);
metadata.setSSEAlgorithm("testAlgorithm");
Mockito.when(metadata.getETag()).thenReturn("test-etag");
s3ObjectResponse.setObjectMetadata(metadata);
Mockito.when(mockS3Client.getObject(Mockito.any())).thenReturn(s3ObjectResponse);
runner.run(1);
ArgumentCaptor<GetObjectRequest> captureRequest = ArgumentCaptor.forClass(GetObjectRequest.class);
Mockito.verify(mockS3Client, Mockito.times(1)).getObject(captureRequest.capture());
GetObjectRequest request = captureRequest.getValue();
assertEquals("request-bucket", request.getBucketName());
assertEquals("request-key", request.getKey());
assertFalse(request.isRequesterPays());
assertNull(request.getVersionId());
runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
MockFlowFile ff = ffs.get(0);
ff.assertAttributeEquals("s3.bucket", "response-bucket-name");
ff.assertAttributeEquals(CoreAttributes.FILENAME.key(), "file.txt");
ff.assertAttributeEquals(CoreAttributes.PATH.key(), "key/path/to");
ff.assertAttributeEquals(CoreAttributes.ABSOLUTE_PATH.key(), "key/path/to/file.txt");
ff.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "text/plain");
ff.assertAttributeEquals("hash.value", "testMD5hash");
ff.assertAttributeEquals("hash.algorithm", "MD5");
ff.assertAttributeEquals("s3.etag", "test-etag");
ff.assertAttributeEquals("s3.expirationTime", String.valueOf(expiration.getTime()));
ff.assertAttributeEquals("s3.expirationTimeRuleId", "testExpirationRuleId");
ff.assertAttributeEquals("userKey1", "userValue1");
ff.assertAttributeEquals("userKey2", "userValue2");
ff.assertAttributeEquals("s3.sseAlgorithm", "testAlgorithm");
ff.assertContentEquals("Some Content");
}
@Test
public void testGetObjectWithRequesterPays() throws IOException {
runner.setProperty(FetchS3Object.REGION, "us-east-1");
runner.setProperty(FetchS3Object.BUCKET, "request-bucket");
runner.setProperty(FetchS3Object.REQUESTER_PAYS, "true");
final Map<String, String> attrs = new HashMap<>();
attrs.put("filename", "request-key");
runner.enqueue(new byte[0], attrs);
S3Object s3ObjectResponse = new S3Object();
s3ObjectResponse.setBucketName("response-bucket-name");
s3ObjectResponse.setKey("response-key");
s3ObjectResponse.setObjectContent(new StringInputStream("Some Content"));
ObjectMetadata metadata = Mockito.spy(ObjectMetadata.class);
metadata.setContentDisposition("key/path/to/file.txt");
metadata.setContentType("text/plain");
metadata.setContentMD5("testMD5hash");
Date expiration = new Date();
metadata.setExpirationTime(expiration);
metadata.setExpirationTimeRuleId("testExpirationRuleId");
Map<String, String> userMetadata = new HashMap<>();
userMetadata.put("userKey1", "userValue1");
userMetadata.put("userKey2", "userValue2");
metadata.setUserMetadata(userMetadata);
metadata.setSSEAlgorithm("testAlgorithm");
Mockito.when(metadata.getETag()).thenReturn("test-etag");
s3ObjectResponse.setObjectMetadata(metadata);
Mockito.when(mockS3Client.getObject(Mockito.any())).thenReturn(s3ObjectResponse);
runner.run(1);
ArgumentCaptor<GetObjectRequest> captureRequest = ArgumentCaptor.forClass(GetObjectRequest.class);
Mockito.verify(mockS3Client, Mockito.times(1)).getObject(captureRequest.capture());
GetObjectRequest request = captureRequest.getValue();
assertEquals("request-bucket", request.getBucketName());
assertEquals("request-key", request.getKey());
assertTrue(request.isRequesterPays());
assertNull(request.getVersionId());
runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
MockFlowFile ff = ffs.get(0);
ff.assertAttributeEquals("s3.bucket", "response-bucket-name");
ff.assertAttributeEquals(CoreAttributes.FILENAME.key(), "file.txt");
ff.assertAttributeEquals(CoreAttributes.PATH.key(), "key/path/to");
ff.assertAttributeEquals(CoreAttributes.ABSOLUTE_PATH.key(), "key/path/to/file.txt");
ff.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "text/plain");
ff.assertAttributeEquals("hash.value", "testMD5hash");
ff.assertAttributeEquals("hash.algorithm", "MD5");
ff.assertAttributeEquals("s3.etag", "test-etag");
ff.assertAttributeEquals("s3.expirationTime", String.valueOf(expiration.getTime()));
ff.assertAttributeEquals("s3.expirationTimeRuleId", "testExpirationRuleId");
ff.assertAttributeEquals("userKey1", "userValue1");
ff.assertAttributeEquals("userKey2", "userValue2");
ff.assertAttributeEquals("s3.sseAlgorithm", "testAlgorithm");
ff.assertContentEquals("Some Content");
}