下面列出了怎么用com.amazonaws.services.s3.model.ObjectMetadata的API类实例代码及写法,或者点击链接到github查看源代码。
private ObjectMetadata getRequiredObjectMetadata() throws FileNotFoundException {
ObjectMetadata metadata = getObjectMetadata();
if (metadata == null) {
StringBuilder builder = new StringBuilder().append("Resource with bucket='")
.append(this.bucketName).append("' and objectName='")
.append(this.objectName);
if (this.versionId != null) {
builder.append("' and versionId='");
builder.append(this.versionId);
}
builder.append("' not found!");
throw new FileNotFoundException(builder.toString());
}
return metadata;
}
/**
* Creates an object in S3 with the prefix constructed from the given parameters. The object's full path will be {prefix}/{UUID}
*
* @param businessObjectFormatEntity business object format
* @param request request with partition values and storage
* @param businessObjectDataVersion business object data version to put
*/
public void createS3Object(BusinessObjectFormatEntity businessObjectFormatEntity, BusinessObjectDataInvalidateUnregisteredRequest request,
int businessObjectDataVersion)
{
StorageEntity storageEntity = storageDao.getStorageByName(request.getStorageName());
String s3BucketName = storageHelper.getS3BucketAccessParams(storageEntity).getS3BucketName();
BusinessObjectDataKey businessObjectDataKey = getBusinessObjectDataKey(request);
businessObjectDataKey.setBusinessObjectDataVersion(businessObjectDataVersion);
String s3KeyPrefix = s3KeyPrefixHelper
.buildS3KeyPrefix(AbstractServiceTest.S3_KEY_PREFIX_VELOCITY_TEMPLATE, businessObjectFormatEntity, businessObjectDataKey, storageEntity.getName());
String s3ObjectKey = s3KeyPrefix + "/test";
PutObjectRequest putObjectRequest = new PutObjectRequest(s3BucketName, s3ObjectKey, new ByteArrayInputStream(new byte[1]), new ObjectMetadata());
s3Operations.putObject(putObjectRequest, null);
}
private ObjectMetadata getObjectMetadata() {
if (this.objectMetadata == null) {
try {
GetObjectMetadataRequest metadataRequest = new GetObjectMetadataRequest(
this.bucketName, this.objectName);
if (this.versionId != null) {
metadataRequest.setVersionId(this.versionId);
}
this.objectMetadata = this.amazonS3.getObjectMetadata(metadataRequest);
}
catch (AmazonS3Exception e) {
// Catch 404 (object not found) and 301 (bucket not found, moved
// permanently)
if (e.getStatusCode() == 404 || e.getStatusCode() == 301) {
this.objectMetadata = null;
}
else {
throw e;
}
}
}
return this.objectMetadata;
}
@Test
void testGetResourceWithVersionId() {
AmazonS3 amazonS3 = mock(AmazonS3.class);
SimpleStorageProtocolResolver resourceLoader = new SimpleStorageProtocolResolver(
amazonS3);
ObjectMetadata metadata = new ObjectMetadata();
when(amazonS3.getObjectMetadata(any(GetObjectMetadataRequest.class)))
.thenReturn(metadata);
String resourceName = "s3://bucket/object^versionIdValue";
Resource resource = resourceLoader.resolve(resourceName,
new DefaultResourceLoader());
assertThat(resource).isNotNull();
}
@Test
void lastModified_withExistingResource_returnsLastModifiedDateOfResource()
throws Exception {
// Arrange
AmazonS3 amazonS3 = mock(AmazonS3.class);
ObjectMetadata objectMetadata = new ObjectMetadata();
Date lastModified = new Date();
objectMetadata.setLastModified(lastModified);
when(amazonS3.getObjectMetadata(any(GetObjectMetadataRequest.class)))
.thenReturn(objectMetadata);
// Act
SimpleStorageResource simpleStorageResource = new SimpleStorageResource(amazonS3,
"bucket", "object", new SyncTaskExecutor());
// Assert
assertThat(simpleStorageResource.lastModified())
.isEqualTo(lastModified.getTime());
}
@VisibleForTesting
CopyObjectResult atomicCopy(
S3ResourceId sourcePath, S3ResourceId destinationPath, ObjectMetadata sourceObjectMetadata)
throws AmazonClientException {
CopyObjectRequest copyObjectRequest =
new CopyObjectRequest(
sourcePath.getBucket(),
sourcePath.getKey(),
destinationPath.getBucket(),
destinationPath.getKey());
copyObjectRequest.setNewObjectMetadata(sourceObjectMetadata);
copyObjectRequest.setStorageClass(options.getS3StorageClass());
copyObjectRequest.setSourceSSECustomerKey(options.getSSECustomerKey());
copyObjectRequest.setDestinationSSECustomerKey(options.getSSECustomerKey());
return amazonS3.get().copyObject(copyObjectRequest);
}
private MultiPartUpload initiateMultiPartUpload() throws IOException {
final ObjectMetadata om = createDefaultMetadata();
final InitiateMultipartUploadRequest initiateMPURequest =
new InitiateMultipartUploadRequest(bucket, key, om);
initiateMPURequest.setCannedACL(cannedACL);
try {
return new MultiPartUpload(
client.initiateMultipartUpload(initiateMPURequest).getUploadId());
} catch (AmazonServiceException ase) {
throw new IOException("Unable to initiate MultiPartUpload (server side)" +
": " + ase, ase);
} catch (AmazonClientException ace) {
throw new IOException("Unable to initiate MultiPartUpload (client side)" +
": " + ace, ace);
}
}
@Test
public void testDeleteBusinessObjectDataS3FileExists() throws Exception
{
// Prepare database entries required for testing.
StorageEntity storageEntity = createTestStorageEntity(STORAGE_NAME, s3BucketName);
BusinessObjectDataKey testBusinessObjectKey =
new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES,
DATA_VERSION);
createTestDatabaseEntities(testBusinessObjectKey, BusinessObjectDataStatusEntity.UPLOADING, storageEntity, TARGET_S3_KEY, 15);
// Put a file in S3.
PutObjectRequest putObjectRequest = new PutObjectRequest(s3BucketName, TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), new ObjectMetadata());
s3Operations.putObject(putObjectRequest, null);
// Delete the business object data.
List<BusinessObjectDataKey> resultBusinessObjectDataKeys = fileUploadCleanupService.deleteBusinessObjectData(STORAGE_NAME, 10);
// Validate the results.
assertNotNull(resultBusinessObjectDataKeys);
assertTrue(resultBusinessObjectDataKeys.isEmpty());
validateBusinessObjectDataStatus(testBusinessObjectKey, BusinessObjectDataStatusEntity.UPLOADING);
}
@Test
public void testSimpleGetEncrypted() throws IOException {
putTestFileEncrypted("test-file", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
final TestRunner runner = TestRunners.newTestRunner(new FetchS3Object());
runner.setProperty(FetchS3Object.CREDENTIALS_FILE, CREDENTIALS_FILE);
runner.setProperty(FetchS3Object.REGION, REGION);
runner.setProperty(FetchS3Object.BUCKET, BUCKET_NAME);
final Map<String, String> attrs = new HashMap<>();
attrs.put("filename", "test-file");
runner.enqueue(new byte[0], attrs);
runner.run(1);
runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
MockFlowFile ff = ffs.get(0);
ff.assertAttributeEquals(PutS3Object.S3_SSE_ALGORITHM, ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
ff.assertContentEquals(getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
}
@Test
public void testTagObjects()
{
// Create an S3 file transfer request parameters DTO to access S3 objects.
S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
params.setS3BucketName(S3_BUCKET_NAME);
// Create an S3 object summary.
S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
s3ObjectSummary.setKey(TARGET_S3_KEY);
// Create an S3 object tag.
Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);
// Put a file in S3.
s3Operations.putObject(new PutObjectRequest(S3_BUCKET_NAME, TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), new ObjectMetadata()), null);
// Tag the file with an S3 object tag.
s3Dao.tagObjects(params, new S3FileTransferRequestParamsDto(), Collections.singletonList(s3ObjectSummary), tag);
// Validate that the object got tagged.
GetObjectTaggingResult getObjectTaggingResult = s3Operations.getObjectTagging(new GetObjectTaggingRequest(S3_BUCKET_NAME, TARGET_S3_KEY), null);
assertEquals(Collections.singletonList(tag), getObjectTaggingResult.getTagSet());
}
private void upload(@NotNull File src, @NotNull String key, @NotNull CallbackPayload payload) {
log.info("Uploading {} to {} in {}", src, key, bucketName);
final PutObjectRequest request = new PutObjectRequest(bucketName, key, src);
final ObjectMetadata metadata = request.getMetadata();
final String commitId = payload.getAfter();
if (commitId != null) {
metadata.addUserMetadata("commit-id", commitId);
}
final DateTime timestamp = payload.getTimestamp();
if (timestamp != null) {
metadata.addUserMetadata("hook-timestamp",
ISODateTimeFormat.basicTime().print(timestamp));
}
final TimerContext time = uploadTimer.time();
try {
s3Client.putObject(request);
} catch (Exception e) {
log.error("Couldn't upload to {} in {}", key, bucketName, e);
throw e;
} finally {
time.stop();
}
log.info("Uploaded to {} in {}", key, bucketName);
}
public String putObject(String bucketName, String originFileName, MultipartFile multipartFile) {
String uuid = UUID.randomUUID().toString().replaceAll("-", "");
String fileName = FILE + "_" + uuid + "_" + originFileName;
try {
InputStream inputStream = multipartFile.getInputStream();
if (isAwsS3) {
ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.addUserMetadata("Content-Type", "application/octet-stream");
amazonS3.putObject(bucketName, fileName, inputStream, objectMetadata);
} else {
minioClient.putObject(bucketName, fileName, inputStream, "application/octet-stream");
}
} catch (Exception e) {
throw new FileUploadException("error.file.upload", e);
}
return fileName;
}
/**
* Get MD5, size, owner, storage class and last modification time for a desired file in the pointed bucket
*
* @param fileName the file name
*/
@PublicAtsApi
public S3ObjectInfo getFileMetadata( String fileName ) {
try {
S3Object element = s3Client.getObject(bucketName, fileName);
if (element != null) {
ObjectMetadata metaData = element.getObjectMetadata();
S3ObjectInfo s3Info = new S3ObjectInfo();
s3Info.setBucketName(fileName);
s3Info.setLastModified(metaData.getLastModified());
s3Info.setMd5(metaData.getETag());
s3Info.setName(element.getKey());
s3Info.setSize(metaData.getContentLength());
return s3Info;
} else {
throw new NoSuchElementException("File with name '" + fileName + "' does not exist!");
}
} catch (Exception e) {
handleExeption(e, "Could not retrieve metadata for S3 object with key '" + fileName + "'");
}
return null;
}
private void createEmptyObject(final String bucketName, final String objectName)
throws AmazonClientException, AmazonServiceException {
final InputStream im = new InputStream() {
@Override
public int read() throws IOException {
return -1;
}
};
final ObjectMetadata om = new ObjectMetadata();
om.setContentLength(0L);
if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
om.setServerSideEncryption(serverSideEncryptionAlgorithm);
}
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, objectName, im, om);
putObjectRequest.setCannedAcl(cannedACL);
s3.putObject(putObjectRequest);
statistics.incrementWriteOps(1);
}
@Override
@Guarded(by = STARTED)
public Blob create(final InputStream blobData, final Map<String, String> headers) {
checkNotNull(blobData);
return create(headers, destination -> {
try (InputStream data = blobData) {
MetricsInputStream input = new MetricsInputStream(data);
TransferManager transferManager = new TransferManager(s3);
transferManager.upload(getConfiguredBucket(), destination, input, new ObjectMetadata())
.waitForCompletion();
return input.getMetrics();
} catch (InterruptedException e) {
throw new BlobStoreException("error uploading blob", e, null);
}
});
}
Upload doUpload(String bucket, String fileName, InputStream is, ObjectMetadata metadata) {
final PutObjectRequest putObjectRequest = new PutObjectRequest(
bucket,
fileName,
is,
metadata
);
final String object = bucket + s3TargetConfigBean.s3Config.delimiter + fileName;
Upload upload = transferManager.upload(putObjectRequest);
upload.addProgressListener((ProgressListener) progressEvent -> {
switch (progressEvent.getEventType()) {
case TRANSFER_STARTED_EVENT:
LOG.debug("Started uploading object {} into Amazon S3", object);
break;
case TRANSFER_COMPLETED_EVENT:
LOG.debug("Completed uploading object {} into Amazon S3", object);
break;
case TRANSFER_FAILED_EVENT:
LOG.debug("Failed uploading object {} into Amazon S3", object);
break;
default:
break;
}
});
return upload;
}
/**
* The method is successful when both bucket and key exists.
*/
@Test
public void testS3FileExists()
{
String expectedKey = "foo";
String expectedValue = "bar";
ByteArrayInputStream inputStream = new ByteArrayInputStream((expectedKey + "=" + expectedValue).getBytes());
PutObjectRequest putObjectRequest = new PutObjectRequest(S3_BUCKET_NAME, TARGET_S3_KEY, inputStream, new ObjectMetadata());
s3Operations.putObject(putObjectRequest, null);
S3FileTransferRequestParamsDto params = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto();
params.setS3BucketName(S3_BUCKET_NAME);
params.setS3KeyPrefix(TARGET_S3_KEY);
Assert.assertTrue(s3Dao.s3FileExists(params));
}
@Override
public ObjectMetadata getObjectMetadata(final S3FileTransferRequestParamsDto params)
{
AmazonS3Client s3Client = getAmazonS3(params);
try
{
return s3Operations.getObjectMetadata(params.getS3BucketName(), params.getS3KeyPrefix(), s3Client);
}
catch (AmazonServiceException e)
{
if (e.getStatusCode() == HttpStatus.SC_NOT_FOUND)
{
return null;
}
throw new IllegalStateException(String
.format("Failed to get S3 metadata for object key \"%s\" from bucket \"%s\". Reason: %s", params.getS3KeyPrefix(), params.getS3BucketName(),
e.getMessage()), e);
}
finally
{
// Shutdown the AmazonS3Client instance to release resources.
s3Client.shutdown();
}
}
private void testEncryptionServiceWithServerSideS3EncryptionStrategy(byte[] data) throws IOException, InitializationException {
TestRunner runner = createPutEncryptionTestRunner(AmazonS3EncryptionService.STRATEGY_NAME_SSE_S3, null);
Map<String, String> attrs = new HashMap<>();
attrs.put("filename", "test.txt");
runner.enqueue(data, attrs);
runner.assertValid();
runner.run();
runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS);
List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(PutS3Object.REL_SUCCESS);
Assert.assertEquals(1, flowFiles.size());
Assert.assertEquals(0, runner.getFlowFilesForRelationship(PutS3Object.REL_FAILURE).size());
MockFlowFile putSuccess = flowFiles.get(0);
Assert.assertEquals(putSuccess.getAttribute(PutS3Object.S3_ENCRYPTION_STRATEGY), AmazonS3EncryptionService.STRATEGY_NAME_SSE_S3);
MockFlowFile flowFile = fetchEncryptedFlowFile(attrs, AmazonS3EncryptionService.STRATEGY_NAME_SSE_S3, null);
flowFile.assertContentEquals(data);
flowFile.assertAttributeEquals(PutS3Object.S3_SSE_ALGORITHM, ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
flowFile.assertAttributeEquals(PutS3Object.S3_ENCRYPTION_STRATEGY, AmazonS3EncryptionService.STRATEGY_NAME_SSE_S3);
}
@Test
public void testAwsV4SignatureBadCredential() throws Exception {
client = AmazonS3ClientBuilder.standard()
.withCredentials(new AWSStaticCredentialsProvider(
new BasicAWSCredentials(
awsCreds.getAWSAccessKeyId(),
"bad-secret-key")))
.withEndpointConfiguration(s3EndpointConfig)
.build();
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(BYTE_SOURCE.size());
try {
client.putObject(containerName, "foo",
BYTE_SOURCE.openStream(), metadata);
Fail.failBecauseExceptionWasNotThrown(AmazonS3Exception.class);
} catch (AmazonS3Exception e) {
assertThat(e.getErrorCode()).isEqualTo("SignatureDoesNotMatch");
}
}
@Test
public void create() throws URISyntaxException {
S3LocationResolver resolver = new S3LocationResolver(s3, "bucket", "prefix");
resolver.resolveLocation("tableName", true);
ArgumentCaptor<ByteArrayInputStream> contentCaptor = ArgumentCaptor.forClass(ByteArrayInputStream.class);
ArgumentCaptor<ObjectMetadata> metadataCaptor = ArgumentCaptor.forClass(ObjectMetadata.class);
verify(s3).putObject(eq("bucket"), eq("prefix/tableName/"), contentCaptor.capture(), metadataCaptor.capture());
assertThat(contentCaptor.getValue().available(), is(0));
assertThat(metadataCaptor.getValue().getContentLength(), is(0L));
}
/**
* Create a new object metadata instance.
* Any standard metadata headers are added here, for example:
* encryption
* @param length size, if known. Use -1 for not known
* @return a new metadata instance
*/
public ObjectMetadata newObjectMetadata(long length) {
final ObjectMetadata om = new ObjectMetadata();
if (length >= 0) {
om.setContentLength(length);
}
return om;
}
@Test(expected = MetaStoreException.class)
public void resolveAmazonClientExceptionOnUpload() throws Exception {
when(transferManager.upload(anyString(), anyString(), any(InputStream.class), any(ObjectMetadata.class)))
.thenThrow(AmazonClientException.class);
uriResolver.resolve(SCHEMA, ROAD, VERSION);
}
@Test(expected = MetaStoreException.class)
public void resolveInterruptedExceptionOnUpload() throws Exception {
when(transferManager.upload(anyString(), anyString(), any(InputStream.class), any(ObjectMetadata.class)))
.thenThrow(InterruptedException.class);
uriResolver.resolve(SCHEMA, ROAD, VERSION);
}
private boolean walk(Iterator<S3ObjectSummary> iter, ObjectId file, String path) throws IOException {
byte[] content;
byte[] newHash;
LOG.debug("Start processing file: {}", path);
try (DigestInputStream is = new DigestInputStream(repository.open(file).openStream(), DigestUtils.getMd5Digest())) {
// Get content
content = IOUtils.toByteArray(is);
// Get hash
newHash = is.getMessageDigest().digest();
}
if (isUploadFile(iter, path, Hex.encodeHexString(newHash))) {
LOG.info("Uploading file: {}", path);
ObjectMetadata bucketMetadata = new ObjectMetadata();
bucketMetadata.setContentMD5(Base64.encodeAsString(newHash));
bucketMetadata.setContentLength(content.length);
// Give Tika a few hints for the content detection
Metadata tikaMetadata = new Metadata();
tikaMetadata.set(Metadata.RESOURCE_NAME_KEY, FilenameUtils.getName(FilenameUtils.normalize(path)));
// Fire!
try (InputStream bis = TikaInputStream.get(content, tikaMetadata)) {
bucketMetadata.setContentType(TIKA_DETECTOR.detect(bis, tikaMetadata).toString());
s3.putObject(bucket.getName(), path, bis, bucketMetadata);
return true;
}
}
LOG.info("Skipping file (same checksum): {}", path);
return false;
}
protected void writeToS3(final String content, final String prefix, final String instanceId) {
final byte[] bytes = content.getBytes(UTF_8);
final InputStream stream = new ByteArrayInputStream(bytes);
final ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(bytes.length);
final String fileName = instanceId + SECURITY_GROUPS + new DateTime(UTC) + JSON;
s3Writer.putObjectToS3(bucketName, fileName, prefix, metadata, stream);
}
private void copyFile(String srcKey, String dstKey) throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("copyFile " + srcKey + " -> " + dstKey);
}
ObjectMetadata srcom = s3.getObjectMetadata(bucket, srcKey);
final ObjectMetadata dstom = srcom.clone();
if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
dstom.setServerSideEncryption(serverSideEncryptionAlgorithm);
}
CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucket, srcKey, bucket, dstKey);
copyObjectRequest.setCannedAccessControlList(cannedACL);
copyObjectRequest.setNewObjectMetadata(dstom);
ProgressListener progressListener = new ProgressListener() {
public void progressChanged(ProgressEvent progressEvent) {
switch (progressEvent.getEventCode()) {
case ProgressEvent.PART_COMPLETED_EVENT_CODE:
statistics.incrementWriteOps(1);
break;
default:
break;
}
}
};
Copy copy = transfers.copy(copyObjectRequest);
copy.addProgressListener(progressListener);
try {
copy.waitForCopyResult();
statistics.incrementWriteOps(1);
} catch (InterruptedException e) {
throw new IOException("Got interrupted, cancelling");
}
}
public S3StorageObjectMetadata(ObjectMetadata s3Metadata)
{
if (s3Metadata == null)
{
throw new IllegalArgumentException("s3Metadata must not be null");
}
this.s3Metadata = s3Metadata;
}
private void uploadContent(String bucketName, String objectKey, byte[] content, String contentType)
{
ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.setContentType(contentType);
objectMetadata.setContentLength(content.length);
InputStream inputStream = new ByteArrayInputStream(content);
amazonS3Client.putObject(bucketName, objectKey, inputStream, objectMetadata);
}
@Override
public FileStatus getFileStatus(Path path)
throws IOException
{
if (path.getName().isEmpty()) {
// the bucket root requires special handling
if (getS3ObjectMetadata(path) != null) {
return new FileStatus(0, true, 1, 0, 0, qualifiedPath(path));
}
throw new FileNotFoundException("File does not exist: " + path);
}
ObjectMetadata metadata = getS3ObjectMetadata(path);
if (metadata == null) {
// check if this path is a directory
Iterator<LocatedFileStatus> iterator = listPrefix(path);
if (iterator.hasNext()) {
return new FileStatus(0, true, 1, 0, 0, qualifiedPath(path));
}
throw new FileNotFoundException("File does not exist: " + path);
}
return new FileStatus(
getObjectSize(path, metadata),
S3_DIRECTORY_OBJECT_CONTENT_TYPE.equals(metadata.getContentType()),
1,
BLOCK_SIZE.toBytes(),
lastModifiedTime(metadata),
qualifiedPath(path));
}