下面列出了com.amazonaws.services.s3.model.ObjectMetadata#setContentLength ( ) 实例代码，或者点击链接到github查看源代码，也可以在右侧发表评论。
@Test
public void testAwsV4SignatureChunkedAnonymous() throws Exception {
client = AmazonS3ClientBuilder.standard()
.withChunkedEncodingDisabled(false)
.withEndpointConfiguration(s3EndpointConfig)
.build();
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(BYTE_SOURCE.size());
client.putObject(containerName, "foo", BYTE_SOURCE.openStream(),
metadata);
S3Object object = client.getObject(containerName, "foo");
assertThat(object.getObjectMetadata().getContentLength()).isEqualTo(
BYTE_SOURCE.size());
try (InputStream actual = object.getObjectContent();
InputStream expected = BYTE_SOURCE.openStream()) {
assertThat(actual).hasContentEqualTo(expected);
}
}
private void writeBufferToS3(byte[] toWrite, int limit) {
try (EtlProfilingScope scope = new EtlProfilingScope(parentMetrics, "S3FastLoader.writeToS3")) {
InputStream inputStream = new ByteArrayInputStream(toWrite, 0, limit);
String s3Key = s3PartFileKeyGenerator.apply(++fileSequenceNumber);
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(limit);
PutObjectRequest putObjectRequest = new PutObjectRequest(s3Bucket, s3Key, inputStream, metadata);
if (sseKmsArn != null && !sseKmsArn.isEmpty()) {
putObjectRequest.setSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(sseKmsArn));
}
try {
amazonS3.putObject(putObjectRequest);
emitSuccessAndFailureMetrics(scope, true);
} catch (AmazonClientException e) {
logger.error(e);
scope.addCounter(e.getClass().getSimpleName(), 1);
emitSuccessAndFailureMetrics(scope, false);
throw new UnrecoverableStreamFailureException("Exception caught trying to write object to S3: ", e);
}
}
}
@Test
public void testBlobRemove() throws Exception {
String blobName = "blob";
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(BYTE_SOURCE.size());
client.putObject(containerName, blobName, BYTE_SOURCE.openStream(),
metadata);
assertThat(client.getObjectMetadata(containerName, blobName))
.isNotNull();
client.deleteObject(containerName, blobName);
try {
client.getObjectMetadata(containerName, blobName);
Fail.failBecauseExceptionWasNotThrown(AmazonS3Exception.class);
} catch (AmazonS3Exception e) {
assertThat(e.getErrorCode()).isEqualTo("404 Not Found");
}
client.deleteObject(containerName, blobName);
}
@Override
public void store(BuildCacheKey key, BuildCacheEntryWriter writer) {
final String bucketPath = getBucketPath(key);
logger.info("Start storing cache entry '{}' in S3 bucket", bucketPath);
ObjectMetadata meta = new ObjectMetadata();
meta.setContentType(BUILD_CACHE_CONTENT_TYPE);
try (ByteArrayOutputStream os = new ByteArrayOutputStream()) {
writer.writeTo(os);
meta.setContentLength(os.size());
try (InputStream is = new ByteArrayInputStream(os.toByteArray())) {
PutObjectRequest request = getPutObjectRequest(bucketPath, meta, is);
if(this.reducedRedundancy) {
request.withStorageClass(StorageClass.ReducedRedundancy);
}
s3.putObject(request);
}
} catch (IOException e) {
throw new BuildCacheException("Error while storing cache object in S3 bucket", e);
}
}
@Test
public void testHttpClient() throws Exception {
String blobName = "blob-name";
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(BYTE_SOURCE.size());
client.putObject(containerName, blobName, BYTE_SOURCE.openStream(),
metadata);
if (Quirks.NO_BLOB_ACCESS_CONTROL.contains(blobStoreType)) {
client.setBucketAcl(containerName,
CannedAccessControlList.PublicRead);
} else {
client.setObjectAcl(containerName, blobName,
CannedAccessControlList.PublicRead);
}
HttpClient httpClient = context.utils().http();
URI uri = new URI(s3Endpoint.getScheme(), s3Endpoint.getUserInfo(),
s3Endpoint.getHost(), s3Proxy.getSecurePort(),
servicePath + "/" + containerName + "/" + blobName,
/*query=*/ null, /*fragment=*/ null);
try (InputStream actual = httpClient.get(uri);
InputStream expected = BYTE_SOURCE.openStream()) {
assertThat(actual).hasContentEqualTo(expected);
}
}
@Test
public void testUnicodeObject() throws Exception {
String blobName = "ŪņЇЌœđЗ/☺ unicode € rocks ™";
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(BYTE_SOURCE.size());
client.putObject(containerName, blobName, BYTE_SOURCE.openStream(),
metadata);
metadata = client.getObjectMetadata(containerName, blobName);
assertThat(metadata).isNotNull();
ObjectListing listing = client.listObjects(containerName);
List<S3ObjectSummary> summaries = listing.getObjectSummaries();
assertThat(summaries).hasSize(1);
S3ObjectSummary summary = summaries.iterator().next();
assertThat(summary.getKey()).isEqualTo(blobName);
}
@Test
public void testDeleteMultipleObjects() throws Exception {
String blobName = "foo";
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(BYTE_SOURCE.size());
DeleteObjectsRequest request = new DeleteObjectsRequest(containerName)
.withKeys(blobName);
// without quiet
client.putObject(containerName, blobName, BYTE_SOURCE.openStream(),
metadata);
DeleteObjectsResult result = client.deleteObjects(request);
assertThat(result.getDeletedObjects()).hasSize(1);
assertThat(result.getDeletedObjects().iterator().next().getKey())
.isEqualTo(blobName);
// with quiet
client.putObject(containerName, blobName, BYTE_SOURCE.openStream(),
metadata);
result = client.deleteObjects(request.withQuiet(true));
assertThat(result.getDeletedObjects()).isEmpty();
}
@Test
public void testBlobMetadata() throws Exception {
String blobName = "blob";
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(BYTE_SOURCE.size());
client.putObject(containerName, blobName, BYTE_SOURCE.openStream(),
metadata);
ObjectMetadata newMetadata = client.getObjectMetadata(containerName,
blobName);
assertThat(newMetadata.getContentLength())
.isEqualTo(BYTE_SOURCE.size());
}
/**
* Uploads a blob using a single upload request
*/
void executeSingleUpload(final S3BlobStore blobStore,
final String blobName,
final InputStream input,
final long blobSize) throws IOException {
// Extra safety checks
if (blobSize > MAX_FILE_SIZE.getBytes()) {
throw new IllegalArgumentException("Upload request size [" + blobSize + "] can't be larger than " + MAX_FILE_SIZE);
}
if (blobSize > blobStore.bufferSizeInBytes()) {
throw new IllegalArgumentException("Upload request size [" + blobSize + "] can't be larger than buffer size");
}
final ObjectMetadata md = new ObjectMetadata();
md.setContentLength(blobSize);
if (blobStore.serverSideEncryption()) {
md.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
}
final PutObjectRequest putRequest = new PutObjectRequest(blobStore.bucket(), blobName, input, md);
putRequest.setStorageClass(blobStore.getStorageClass());
putRequest.setCannedAcl(blobStore.getCannedACL());
try (AmazonS3Reference clientReference = blobStore.clientReference()) {
clientReference.client().putObject(putRequest);
} catch (final AmazonClientException e) {
throw new IOException("Unable to upload object [" + blobName + "] using a single upload", e);
}
}
@Test
public void testAwsV2SignatureWithOverrideParameters() throws Exception {
client = AmazonS3ClientBuilder.standard()
.withClientConfiguration(V2_SIGNER_CONFIG)
.withCredentials(new AWSStaticCredentialsProvider(awsCreds))
.withEndpointConfiguration(s3EndpointConfig).build();
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(BYTE_SOURCE.size());
client.putObject(containerName, "foo", BYTE_SOURCE.openStream(),
metadata);
String blobName = "foo";
ResponseHeaderOverrides headerOverride = new ResponseHeaderOverrides();
String expectedContentDisposition = "attachment; " + blobName;
headerOverride.setContentDisposition(expectedContentDisposition);
String expectedContentType = "text/plain";
headerOverride.setContentType(expectedContentType);
GetObjectRequest request = new GetObjectRequest(containerName,
blobName);
request.setResponseHeaders(headerOverride);
S3Object object = client.getObject(request);
assertThat(object.getObjectMetadata().getContentLength()).isEqualTo(
BYTE_SOURCE.size());
assertThat(object.getObjectMetadata().getContentDisposition())
.isEqualTo(expectedContentDisposition);
assertThat(object.getObjectMetadata().getContentType()).isEqualTo(
expectedContentType);
try (InputStream actual = object.getObjectContent();
InputStream expected = BYTE_SOURCE.openStream()) {
assertThat(actual).hasContentEqualTo(expected);
}
}
private void upload(String path, byte[] content) {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(content.length);
metadata.setContentType(getContentType(path));
PutObjectRequest request = new PutObjectRequest(bucketName, basePath + path, new ByteArrayInputStream(content),
metadata).withCannedAcl(CannedAccessControlList.PublicRead);
s3Client.putObject(request);
}
@Override
public void run() {
ObjectMetadata meta_data = new ObjectMetadata();
if (p_content_type != null)
meta_data.setContentType(p_content_type);
meta_data.setContentLength(p_size);
PutObjectRequest putObjectRequest = new PutObjectRequest(p_bucket_name, p_s3_key, p_file_stream, meta_data);
putObjectRequest.setCannedAcl(CannedAccessControlList.PublicRead);
PutObjectResult res = s3Client.putObject(putObjectRequest);
}
private void putObject(final String key, final byte[] content, Map<String, String> userMetadata) {
final ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(content.length);
metadata.setContentMD5(md5b64(content));
metadata.setUserMetadata(userMetadata);
final PutObjectRequest request = new PutObjectRequest(bucket, key, new
ByteArrayInputStream(content), metadata);
S3.putObject(request);
}
@Test
public void matchNonGlobNotReadSeekEfficient() {
S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());
S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/testdirectory/filethatexists");
long lastModifiedMillis = 1540000000000L;
ObjectMetadata s3ObjectMetadata = new ObjectMetadata();
s3ObjectMetadata.setContentLength(100);
s3ObjectMetadata.setLastModified(new Date(lastModifiedMillis));
s3ObjectMetadata.setContentEncoding("gzip");
when(s3FileSystem
.getAmazonS3Client()
.getObjectMetadata(
argThat(
new GetObjectMetadataRequestMatcher(
new GetObjectMetadataRequest(path.getBucket(), path.getKey())))))
.thenReturn(s3ObjectMetadata);
MatchResult result = s3FileSystem.matchNonGlobPath(path);
assertThat(
result,
MatchResultMatcher.create(
ImmutableList.of(
MatchResult.Metadata.builder()
.setSizeBytes(100)
.setLastModifiedMillis(lastModifiedMillis)
.setResourceId(path)
.setIsReadSeekEfficient(false)
.build())));
}
/**
* Metadata object creation
* @param length
*
* @return Metadata with AES_256 encryption if enabled
*/
private ObjectMetadata createMetadata(int length) {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(length);
if (aes256Encryption)
metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
return metadata;
}
@Test
public void testCopyObjectReplaceMetadata() throws Exception {
String fromName = "from-name";
String toName = "to-name";
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(BYTE_SOURCE.size());
if (!Quirks.NO_CACHE_CONTROL_SUPPORT.contains(blobStoreType)) {
metadata.setCacheControl("max-age=3600");
}
if (!Quirks.NO_CONTENT_DISPOSITION.contains(blobStoreType)) {
metadata.setContentDisposition("attachment; filename=old.jpg");
}
if (!Quirks.NO_CONTENT_ENCODING.contains(blobStoreType)) {
metadata.setContentEncoding("compress");
}
if (!Quirks.NO_CONTENT_LANGUAGE.contains(blobStoreType)) {
metadata.setContentLanguage("en");
}
metadata.setContentType("audio/ogg");
// TODO: expires
metadata.setUserMetadata(ImmutableMap.of(
"key1", "value1",
"key2", "value2"));
client.putObject(containerName, fromName, BYTE_SOURCE.openStream(),
metadata);
String cacheControl = "max-age=1800";
String contentDisposition = "attachment; filename=new.jpg";
String contentEncoding = "gzip";
String contentLanguage = "fr";
String contentType = "audio/mp4";
ObjectMetadata contentMetadata = new ObjectMetadata();
if (!Quirks.NO_CACHE_CONTROL_SUPPORT.contains(blobStoreType)) {
contentMetadata.setCacheControl(cacheControl);
}
if (!Quirks.NO_CONTENT_DISPOSITION.contains(blobStoreType)) {
contentMetadata.setContentDisposition(contentDisposition);
}
if (!Quirks.NO_CONTENT_ENCODING.contains(blobStoreType)) {
contentMetadata.setContentEncoding(contentEncoding);
}
if (!Quirks.NO_CONTENT_LANGUAGE.contains(blobStoreType)) {
contentMetadata.setContentLanguage(contentLanguage);
}
contentMetadata.setContentType(contentType);
// TODO: expires
Map<String, String> userMetadata = ImmutableMap.of(
"key3", "value3",
"key4", "value4");
contentMetadata.setUserMetadata(userMetadata);
client.copyObject(new CopyObjectRequest(
containerName, fromName, containerName, toName)
.withNewObjectMetadata(contentMetadata));
S3Object object = client.getObject(containerName, toName);
try (InputStream actual = object.getObjectContent();
InputStream expected = BYTE_SOURCE.openStream()) {
assertThat(actual).hasContentEqualTo(expected);
}
ObjectMetadata toContentMetadata = object.getObjectMetadata();
if (!Quirks.NO_CACHE_CONTROL_SUPPORT.contains(blobStoreType)) {
assertThat(contentMetadata.getCacheControl()).isEqualTo(
cacheControl);
}
if (!Quirks.NO_CONTENT_DISPOSITION.contains(blobStoreType)) {
assertThat(toContentMetadata.getContentDisposition()).isEqualTo(
contentDisposition);
}
if (!Quirks.NO_CONTENT_ENCODING.contains(blobStoreType)) {
assertThat(toContentMetadata.getContentEncoding()).isEqualTo(
contentEncoding);
}
if (!Quirks.NO_CONTENT_LANGUAGE.contains(blobStoreType)) {
assertThat(toContentMetadata.getContentLanguage()).isEqualTo(
contentLanguage);
}
assertThat(toContentMetadata.getContentType()).isEqualTo(
contentType);
// TODO: expires
assertThat(toContentMetadata.getUserMetadata()).isEqualTo(
userMetadata);
}
@Test
public void matchVariousInvokeThreadPool() throws IOException {
S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());
AmazonS3Exception notFoundException = new AmazonS3Exception("mock exception");
notFoundException.setStatusCode(404);
S3ResourceId pathNotExist =
S3ResourceId.fromUri("s3://testbucket/testdirectory/nonexistentfile");
when(s3FileSystem
.getAmazonS3Client()
.getObjectMetadata(
argThat(
new GetObjectMetadataRequestMatcher(
new GetObjectMetadataRequest(
pathNotExist.getBucket(), pathNotExist.getKey())))))
.thenThrow(notFoundException);
AmazonS3Exception forbiddenException = new AmazonS3Exception("mock exception");
forbiddenException.setStatusCode(403);
S3ResourceId pathForbidden =
S3ResourceId.fromUri("s3://testbucket/testdirectory/forbiddenfile");
when(s3FileSystem
.getAmazonS3Client()
.getObjectMetadata(
argThat(
new GetObjectMetadataRequestMatcher(
new GetObjectMetadataRequest(
pathForbidden.getBucket(), pathForbidden.getKey())))))
.thenThrow(forbiddenException);
S3ResourceId pathExist = S3ResourceId.fromUri("s3://testbucket/testdirectory/filethatexists");
ObjectMetadata s3ObjectMetadata = new ObjectMetadata();
s3ObjectMetadata.setContentLength(100);
s3ObjectMetadata.setLastModified(new Date(1540000000000L));
s3ObjectMetadata.setContentEncoding("not-gzip");
when(s3FileSystem
.getAmazonS3Client()
.getObjectMetadata(
argThat(
new GetObjectMetadataRequestMatcher(
new GetObjectMetadataRequest(pathExist.getBucket(), pathExist.getKey())))))
.thenReturn(s3ObjectMetadata);
S3ResourceId pathGlob = S3ResourceId.fromUri("s3://testbucket/path/part*");
S3ObjectSummary foundListObject = new S3ObjectSummary();
foundListObject.setBucketName(pathGlob.getBucket());
foundListObject.setKey("path/part-0");
foundListObject.setSize(200);
foundListObject.setLastModified(new Date(1541000000000L));
ListObjectsV2Result listObjectsResult = new ListObjectsV2Result();
listObjectsResult.setNextContinuationToken(null);
listObjectsResult.getObjectSummaries().add(foundListObject);
when(s3FileSystem.getAmazonS3Client().listObjectsV2(notNull(ListObjectsV2Request.class)))
.thenReturn(listObjectsResult);
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentEncoding("");
when(s3FileSystem
.getAmazonS3Client()
.getObjectMetadata(
argThat(
new GetObjectMetadataRequestMatcher(
new GetObjectMetadataRequest(pathGlob.getBucket(), "path/part-0")))))
.thenReturn(metadata);
assertThat(
s3FileSystem.match(
ImmutableList.of(
pathNotExist.toString(),
pathForbidden.toString(),
pathExist.toString(),
pathGlob.toString())),
contains(
MatchResultMatcher.create(MatchResult.Status.NOT_FOUND, new FileNotFoundException()),
MatchResultMatcher.create(
MatchResult.Status.ERROR, new IOException(forbiddenException)),
MatchResultMatcher.create(100, 1540000000000L, pathExist, true),
MatchResultMatcher.create(
200,
1541000000000L,
S3ResourceId.fromComponents(pathGlob.getBucket(), foundListObject.getKey()),
true)));
}
@Test
public void testAwsV2UrlSigningWithOverrideParameters() throws Exception {
client = AmazonS3ClientBuilder.standard()
.withClientConfiguration(V2_SIGNER_CONFIG)
.withCredentials(new AWSStaticCredentialsProvider(awsCreds))
.withEndpointConfiguration(s3EndpointConfig).build();
String blobName = "foo";
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(BYTE_SOURCE.size());
client.putObject(containerName, blobName, BYTE_SOURCE.openStream(),
metadata);
GeneratePresignedUrlRequest generatePresignedUrlRequest =
new GeneratePresignedUrlRequest(containerName, blobName);
generatePresignedUrlRequest.setMethod(HttpMethod.GET);
ResponseHeaderOverrides headerOverride = new ResponseHeaderOverrides();
headerOverride.setContentDisposition("attachment; " + blobName);
headerOverride.setContentType("text/plain");
generatePresignedUrlRequest.setResponseHeaders(headerOverride);
Date expiration = new Date(System.currentTimeMillis() +
TimeUnit.HOURS.toMillis(1));
generatePresignedUrlRequest.setExpiration(expiration);
URL url = client.generatePresignedUrl(generatePresignedUrlRequest);
URLConnection connection = url.openConnection();
try (InputStream actual = connection.getInputStream();
InputStream expected = BYTE_SOURCE.openStream()) {
String value = connection.getHeaderField("Content-Disposition");
assertThat(value).isEqualTo(headerOverride.getContentDisposition());
value = connection.getHeaderField("Content-Type");
assertThat(value).isEqualTo(headerOverride.getContentType());
assertThat(actual).hasContentEqualTo(expected);
}
}
private void createEmptyObject(final String bucketName, final String objectName) {
final InputStream im = new InputStream() {
@Override
public int read() throws IOException {
return -1;
}
};
final ObjectMetadata om = new ObjectMetadata();
om.setContentLength(0L);
this.s3Client.putObject(bucketName, objectName, im, om);
}
@Test
public void testMultipartCopy() throws Exception {
// B2 requires two parts to issue an MPU
assumeTrue(!blobStoreType.equals("b2"));
String sourceBlobName = "testMultipartCopy-source";
String targetBlobName = "testMultipartCopy-target";
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(BYTE_SOURCE.size());
client.putObject(containerName, sourceBlobName,
BYTE_SOURCE.openStream(), metadata);
InitiateMultipartUploadRequest initiateRequest =
new InitiateMultipartUploadRequest(containerName,
targetBlobName);
InitiateMultipartUploadResult initResult =
client.initiateMultipartUpload(initiateRequest);
String uploadId = initResult.getUploadId();
CopyPartRequest copyRequest = new CopyPartRequest()
.withDestinationBucketName(containerName)
.withDestinationKey(targetBlobName)
.withSourceBucketName(containerName)
.withSourceKey(sourceBlobName)
.withUploadId(uploadId)
.withFirstByte(0L)
.withLastByte(BYTE_SOURCE.size() - 1)
.withPartNumber(1);
CopyPartResult copyPartResult = client.copyPart(copyRequest);
CompleteMultipartUploadRequest completeRequest =
new CompleteMultipartUploadRequest(
containerName, targetBlobName, uploadId,
ImmutableList.of(copyPartResult.getPartETag()));
client.completeMultipartUpload(completeRequest);
S3Object object = client.getObject(containerName, targetBlobName);
assertThat(object.getObjectMetadata().getContentLength()).isEqualTo(
BYTE_SOURCE.size());
try (InputStream actual = object.getObjectContent();
InputStream expected = BYTE_SOURCE.openStream()) {
assertThat(actual).hasContentEqualTo(expected);
}
}