下面列出了com.amazonaws.services.s3.model.S3Object#getObjectContent ( ) 实例代码，或者点击链接到github查看源代码，也可以在右侧发表评论。
/**
* @return the new config file (datapipelines.properties) from s3 as a String
*/
private static String getNewConfig () {
StringBuilder config = new StringBuilder();
// get config object from S3
AmazonS3Client s3Client = new AmazonS3Client();
S3Object configObj = s3Client.getObject(DATAPIPELINES_CONFIG_S3_BUCKET,
DATAPIPELINES_CONFIG_S3_KEY);
// write object to String
BufferedReader reader = new BufferedReader(new InputStreamReader(configObj.getObjectContent()));
String line;
try {
while ((line = reader.readLine()) != null) {
config.append(String.format("%s\n", line));
}
} catch (IOException e) {
LOG.error("Failed to read config ({}) S3Object to String, exception: {}.",
String.format("%s/%s", DATAPIPELINES_CONFIG_S3_BUCKET, DATAPIPELINES_CONFIG_S3_KEY),
ExceptionUtils.getFullStackTrace(e));
}
return config.toString();
}
@Override
public void run() {
SignedDomain signedDomain = null;
try {
S3Object object = s3.getObject(s3BucketName, domainName);
try (S3ObjectInputStream s3is = object.getObjectContent()) {
signedDomain = jsonMapper.readValue(s3is, SignedDomain.class);
}
} catch (Exception ex) {
LOGGER.error("AWSS3ChangeLogThread: ObjectS3Thread- getSignedDomain - unable to get domain {} error: {}",
domainName, ex.getMessage());
}
if (signedDomain != null) {
signedDomainMap.put(domainName, signedDomain);
}
}
@Test
public void testAtomicMpuAbort() throws Exception {
String key = "testAtomicMpuAbort";
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(BYTE_SOURCE.size());
client.putObject(containerName, key, BYTE_SOURCE.openStream(),
metadata);
InitiateMultipartUploadRequest initRequest =
new InitiateMultipartUploadRequest(containerName, key);
InitiateMultipartUploadResult initResponse =
client.initiateMultipartUpload(initRequest);
String uploadId = initResponse.getUploadId();
client.abortMultipartUpload(new AbortMultipartUploadRequest(
containerName, key, uploadId));
S3Object object = client.getObject(containerName, key);
assertThat(object.getObjectMetadata().getContentLength()).isEqualTo(
BYTE_SOURCE.size());
try (InputStream actual = object.getObjectContent();
InputStream expected = BYTE_SOURCE.openStream()) {
assertThat(actual).hasContentEqualTo(expected);
}
}
@Override
public boolean load(BuildCacheKey key, BuildCacheEntryReader reader) {
final String bucketPath = getBucketPath(key);
if (s3.doesObjectExist(bucketName, bucketPath)) {
logger.info("Found cache item '{}' in S3 bucket", bucketPath);
S3Object object = s3.getObject(bucketName, bucketPath);
try (InputStream is = object.getObjectContent()) {
reader.readFrom(is);
return true;
} catch (IOException e) {
throw new BuildCacheException("Error while reading cache object from S3 bucket", e);
}
} else {
logger.info("Did not find cache item '{}' in S3 bucket", bucketPath);
return false;
}
}
@Test
public void testConditionalGet() throws Exception {
assumeTrue(!blobStoreType.equals("b2"));
String blobName = "blob-name";
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(BYTE_SOURCE.size());
PutObjectResult result = client.putObject(containerName, blobName,
BYTE_SOURCE.openStream(), metadata);
S3Object object = client.getObject(
new GetObjectRequest(containerName, blobName)
.withMatchingETagConstraint(result.getETag()));
try (InputStream is = object.getObjectContent()) {
assertThat(is).isNotNull();
ByteStreams.copy(is, ByteStreams.nullOutputStream());
}
object = client.getObject(
new GetObjectRequest(containerName, blobName)
.withNonmatchingETagConstraint(result.getETag()));
assertThat(object).isNull();
}
/**
* S3 block read would be achieved through the AmazonS3 client. Following
* are the steps to achieve: (1) Create the objectRequest from bucketName
* and filePath. (2) Set the range to the above created objectRequest. (3)
* Get the object portion through AmazonS3 client API. (4) Get the object
* content from the above object portion.
*
* @param bytesFromCurrentOffset
* bytes read till now from current offset
* @param bytesToFetch
* the number of bytes to be fetched
* @return the number of bytes read, -1 if 0 bytes read
* @throws IOException
*/
@Override
protected int readData(final long bytesFromCurrentOffset, final int bytesToFetch) throws IOException
{
GetObjectRequest rangeObjectRequest = new GetObjectRequest(s3Params.bucketName, s3Params.filePath);
rangeObjectRequest.setRange(offset + bytesFromCurrentOffset, offset + bytesFromCurrentOffset + bytesToFetch - 1);
S3Object objectPortion = s3Params.s3Client.getObject(rangeObjectRequest);
S3ObjectInputStream wrappedStream = objectPortion.getObjectContent();
buffer = ByteStreams.toByteArray(wrappedStream);
wrappedStream.close();
int bufferLength = buffer.length;
if (bufferLength <= 0) {
return -1;
}
return bufferLength;
}
@Override
public InputStream read( UUID appId, Entity entity, long offset, long length ) throws Exception {
S3Object object = getS3Client().getObject( bucketName, AssetUtils.buildAssetKey( appId, entity ) );
byte data[] = null;
if ( offset == 0 && length == FIVE_MB ) {
return object.getObjectContent();
}
else {
object.getObjectContent().read(data, Ints.checkedCast(offset), Ints.checkedCast(length));
}
return new ByteArrayInputStream(data);
}
@Override
public InputStream readBlob(String blobName) throws IOException {
try (AmazonS3Reference clientReference = blobStore.clientReference()) {
final S3Object s3Object = clientReference.client().getObject(blobStore.bucket(), buildKey(blobName));
return s3Object.getObjectContent();
} catch (final AmazonClientException e) {
if (e instanceof AmazonS3Exception) {
if (404 == ((AmazonS3Exception) e).getStatusCode()) {
throw new NoSuchFileException("Blob object [" + blobName + "] not found: " + e.getMessage());
}
}
throw e;
}
}
/**
* Constructor for the S3 version of the Bloom filter.
* @param name String. The name of the object.
* @param object S3Object. The object that contains the file.
* @throws Exception on S3 errors.
*/
public Bloom(String name, S3Object object, long size) throws Exception {
InputStream objectData = object.getObjectContent();
BufferedReader br=new BufferedReader(new InputStreamReader(objectData));
makeFilter(br,size);
symbols.put(name, bloomFilter);
}
public String getString(String bucket, String key) {
GetObjectRequest request = new GetObjectRequest(bucket, key);
S3Object response = client.getObject(request);
try (S3ObjectInputStream is = response.getObjectContent()) {
return CharStreams.toString(new InputStreamReader(is, Charsets.UTF_8));
} catch (IOException ex) {
throw new RuntimeException(ex);
}
}
@Override
public void open(Configuration parameters) throws Exception {
super.open(parameters);
StreamingRuntimeContext context = (StreamingRuntimeContext) getRuntimeContext();
int parallelism = context.getNumberOfParallelSubtasks();
if (parallelism != 1) {
throw new IllegalStateException("SeedUrlSource only supports a parallelism of 1");
}
if (_terminator == null) {
throw new IllegalStateException("Crawl terminator must be set for the seed URL source");
}
LOGGER.info("Opening seed URL source");
// Open the terminator, so that it knows when we really started running.
_terminator.open();
_urlIndex = 0;
if (useS3File()) {
AmazonS3 s3Client = S3Utils.makeS3Client();
S3Object object = s3Client
.getObject(new GetObjectRequest(_seedUrlsS3Bucket, _seedUrlsS3Path));
_s3FileStream = object.getObjectContent();
}
}
@Test
public void testCopyObject() throws Exception {
String newName = UUID.randomUUID().toString();
AmazonS3ExecutorConfig config = getConfig();
config.taskConfig.taskType = TaskType.COPY_OBJECT;
config.taskConfig.copyTargetLocation = newName;
AmazonS3Executor executor = new AmazonS3Executor(config);
TargetRunner runner = new TargetRunner.Builder(AmazonS3DExecutor.class, executor)
.build();
runner.runInit();
try {
s3client.putObject(new PutObjectRequest(BUCKET_NAME, objectName, IOUtils.toInputStream("content"), new ObjectMetadata()));
runner.runWrite(ImmutableList.of(getTestRecord()));
S3Object object = s3client.getObject(BUCKET_NAME, newName);
S3ObjectInputStream objectContent = object.getObjectContent();
List<String> stringList = IOUtils.readLines(objectContent);
Assert.assertEquals(1, stringList.size());
Assert.assertEquals("content", stringList.get(0));
Assert.assertTrue(s3client.doesObjectExist(BUCKET_NAME, objectName));
Assert.assertEquals(1, runner.getEventRecords().size());
assertEvent(runner.getEventRecords().get(0), newName);
} finally {
runner.runDestroy();
}
}
@Test
public void testMultipartCopy() throws Exception {
// B2 requires two parts to issue an MPU
assumeTrue(!blobStoreType.equals("b2"));
String sourceBlobName = "testMultipartCopy-source";
String targetBlobName = "testMultipartCopy-target";
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(BYTE_SOURCE.size());
client.putObject(containerName, sourceBlobName,
BYTE_SOURCE.openStream(), metadata);
InitiateMultipartUploadRequest initiateRequest =
new InitiateMultipartUploadRequest(containerName,
targetBlobName);
InitiateMultipartUploadResult initResult =
client.initiateMultipartUpload(initiateRequest);
String uploadId = initResult.getUploadId();
CopyPartRequest copyRequest = new CopyPartRequest()
.withDestinationBucketName(containerName)
.withDestinationKey(targetBlobName)
.withSourceBucketName(containerName)
.withSourceKey(sourceBlobName)
.withUploadId(uploadId)
.withFirstByte(0L)
.withLastByte(BYTE_SOURCE.size() - 1)
.withPartNumber(1);
CopyPartResult copyPartResult = client.copyPart(copyRequest);
CompleteMultipartUploadRequest completeRequest =
new CompleteMultipartUploadRequest(
containerName, targetBlobName, uploadId,
ImmutableList.of(copyPartResult.getPartETag()));
client.completeMultipartUpload(completeRequest);
S3Object object = client.getObject(containerName, targetBlobName);
assertThat(object.getObjectMetadata().getContentLength()).isEqualTo(
BYTE_SOURCE.size());
try (InputStream actual = object.getObjectContent();
InputStream expected = BYTE_SOURCE.openStream()) {
assertThat(actual).hasContentEqualTo(expected);
}
}
public InputStream get(String bucket, String key) {
GetObjectRequest request = new GetObjectRequest(bucket, key);
S3Object response = client.getObject(request);
return response.getObjectContent();
}
public static void main(String[] args) throws IOException {
//set-up the client
AmazonS3 s3client = AmazonS3ClientBuilder
.standard()
.withCredentials(new AWSStaticCredentialsProvider(credentials))
.withRegion(Regions.US_EAST_2)
.build();
AWSS3Service awsService = new AWSS3Service(s3client);
bucketName = "baeldung-bucket";
//creating a bucket
if(awsService.doesBucketExist(bucketName)) {
System.out.println("Bucket name is not available."
+ " Try again with a different Bucket name.");
return;
}
awsService.createBucket(bucketName);
//list all the buckets
for(Bucket s : awsService.listBuckets() ) {
System.out.println(s.getName());
}
//deleting bucket
awsService.deleteBucket("baeldung-bucket-test2");
//uploading object
awsService.putObject(
bucketName,
"Document/hello.txt",
new File("/Users/user/Document/hello.txt")
);
//listing objects
ObjectListing objectListing = awsService.listObjects(bucketName);
for(S3ObjectSummary os : objectListing.getObjectSummaries()) {
System.out.println(os.getKey());
}
//downloading an object
S3Object s3object = awsService.getObject(bucketName, "Document/hello.txt");
S3ObjectInputStream inputStream = s3object.getObjectContent();
FileUtils.copyInputStreamToFile(inputStream, new File("/Users/user/Desktop/hello.txt"));
//copying an object
awsService.copyObject(
"baeldung-bucket",
"picture/pic.png",
"baeldung-bucket2",
"Document/picture.png"
);
//deleting an object
awsService.deleteObject(bucketName, "Document/hello.txt");
//deleting multiple objects
String objkeyArr[] = {
"Document/hello2.txt",
"Document/picture.png"
};
DeleteObjectsRequest delObjReq = new DeleteObjectsRequest("baeldung-bucket")
.withKeys(objkeyArr);
awsService.deleteObjects(delObjReq);
}
/**
* Fetchs a remote object from S3; datasource, bucket, key, aes256key supported
*
* @param props
* @param _Session
* @throws cfmRunTimeException
*/
private void remoteFetchS3( cfStructData props, cfSession _Session ) throws cfmRunTimeException {
if ( !props.containsKey( "datasource" ) ||
!props.containsKey( "bucket" ) ||
!props.containsKey( "key" ) )
throw newRunTimeException( "'remote'.type=s3; minimum keys are datasource, bucket and key" );
String datasource = props.getData( "datasource" ).getString();
String bucket = props.getData( "bucket" ).getString();
String key = props.getData( "key" ).getString();
// Get the Amazon datasource
AmazonKey amazonKey = AmazonKeyFactory.getDS( datasource );
if ( amazonKey == null )
throw newRunTimeException( "Amazon Datasource [" + datasource + "] has not been registered; use AmazonRegisterDataSource()" );
amazonKey.setDataSource( datasource );
AmazonS3 s3Client = new AmazonBase().getAmazonS3( amazonKey );
GetObjectRequest gor = new GetObjectRequest( bucket, key );
if ( props.containsKey( "aes256key" ) ) {
String aes256key = props.getData( "aes256key" ).getString();
if ( !aes256key.isEmpty() )
gor.setSSECustomerKey( new SSECustomerKey( aes256key ) );
}
// Get the object
try {
S3Object s3object = s3Client.getObject( gor );
_Session.setContentType( s3object.getObjectMetadata().getContentType() );
InputStream in = s3object.getObjectContent();
byte[] buffer = new byte[65536];
int readCount = 0;
while ( ( readCount = in.read( buffer ) ) != -1 ) {
_Session.write( buffer, 0, readCount );
_Session.pageFlush();
}
} catch ( Exception e ) {
if ( e.getMessage().indexOf("404") != -1 ){
_Session.setStatus( 404 );
return;
}else{
cfEngine.log( e.getMessage() );
throw newRunTimeException( e.getMessage() + "; key=" + key + "; bucket=" + bucket );
}
}
}
@Test
public void testCopyObjectPreserveMetadata() throws Exception {
String fromName = "from-name";
String toName = "to-name";
String cacheControl = "max-age=3600";
String contentDisposition = "attachment; filename=old.jpg";
String contentEncoding = "gzip";
String contentLanguage = "en";
String contentType = "audio/ogg";
Map<String, String> userMetadata = ImmutableMap.of(
"key1", "value1",
"key2", "value2");
ObjectMetadata metadata = new ObjectMetadata();
if (!Quirks.NO_CACHE_CONTROL_SUPPORT.contains(blobStoreType)) {
metadata.setCacheControl(cacheControl);
}
metadata.setContentLength(BYTE_SOURCE.size());
if (!Quirks.NO_CONTENT_DISPOSITION.contains(blobStoreType)) {
metadata.setContentDisposition(contentDisposition);
}
if (!Quirks.NO_CONTENT_ENCODING.contains(blobStoreType)) {
metadata.setContentEncoding(contentEncoding);
}
if (!Quirks.NO_CONTENT_LANGUAGE.contains(blobStoreType)) {
metadata.setContentLanguage(contentLanguage);
}
metadata.setContentType(contentType);
// TODO: expires
metadata.setUserMetadata(userMetadata);
client.putObject(containerName, fromName, BYTE_SOURCE.openStream(),
metadata);
client.copyObject(containerName, fromName, containerName, toName);
S3Object object = client.getObject(containerName, toName);
try (InputStream actual = object.getObjectContent();
InputStream expected = BYTE_SOURCE.openStream()) {
assertThat(actual).hasContentEqualTo(expected);
}
ObjectMetadata contentMetadata = object.getObjectMetadata();
assertThat(contentMetadata.getContentLength()).isEqualTo(
BYTE_SOURCE.size());
if (!Quirks.NO_CACHE_CONTROL_SUPPORT.contains(blobStoreType)) {
assertThat(contentMetadata.getCacheControl()).isEqualTo(
cacheControl);
}
if (!Quirks.NO_CONTENT_DISPOSITION.contains(blobStoreType)) {
assertThat(contentMetadata.getContentDisposition()).isEqualTo(
contentDisposition);
}
if (!Quirks.NO_CONTENT_ENCODING.contains(blobStoreType)) {
assertThat(contentMetadata.getContentEncoding()).isEqualTo(
contentEncoding);
}
if (!Quirks.NO_CONTENT_LANGUAGE.contains(blobStoreType)) {
assertThat(contentMetadata.getContentLanguage()).isEqualTo(
contentLanguage);
}
assertThat(contentMetadata.getContentType()).isEqualTo(
contentType);
// TODO: expires
assertThat(contentMetadata.getUserMetadata()).isEqualTo(
userMetadata);
}
public String handleRequest(S3Event s3event, Context context) {
_logger = context.getLogger();
_logger.log("Received S3 Event: " + s3event.toJson());
try {
S3EventNotificationRecord record = s3event.getRecords().get(0);
String bucket = record.getS3().getBucket().getName();
String extractBucket = "extracts." + bucket;
// Object key may have spaces or unicode non-ASCII characters.
String key = URLDecoder.decode(record.getS3().getObject().getKey().replace('+', ' '), "UTF-8");
// Short-circuit ignore .extract files because they have already been extracted, this prevents an endless loop
if (key.toLowerCase().endsWith(".extract")) {
_logger.log("Ignoring extract file " + key);
return "Ignored";
}
AmazonS3 s3Client = new AmazonS3Client();
S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucket, key));
try (InputStream objectData = s3Object.getObjectContent()) {
String extractJson = doTikaStuff(bucket, key, objectData);
byte[] extractBytes = extractJson.getBytes(Charset.forName("UTF-8"));
int extractLength = extractBytes.length;
ObjectMetadata metaData = new ObjectMetadata();
metaData.setContentLength(extractLength);
_logger.log("Saving extract file to S3");
InputStream inputStream = new ByteArrayInputStream(extractBytes);
s3Client.putObject(extractBucket, key + ".extract", inputStream, metaData);
}
} catch (IOException | TransformerConfigurationException | SAXException e) {
_logger.log("Exception: " + e.getLocalizedMessage());
throw new RuntimeException(e);
}
return "Success";
}
@Test
public void testCopyObjectReplaceMetadata() throws Exception {
String fromName = "from-name";
String toName = "to-name";
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(BYTE_SOURCE.size());
if (!Quirks.NO_CACHE_CONTROL_SUPPORT.contains(blobStoreType)) {
metadata.setCacheControl("max-age=3600");
}
if (!Quirks.NO_CONTENT_DISPOSITION.contains(blobStoreType)) {
metadata.setContentDisposition("attachment; filename=old.jpg");
}
if (!Quirks.NO_CONTENT_ENCODING.contains(blobStoreType)) {
metadata.setContentEncoding("compress");
}
if (!Quirks.NO_CONTENT_LANGUAGE.contains(blobStoreType)) {
metadata.setContentLanguage("en");
}
metadata.setContentType("audio/ogg");
// TODO: expires
metadata.setUserMetadata(ImmutableMap.of(
"key1", "value1",
"key2", "value2"));
client.putObject(containerName, fromName, BYTE_SOURCE.openStream(),
metadata);
String cacheControl = "max-age=1800";
String contentDisposition = "attachment; filename=new.jpg";
String contentEncoding = "gzip";
String contentLanguage = "fr";
String contentType = "audio/mp4";
ObjectMetadata contentMetadata = new ObjectMetadata();
if (!Quirks.NO_CACHE_CONTROL_SUPPORT.contains(blobStoreType)) {
contentMetadata.setCacheControl(cacheControl);
}
if (!Quirks.NO_CONTENT_DISPOSITION.contains(blobStoreType)) {
contentMetadata.setContentDisposition(contentDisposition);
}
if (!Quirks.NO_CONTENT_ENCODING.contains(blobStoreType)) {
contentMetadata.setContentEncoding(contentEncoding);
}
if (!Quirks.NO_CONTENT_LANGUAGE.contains(blobStoreType)) {
contentMetadata.setContentLanguage(contentLanguage);
}
contentMetadata.setContentType(contentType);
// TODO: expires
Map<String, String> userMetadata = ImmutableMap.of(
"key3", "value3",
"key4", "value4");
contentMetadata.setUserMetadata(userMetadata);
client.copyObject(new CopyObjectRequest(
containerName, fromName, containerName, toName)
.withNewObjectMetadata(contentMetadata));
S3Object object = client.getObject(containerName, toName);
try (InputStream actual = object.getObjectContent();
InputStream expected = BYTE_SOURCE.openStream()) {
assertThat(actual).hasContentEqualTo(expected);
}
ObjectMetadata toContentMetadata = object.getObjectMetadata();
if (!Quirks.NO_CACHE_CONTROL_SUPPORT.contains(blobStoreType)) {
assertThat(contentMetadata.getCacheControl()).isEqualTo(
cacheControl);
}
if (!Quirks.NO_CONTENT_DISPOSITION.contains(blobStoreType)) {
assertThat(toContentMetadata.getContentDisposition()).isEqualTo(
contentDisposition);
}
if (!Quirks.NO_CONTENT_ENCODING.contains(blobStoreType)) {
assertThat(toContentMetadata.getContentEncoding()).isEqualTo(
contentEncoding);
}
if (!Quirks.NO_CONTENT_LANGUAGE.contains(blobStoreType)) {
assertThat(toContentMetadata.getContentLanguage()).isEqualTo(
contentLanguage);
}
assertThat(toContentMetadata.getContentType()).isEqualTo(
contentType);
// TODO: expires
assertThat(toContentMetadata.getUserMetadata()).isEqualTo(
userMetadata);
}
public static InputStream makeS3FileStream(String bucketName, String key) {
AmazonS3 s3Client = makeS3Client();
S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, key));
return object.getObjectContent();
}