下面列出了com.amazonaws.services.s3.model.PutObjectRequest#setMetadata ( ) 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
private boolean uploadFile(File src, S3Details dstDetails, boolean archive) {
PutObjectRequest request = new PutObjectRequest(dstDetails.getBucket(),
FileUtil.appendPath(dstDetails.getKey(), src.getName()), src);
ObjectMetadata meta = new ObjectMetadata();
if (archive) {
meta
.setHeader(Headers.STORAGE_CLASS,
StorageClass.fromValue(StringUtils.upperCase(archiveStorageClass)));
}
if (encrypt) {
meta.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
}
request.setMetadata(meta);
try {
amazonS3.putObject(request);
return true;
} catch (Exception e) {
log.error("Error while uploading file: {}", src, e);
}
return false;
}
@Override
public void transferProduct(Product product) throws DataTransferException, IOException {
for (Reference ref : product.getProductReferences()) {
String origRef = stripProtocol(ref.getOrigReference(), false);
String dataStoreRef = stripProtocol(ref.getDataStoreReference(), true);
try {
PutObjectRequest request = new PutObjectRequest(
bucketName, dataStoreRef, new File(origRef));
if (encrypt) {
ObjectMetadata requestMetadata = new ObjectMetadata();
requestMetadata.setSSEAlgorithm(AES_256_SERVER_SIDE_ENCRYPTION);
request.setMetadata(requestMetadata);
}
s3Client.putObject(request);
} catch (AmazonClientException e) {
throw new DataTransferException(String.format(
"Failed to upload product reference %s to S3 at %s", origRef,
dataStoreRef), e);
}
}
}
@Override
public void save(Note note, AuthenticationInfo subject) throws IOException {
String json = note.toJson();
String key = user + "/" + "notebook" + "/" + note.getId() + "/" + "note.json";
File file = File.createTempFile("note", "json");
try {
Writer writer = new OutputStreamWriter(new FileOutputStream(file));
writer.write(json);
writer.close();
PutObjectRequest putRequest = new PutObjectRequest(bucketName, key, file);
if (useServerSideEncryption) {
// Request server-side encryption.
ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
putRequest.setMetadata(objectMetadata);
}
if (objectCannedAcl != null) {
putRequest.withCannedAcl(objectCannedAcl);
}
s3client.putObject(putRequest);
}
catch (AmazonClientException ace) {
throw new IOException("Unable to store note in S3: " + ace, ace);
}
finally {
FileUtils.deleteQuietly(file);
}
}
@Override
public void save(Note note, AuthenticationInfo subject) throws IOException {
String json = note.toJson();
String key = rootFolder + "/" + buildNoteFileName(note);
File file = File.createTempFile("note", "zpln");
try {
Writer writer = new OutputStreamWriter(new FileOutputStream(file));
writer.write(json);
writer.close();
PutObjectRequest putRequest = new PutObjectRequest(bucketName, key, file);
if (useServerSideEncryption) {
// Request server-side encryption.
ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
putRequest.setMetadata(objectMetadata);
}
if (objectCannedAcl != null) {
putRequest.withCannedAcl(objectCannedAcl);
}
s3client.putObject(putRequest);
}
catch (AmazonClientException ace) {
throw new IOException("Fail to store note: " + note.getPath() + " in S3", ace);
}
finally {
FileUtils.deleteQuietly(file);
}
}
@Override
public synchronized void close() throws IOException {
if (closed) {
return;
}
backupStream.close();
if (LOG.isDebugEnabled()) {
LOG.debug("OutputStream for key '" + key + "' closed. Now beginning upload");
LOG.debug("Minimum upload part size: " + partSize + " threshold " + partSizeThreshold);
}
try {
final ObjectMetadata om = new ObjectMetadata();
if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
om.setServerSideEncryption(serverSideEncryptionAlgorithm);
}
PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, backupFile);
putObjectRequest.setCannedAcl(cannedACL);
putObjectRequest.setMetadata(om);
Upload upload = transfers.upload(putObjectRequest);
ProgressableProgressListener listener =
new ProgressableProgressListener(upload, progress, statistics);
upload.addProgressListener(listener);
upload.waitForUploadResult();
long delta = upload.getProgress().getBytesTransferred() - listener.getLastBytesTransferred();
if (statistics != null && delta != 0) {
if (LOG.isDebugEnabled()) {
LOG.debug("S3A write delta changed after finished: " + delta + " bytes");
}
statistics.incrementBytesWritten(delta);
}
// This will delete unnecessary fake parent directories
fs.finishedWrite(key);
} catch (InterruptedException e) {
throw new IOException(e);
} finally {
if (!backupFile.delete()) {
LOG.warn("Could not delete temporary s3a file: {}", backupFile);
}
super.close();
closed = true;
}
if (LOG.isDebugEnabled()) {
LOG.debug("OutputStream for key '" + key + "' upload complete");
}
}
/**
* The src file is on the local disk. Add it to FS at
* the given dst name.
*
* This version doesn't need to create a temporary file to calculate the md5.
* Sadly this doesn't seem to be used by the shell cp :(
*
* delSrc indicates if the source should be removed
* @param delSrc whether to delete the src
* @param overwrite whether to overwrite an existing file
* @param src path
* @param dst path
*/
@Override
public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src,
Path dst) throws IOException {
String key = pathToKey(dst);
if (!overwrite && exists(dst)) {
throw new IOException(dst + " already exists");
}
if (LOG.isDebugEnabled()) {
LOG.debug("Copying local file from " + src + " to " + dst);
}
// Since we have a local file, we don't need to stream into a temporary file
LocalFileSystem local = getLocal(getConf());
File srcfile = local.pathToFile(src);
final ObjectMetadata om = new ObjectMetadata();
if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
om.setServerSideEncryption(serverSideEncryptionAlgorithm);
}
PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, srcfile);
putObjectRequest.setCannedAcl(cannedACL);
putObjectRequest.setMetadata(om);
ProgressListener progressListener = new ProgressListener() {
public void progressChanged(ProgressEvent progressEvent) {
switch (progressEvent.getEventCode()) {
case ProgressEvent.PART_COMPLETED_EVENT_CODE:
statistics.incrementWriteOps(1);
break;
default:
break;
}
}
};
Upload up = transfers.upload(putObjectRequest);
up.addProgressListener(progressListener);
try {
up.waitForUploadResult();
statistics.incrementWriteOps(1);
} catch (InterruptedException e) {
throw new IOException("Got interrupted, cancelling");
}
// This will delete unnecessary fake parent directories
finishedWrite(key);
if (delSrc) {
local.delete(src, false);
}
}
@Override
public synchronized void close() throws IOException {
if (closed) {
return;
}
backupStream.close();
if (LOG.isDebugEnabled()) {
LOG.debug("OutputStream for key '" + key + "' closed. Now beginning upload");
LOG.debug("Minimum upload part size: " + partSize + " threshold " + partSizeThreshold);
}
try {
final ObjectMetadata om = new ObjectMetadata();
if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
om.setServerSideEncryption(serverSideEncryptionAlgorithm);
}
PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, backupFile);
putObjectRequest.setCannedAcl(cannedACL);
putObjectRequest.setMetadata(om);
Upload upload = transfers.upload(putObjectRequest);
ProgressableProgressListener listener =
new ProgressableProgressListener(upload, progress, statistics);
upload.addProgressListener(listener);
upload.waitForUploadResult();
long delta = upload.getProgress().getBytesTransferred() - listener.getLastBytesTransferred();
if (statistics != null && delta != 0) {
if (LOG.isDebugEnabled()) {
LOG.debug("S3A write delta changed after finished: " + delta + " bytes");
}
statistics.incrementBytesWritten(delta);
}
// This will delete unnecessary fake parent directories
fs.finishedWrite(key);
} catch (InterruptedException e) {
throw new IOException(e);
} finally {
if (!backupFile.delete()) {
LOG.warn("Could not delete temporary s3a file: {}", backupFile);
}
super.close();
closed = true;
}
if (LOG.isDebugEnabled()) {
LOG.debug("OutputStream for key '" + key + "' upload complete");
}
}
/**
* The src file is on the local disk. Add it to FS at
* the given dst name.
*
* This version doesn't need to create a temporary file to calculate the md5.
* Sadly this doesn't seem to be used by the shell cp :(
*
* delSrc indicates if the source should be removed
* @param delSrc whether to delete the src
* @param overwrite whether to overwrite an existing file
* @param src path
* @param dst path
*/
@Override
public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src,
Path dst) throws IOException {
String key = pathToKey(dst);
if (!overwrite && exists(dst)) {
throw new IOException(dst + " already exists");
}
if (LOG.isDebugEnabled()) {
LOG.debug("Copying local file from " + src + " to " + dst);
}
// Since we have a local file, we don't need to stream into a temporary file
LocalFileSystem local = getLocal(getConf());
File srcfile = local.pathToFile(src);
final ObjectMetadata om = new ObjectMetadata();
if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
om.setServerSideEncryption(serverSideEncryptionAlgorithm);
}
PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, srcfile);
putObjectRequest.setCannedAcl(cannedACL);
putObjectRequest.setMetadata(om);
ProgressListener progressListener = new ProgressListener() {
public void progressChanged(ProgressEvent progressEvent) {
switch (progressEvent.getEventCode()) {
case ProgressEvent.PART_COMPLETED_EVENT_CODE:
statistics.incrementWriteOps(1);
break;
default:
break;
}
}
};
Upload up = transfers.upload(putObjectRequest);
up.addProgressListener(progressListener);
try {
up.waitForUploadResult();
statistics.incrementWriteOps(1);
} catch (InterruptedException e) {
throw new IOException("Got interrupted, cancelling");
}
// This will delete unnecessary fake parent directories
finishedWrite(key);
if (delSrc) {
local.delete(src, false);
}
}
/**
* Upload the current block as a single PUT request; if the buffer is empty a
* 0-byte PUT will be invoked, as it is needed to create an entry at the far
* end.
*
* @throws IOException any problem
*/
private void putObject() throws IOException {
LOG.debug("Executing regular upload for {}", writeOperationHelper);
final COSDataBlocks.DataBlock block = getActiveBlock();
int size = block.dataSize();
final COSDataBlocks.BlockUploadData uploadData = block.startUpload();
final PutObjectRequest putObjectRequest = uploadData.hasFile()
? writeOperationHelper.newPutRequest(uploadData.getFile())
: writeOperationHelper.newPutRequest(uploadData.getUploadStream(), size);
final ObjectMetadata om = new ObjectMetadata();
om.setUserMetadata(mMetadata);
if (contentType != null && !contentType.isEmpty()) {
om.setContentType(contentType);
} else {
om.setContentType("application/octet-stream");
}
// if atomic write is enabled use the etag to ensure put request is atomic
if (mAtomicWriteEnabled) {
if (mEtag != null) {
LOG.debug("Atomic write - setting If-Match header");
om.setHeader("If-Match", mEtag);
} else {
LOG.debug("Atomic write - setting If-None-Match header");
om.setHeader("If-None-Match", "*");
}
}
putObjectRequest.setMetadata(om);
ListenableFuture<PutObjectResult> putObjectResult =
executorService.submit(new Callable<PutObjectResult>() {
@Override
public PutObjectResult call() throws Exception {
PutObjectResult result;
try {
// the putObject call automatically closes the input
// stream afterwards.
result = writeOperationHelper.putObject(putObjectRequest);
} finally {
closeAll(LOG, uploadData, block);
}
return result;
}
});
clearActiveBlock();
// wait for completion
try {
putObjectResult.get();
} catch (InterruptedException ie) {
LOG.warn("Interrupted object upload", ie);
Thread.currentThread().interrupt();
} catch (ExecutionException ee) {
throw extractException("regular upload", key, ee);
}
}
@Override
public MultipleFileUpload uploadFileList(String bucketName, String virtualDirectoryKeyPrefix, File directory, List<File> files,
ObjectMetadataProvider metadataProvider, TransferManager transferManager)
{
LOGGER.debug(
"uploadFileList(): bucketName = " + bucketName + ", virtualDirectoryKeyPrefix = " + virtualDirectoryKeyPrefix + ", directory = " + directory +
", files = " + files);
String directoryPath = directory.getAbsolutePath();
long totalFileLength = 0;
List<Upload> subTransfers = new ArrayList<>();
for (File file : files)
{
// Get path to file relative to the specified directory
String relativeFilePath = file.getAbsolutePath().substring(directoryPath.length());
// Replace any backslashes (i.e. Windows separator) with a forward slash.
relativeFilePath = relativeFilePath.replace("\\", "/");
// Remove any leading slashes
relativeFilePath = relativeFilePath.replaceAll("^/+", "");
long fileLength = file.length();
// Remove any trailing slashes
virtualDirectoryKeyPrefix = virtualDirectoryKeyPrefix.replaceAll("/+$", "");
String s3ObjectKey = virtualDirectoryKeyPrefix + "/" + relativeFilePath;
totalFileLength += fileLength;
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, s3ObjectKey, file);
ObjectMetadata objectMetadata = new ObjectMetadata();
metadataProvider.provideObjectMetadata(null, objectMetadata);
putObjectRequest.setMetadata(objectMetadata);
putObject(putObjectRequest, transferManager.getAmazonS3Client());
subTransfers.add(new UploadImpl(null, null, null, null));
}
TransferProgress progress = new TransferProgress();
progress.setTotalBytesToTransfer(totalFileLength);
progress.updateProgress(totalFileLength);
MultipleFileUploadImpl multipleFileUpload = new MultipleFileUploadImpl(null, progress, null, virtualDirectoryKeyPrefix, bucketName, subTransfers);
multipleFileUpload.setState(TransferState.Completed);
return multipleFileUpload;
}
private void writeFile( AmazonKey amazonKey, String bucket, String key, Map<String, String> metadata, StorageClass storage, String localpath, int retry, int retryseconds, boolean deletefile, boolean background, String callback, String callbackdata, String appname, String acl, String aes256key, Map<String, String> customheaders ) throws Exception {
File localFile = new File( localpath );
if ( !localFile.isFile() )
throw new Exception( "The file specified does not exist: " + localpath );
// Push this to the background loader to handle and return immediately
if ( background ) {
BackgroundUploader.acceptFile( amazonKey, bucket, key, metadata, storage, localpath, retry, retryseconds, deletefile, callback, callbackdata, appname, acl, aes256key, customheaders );
return;
}
// Setup the object data
ObjectMetadata omd = new ObjectMetadata();
if ( metadata != null )
omd.setUserMetadata( metadata );
AmazonS3 s3Client = getAmazonS3( amazonKey );
// Let us run around the number of attempts
int attempts = 0;
while ( attempts < retry ) {
try {
PutObjectRequest por = new PutObjectRequest( bucket, key, localFile );
por.setMetadata( omd );
por.setStorageClass( storage );
if ( acl != null && !acl.isEmpty() )
por.setCannedAcl( amazonKey.getAmazonCannedAcl( acl ) );
if ( aes256key != null && !aes256key.isEmpty() )
por.setSSECustomerKey( new SSECustomerKey( aes256key ) );
if ( customheaders != null && !customheaders.isEmpty() ) {
Iterator<String> it = customheaders.keySet().iterator();
while ( it.hasNext() ) {
String k = it.next();
por.putCustomRequestHeader( k, customheaders.get( k ) );
}
}
s3Client.putObject( por );
break;
} catch ( Exception e ) {
cfEngine.log( "Failed: AmazonS3Write(bucket=" + bucket + "key=" + key + "; file=" + localFile + "; attempt=" + ( attempts + 1 ) + "; exception=" + e.getMessage() + ")" );
attempts++;
if ( attempts == retry )
throw e;
else
Thread.sleep( retryseconds * 1000 );
}
}
// delete the file now that it is a success
if ( deletefile )
localFile.delete();
}
private void enableS3Encryption(PutObjectRequest uploadRequest) {
ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
uploadRequest.setMetadata(objectMetadata);
}
/**
* Put any file to Amazon S3 storage.
*
* @param bucket
* - S3 bucket name
* @param key
* - S3 storage path. Example:
* DEMO/TestSuiteName/TestMethodName/file.txt
* @param filePath
* - local storage path. Example: C:/Temp/file.txt
* @param metadata
* - custom tags metadata like name etc
*
*/
public void put(String bucket, String key, String filePath, ObjectMetadata metadata) {
/*
* if (mode != S3Mode.WRITE) {
* if (mode == S3Mode.READ) {
* LOGGER.warn("Unable to put data in READ mode!");
* }
* return;
* }
*/
if (key == null) {
throw new RuntimeException("Key is null!");
}
if (key.isEmpty()) {
throw new RuntimeException("Key is empty!");
}
if (filePath == null) {
throw new RuntimeException("FilePath is null!");
}
if (filePath.isEmpty()) {
throw new RuntimeException("FilePath is empty!");
}
File file = new File(filePath);
if (!file.exists()) {
throw new RuntimeException("File does not exist! " + filePath);
}
try {
LOGGER.debug("Uploading a new object to S3 from a file: "
+ filePath);
PutObjectRequest object = new PutObjectRequest(bucket, key, file);
if (metadata != null) {
object.setMetadata(metadata);
}
s3client.putObject(object);
LOGGER.debug("Uploaded to S3: '" + filePath + "' with key '" + key
+ "'");
} catch (AmazonServiceException ase) {
LOGGER.error("Caught an AmazonServiceException, which "
+ "means your request made it "
+ "to Amazon S3, but was rejected with an error response for some reason.\n"
+ "Error Message: " + ase.getMessage() + "\n"
+ "HTTP Status Code: " + ase.getStatusCode() + "\n"
+ "AWS Error Code: " + ase.getErrorCode() + "\n"
+ "Error Type: " + ase.getErrorType() + "\n"
+ "Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
LOGGER.error("Caught an AmazonClientException, which "
+ "means the client encountered "
+ "an internal error while trying to "
+ "communicate with S3, "
+ "such as not being able to access the network.\n"
+ "Error Message: " + ace.getMessage());
}
}
/**
* Create a putObject request.
* Adds the ACL and metadata
* @param key key of object
* @param metadata metadata header
* @param srcfile source file
* @return the request
*/
public PutObjectRequest newPutObjectRequest(String key,
ObjectMetadata metadata, File srcfile) {
PutObjectRequest putObjectRequest = new PutObjectRequest(mBucket, key,
srcfile);
putObjectRequest.setMetadata(metadata);
return putObjectRequest;
}