下面列出了com.amazonaws.services.s3.model.SSEAwsKeyManagementParams#com.amazonaws.services.s3.transfer.TransferManagerBuilder 实例代码，或者点击链接到github查看源代码，也可以在右侧发表评论。
private static void downloadByFullPathAndFileNamePrefix(ParameterTool params) {
final String bucket = params.getRequired("bucket");
final String s3prefix = params.getRequired("s3prefix");
final String localFolder = params.getRequired("localFolder");
final String s3filePrefix = params.get("s3filePrefix", "");
TransferManager tx = TransferManagerBuilder.defaultTransferManager();
Predicate<String> keyPredicate = getKeyFilterByFileNamePrefix(s3filePrefix);
KeyFilter keyFilter = s3filePrefix.isEmpty() ? KeyFilter.INCLUDE_ALL :
objectSummary -> keyPredicate.test(objectSummary.getKey());
try {
tx.downloadDirectory(bucket, s3prefix, new File(localFolder), keyFilter).waitForCompletion();
} catch (InterruptedException e) {
System.out.println("Transfer interrupted");
} finally {
tx.shutdownNow();
}
}
private static void downloadByFullPathAndFileNamePrefix(ParameterTool params) {
final String bucket = params.getRequired("bucket");
final String s3prefix = params.getRequired("s3prefix");
final String localFolder = params.getRequired("localFolder");
final String s3filePrefix = params.get("s3filePrefix", "");
TransferManager tx = TransferManagerBuilder.defaultTransferManager();
Predicate<String> keyPredicate = getKeyFilterByFileNamePrefix(s3filePrefix);
KeyFilter keyFilter = s3filePrefix.isEmpty() ? KeyFilter.INCLUDE_ALL :
objectSummary -> keyPredicate.test(objectSummary.getKey());
try {
tx.downloadDirectory(bucket, s3prefix, new File(localFolder), keyFilter).waitForCompletion();
} catch (InterruptedException e) {
System.out.println("Transfer interrupted");
} finally {
tx.shutdownNow();
}
}
/**
* Implementation of the Mapper::setup() method. This extracts the S3MapReduceCp options specified in the Job's
* configuration, to set up the Job.
*
* @param context Mapper's context.
* @throws IOException On IO failure.
* @throws InterruptedException If the job is interrupted.
*/
@Override
public void setup(Context context) throws IOException, InterruptedException {
conf = new S3MapReduceCpConfiguration(context.getConfiguration());
ignoreFailures = conf.getBoolean(ConfigurationVariable.IGNORE_FAILURES);
targetFinalPath = new Path(conf.get(S3MapReduceCpConstants.CONF_LABEL_TARGET_FINAL_PATH));
AwsS3ClientFactory awsS3ClientFactory = new AwsS3ClientFactory();
transferManager = TransferManagerBuilder
.standard()
.withMinimumUploadPartSize(conf.getLong(ConfigurationVariable.MINIMUM_UPLOAD_PART_SIZE))
.withMultipartUploadThreshold(conf.getLong(ConfigurationVariable.MULTIPART_UPLOAD_THRESHOLD))
.withS3Client(awsS3ClientFactory.newInstance(conf))
.withShutDownThreadPools(true)
.withExecutorFactory(new ExecutorFactory() {
@Override
public ExecutorService newExecutor() {
return Executors.newFixedThreadPool(conf.getInt(ConfigurationVariable.NUMBER_OF_UPLOAD_WORKERS));
}
})
.build();
}
public static void downloadDir(String bucket_name, String key_prefix,
String dir_path, boolean pause) {
System.out.println("downloading to directory: " + dir_path +
(pause ? " (pause)" : ""));
// snippet-start:[s3.java1.s3_xfer_mgr_download.directory]
TransferManager xfer_mgr = TransferManagerBuilder.standard().build();
try {
MultipleFileDownload xfer = xfer_mgr.downloadDirectory(
bucket_name, key_prefix, new File(dir_path));
// loop with Transfer.isDone()
XferMgrProgress.showTransferProgress(xfer);
// or block with Transfer.waitForCompletion()
XferMgrProgress.waitForCompletion(xfer);
} catch (AmazonServiceException e) {
System.err.println(e.getErrorMessage());
System.exit(1);
}
xfer_mgr.shutdownNow();
// snippet-end:[s3.java1.s3_xfer_mgr_download.directory]
}
public static void downloadFile(String bucket_name, String key_name,
String file_path, boolean pause) {
System.out.println("Downloading to file: " + file_path +
(pause ? " (pause)" : ""));
// snippet-start:[s3.java1.s3_xfer_mgr_download.single]
File f = new File(file_path);
TransferManager xfer_mgr = TransferManagerBuilder.standard().build();
try {
Download xfer = xfer_mgr.download(bucket_name, key_name, f);
// loop with Transfer.isDone()
XferMgrProgress.showTransferProgress(xfer);
// or block with Transfer.waitForCompletion()
XferMgrProgress.waitForCompletion(xfer);
} catch (AmazonServiceException e) {
System.err.println(e.getErrorMessage());
System.exit(1);
}
xfer_mgr.shutdownNow();
// snippet-end:[s3.java1.s3_xfer_mgr_download.single]
}
public static void copyObjectSimple(String from_bucket, String from_key,
String to_bucket, String to_key) {
// snippet-start:[s3.java1.s3_xfer_mgr_copy.copy_object]
System.out.println("Copying s3 object: " + from_key);
System.out.println(" from bucket: " + from_bucket);
System.out.println(" to s3 object: " + to_key);
System.out.println(" in bucket: " + to_bucket);
TransferManager xfer_mgr = TransferManagerBuilder.standard().build();
try {
Copy xfer = xfer_mgr.copy(from_bucket, from_key, to_bucket, to_key);
// loop with Transfer.isDone()
XferMgrProgress.showTransferProgress(xfer);
// or block with Transfer.waitForCompletion()
XferMgrProgress.waitForCompletion(xfer);
} catch (AmazonServiceException e) {
System.err.println(e.getErrorMessage());
System.exit(1);
}
xfer_mgr.shutdownNow();
// snippet-end:[s3.java1.s3_xfer_mgr_copy.copy_object]
}
public static void uploadDir(String dir_path, String bucket_name,
String key_prefix, boolean recursive, boolean pause) {
System.out.println("directory: " + dir_path + (recursive ?
" (recursive)" : "") + (pause ? " (pause)" : ""));
// snippet-start:[s3.java1.s3_xfer_mgr_upload.directory]
TransferManager xfer_mgr = TransferManagerBuilder.standard().build();
try {
MultipleFileUpload xfer = xfer_mgr.uploadDirectory(bucket_name,
key_prefix, new File(dir_path), recursive);
// loop with Transfer.isDone()
XferMgrProgress.showTransferProgress(xfer);
// or block with Transfer.waitForCompletion()
XferMgrProgress.waitForCompletion(xfer);
} catch (AmazonServiceException e) {
System.err.println(e.getErrorMessage());
System.exit(1);
}
xfer_mgr.shutdownNow();
// snippet-end:[s3.java1.s3_xfer_mgr_upload.directory]
}
public PrestoS3OutputStream(
AmazonS3 s3,
String bucket,
String key,
File tempFile,
boolean sseEnabled,
PrestoS3SseType sseType,
String sseKmsKeyId,
long multiPartUploadMinFileSize,
long multiPartUploadMinPartSize,
PrestoS3AclType aclType,
boolean requesterPaysEnabled,
PrestoS3StorageClass s3StorageClass)
throws IOException
{
super(new BufferedOutputStream(new FileOutputStream(requireNonNull(tempFile, "tempFile is null"))));
transferManager = TransferManagerBuilder.standard()
.withS3Client(requireNonNull(s3, "s3 is null"))
.withMinimumUploadPartSize(multiPartUploadMinPartSize)
.withMultipartUploadThreshold(multiPartUploadMinFileSize).build();
requireNonNull(aclType, "aclType is null");
requireNonNull(s3StorageClass, "s3StorageClass is null");
this.aclType = aclType.getCannedACL();
this.bucket = requireNonNull(bucket, "bucket is null");
this.key = requireNonNull(key, "key is null");
this.tempFile = tempFile;
this.sseEnabled = sseEnabled;
this.sseType = requireNonNull(sseType, "sseType is null");
this.sseKmsKeyId = sseKmsKeyId;
this.requesterPaysEnabled = requesterPaysEnabled;
this.s3StorageClass = s3StorageClass.getS3StorageClass();
log.debug("OutputStream for key '%s' using file: %s", key, tempFile);
}
private static void downloadFile(ParameterTool params) {
final String bucket = params.getRequired("bucket");
final String s3file = params.getRequired("s3file");
final String localFile = params.getRequired("localFile");
TransferManager tx = TransferManagerBuilder.defaultTransferManager();
try {
tx.download(bucket, s3file, new File(localFile)).waitForCompletion();
} catch (InterruptedException e) {
System.out.println("Transfer interrupted");
} finally {
tx.shutdownNow();
}
}
private static void downloadFile(ParameterTool params) {
final String bucket = params.getRequired("bucket");
final String s3file = params.getRequired("s3file");
final String localFile = params.getRequired("localFile");
TransferManager tx = TransferManagerBuilder.defaultTransferManager();
try {
tx.download(bucket, s3file, new File(localFile)).waitForCompletion();
} catch (InterruptedException e) {
System.out.println("Transfer interrupted");
} finally {
tx.shutdownNow();
}
}
public TransferManager newInstance(AmazonS3 targetS3Client, S3S3CopierOptions s3s3CopierOptions) {
LOG
.debug("Initializing transfer manager with {} threads.", s3s3CopierOptions.getMaxThreadPoolSize());
return TransferManagerBuilder.standard()
.withMultipartCopyThreshold(s3s3CopierOptions.getMultipartCopyThreshold())
.withMultipartCopyPartSize(s3s3CopierOptions.getMultipartCopyPartSize())
.withExecutorFactory(() -> Executors.newFixedThreadPool(s3s3CopierOptions.getMaxThreadPoolSize()))
.withS3Client(targetS3Client)
.build();
}
@RequestMapping("/anuncio/nuevo")
public String nuevoAnuncio(Model model,
@RequestParam String nombre,
@RequestParam String asunto,
@RequestParam String comentario,
@RequestParam String filename,
@RequestParam MultipartFile file) {
if (!file.isEmpty()) {
try {
ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.setContentType(file.getContentType());
TransferManager transferManager = TransferManagerBuilder.defaultTransferManager();
transferManager.upload(bucket, filename, file.getInputStream(), objectMetadata);
} catch (Exception e) {
model.addAttribute("message", "You failed to upload " + filename + " => " + e.getMessage());
return "error";
}
} else {
model.addAttribute("message", "You failed to upload " + filename + " because the file was empty.");
return "error";
}
Anuncio anuncio = new Anuncio(nombre, asunto, comentario);
anuncio.setFoto(s3.getUrl(bucket, filename));
repository.save(anuncio);
return "anuncio_guardado";
}
@Override
public Void invoke(File localFile, VirtualChannel channel) throws IOException, InterruptedException {
TransferManager mgr = TransferManagerBuilder.standard()
.withS3Client(AWSClientFactory.create(this.amazonS3ClientOptions.createAmazonS3ClientBuilder(), this.envVars))
.build();
if (this.path == null || this.path.isEmpty() || this.path.endsWith("/")) {
try {
final MultipleFileDownload fileDownload = mgr.downloadDirectory(this.bucket, this.path, localFile);
fileDownload.waitForCompletion();
RemoteDownloader.this.taskListener.getLogger().println("Finished: " + fileDownload.getDescription());
}
finally {
mgr.shutdownNow();
}
return null;
} else {
try {
final Download download = mgr.download(this.bucket, this.path, localFile);
download.addProgressListener((ProgressListener) progressEvent -> {
if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) {
RemoteDownloader.this.taskListener.getLogger().println("Finished: " + download.getDescription());
}
});
download.waitForCompletion();
}
finally {
mgr.shutdownNow();
}
return null;
}
}
@Before
public void setupSdk() throws Exception {
PowerMockito.mockStatic(AWSClientFactory.class);
AmazonS3 amazonS3 = PowerMockito.mock(AmazonS3.class);
PowerMockito.when(AWSClientFactory.create(Mockito.any(AwsSyncClientBuilder.class), Mockito.any(StepContext.class)))
.thenReturn(amazonS3);
PowerMockito.mockStatic(TransferManagerBuilder.class);
transferManager = Mockito.mock(TransferManager.class);
TransferManagerBuilder transferManagerBuilder = PowerMockito.mock(TransferManagerBuilder.class);
PowerMockito.when(TransferManagerBuilder.standard()).thenReturn(transferManagerBuilder);
PowerMockito.when(transferManagerBuilder.withS3Client(Mockito.any(AmazonS3.class))).thenReturn(transferManagerBuilder);
PowerMockito.when(transferManagerBuilder.build()).thenReturn(transferManager);
}
public static void uploadFileList(String[] file_paths, String bucket_name,
String key_prefix, boolean pause) {
System.out.println("file list: " + Arrays.toString(file_paths) +
(pause ? " (pause)" : ""));
// convert the file paths to a list of File objects (required by the
// uploadFileList method)
// snippet-start:[s3.java1.s3_xfer_mgr_upload.list_of_files]
ArrayList<File> files = new ArrayList<File>();
for (String path : file_paths) {
files.add(new File(path));
}
TransferManager xfer_mgr = TransferManagerBuilder.standard().build();
try {
MultipleFileUpload xfer = xfer_mgr.uploadFileList(bucket_name,
key_prefix, new File("."), files);
// loop with Transfer.isDone()
XferMgrProgress.showTransferProgress(xfer);
// or block with Transfer.waitForCompletion()
XferMgrProgress.waitForCompletion(xfer);
} catch (AmazonServiceException e) {
System.err.println(e.getErrorMessage());
System.exit(1);
}
xfer_mgr.shutdownNow();
// snippet-end:[s3.java1.s3_xfer_mgr_upload.list_of_files]
}
public static void uploadFile(String file_path, String bucket_name,
String key_prefix, boolean pause) {
System.out.println("file: " + file_path +
(pause ? " (pause)" : ""));
String key_name = null;
if (key_prefix != null) {
key_name = key_prefix + '/' + file_path;
} else {
key_name = file_path;
}
// snippet-start:[s3.java1.s3_xfer_mgr_upload.single]
File f = new File(file_path);
TransferManager xfer_mgr = TransferManagerBuilder.standard().build();
try {
Upload xfer = xfer_mgr.upload(bucket_name, key_name, f);
// loop with Transfer.isDone()
XferMgrProgress.showTransferProgress(xfer);
// or block with Transfer.waitForCompletion()
XferMgrProgress.waitForCompletion(xfer);
} catch (AmazonServiceException e) {
System.err.println(e.getErrorMessage());
System.exit(1);
}
xfer_mgr.shutdownNow();
// snippet-end:[s3.java1.s3_xfer_mgr_upload.single]
}
public void upload(final AmazonS3 s3, final String bucket, final String key, final InputStream contents) {
try {
TransferManager transferManager = TransferManagerBuilder.standard().withS3Client(s3).build();
transferManager.upload(bucket, key, contents, new ObjectMetadata())
.waitForCompletion();
} catch (InterruptedException e) {
throw new BlobStoreException("error uploading blob", e, null);
}
}
public S3FileManagerImpl(AmazonS3 amazonS3, @Value("${aws.upload.parallelism}") int parallelism,
@Value("${aws.upload.multipart.mb.size:5}") int multipartSize) {
this.amazonS3 = amazonS3;
this.transferManager = TransferManagerBuilder
.standard()
.withS3Client(amazonS3)
.withMultipartUploadThreshold(multipartSize * 1024L * 1024L)
.withExecutorFactory(() -> Executors.newFixedThreadPool(parallelism))
.build();
}
S3Manager(EmrClusterConfig pipelineEmrConfigs, String pipelineId, String uniquePrefix, List<File> filesToUpload) {
this.pipelineEmrConfigs = pipelineEmrConfigs;
this.pipelineId = pipelineId;
this.uniquePrefix = uniquePrefix;
this.filesToUpload = filesToUpload;
s3Client = getS3Client();
int uploadThreads = Integer.parseInt(System.getProperty("sdc.emr.s3.uploadThreads", "5"));
s3TransferManager = TransferManagerBuilder.standard()
.withS3Client(s3Client)
.withMultipartUploadThreshold(10L*1024*1024)
.withMinimumUploadPartSize(10L*1024*1024)
.withExecutorFactory(() -> Executors.newFixedThreadPool(uploadThreads))
.withShutDownThreadPools(true)
.build();
}
private static TransferManager getTransferManager(AmazonS3 s3Client, ExecutorService executorService) {
return TransferManagerBuilder
.standard()
.withS3Client(s3Client)
.withMultipartUploadThreshold(MULTIPART_UPLOAD_THRESHOLD.getValue())
.withExecutorFactory(() -> executorService)
.withShutDownThreadPools(DO_NOT_SHUTDOWN_THREAD_POOL)
.build();
}
public static void main(String[] args) throws Exception {
String existingBucketName = "baeldung-bucket";
String keyName = "my-picture.jpg";
String filePath = "documents/my-picture.jpg";
AmazonS3 amazonS3 = AmazonS3ClientBuilder
.standard()
.withCredentials(new DefaultAWSCredentialsProviderChain())
.withRegion(Regions.DEFAULT_REGION)
.build();
int maxUploadThreads = 5;
TransferManager tm = TransferManagerBuilder
.standard()
.withS3Client(amazonS3)
.withMultipartUploadThreshold((long) (5 * 1024 * 1024))
.withExecutorFactory(() -> Executors.newFixedThreadPool(maxUploadThreads))
.build();
ProgressListener progressListener =
progressEvent -> System.out.println("Transferred bytes: " + progressEvent.getBytesTransferred());
PutObjectRequest request = new PutObjectRequest(existingBucketName, keyName, new File(filePath));
request.setGeneralProgressListener(progressListener);
Upload upload = tm.upload(request);
try {
upload.waitForCompletion();
System.out.println("Upload complete.");
} catch (AmazonClientException e) {
System.out.println("Error occurred while uploading file");
e.printStackTrace();
}
}
@Before
public void setup() {
amazonS3 = mock(AmazonS3.class);
tm = TransferManagerBuilder
.standard()
.withS3Client(amazonS3)
.withMultipartUploadThreshold((long) (5 * 1024 * 1025))
.withExecutorFactory(() -> Executors.newFixedThreadPool(5))
.build();
progressListener =
progressEvent -> System.out.println("Transferred bytes: " + progressEvent.getBytesTransferred());
}
public S3SchemaUriResolver(AmazonS3 s3Client, String bucket, String keyPrefix, boolean enableServerSideEncryption) {
this(TransferManagerBuilder.standard().withS3Client(s3Client).build(), S3_URI_FORMAT, bucket, keyPrefix,
enableServerSideEncryption);
}
public TransferManager build(final AbstractOperationRequest operationRequest) {
final AmazonS3 amazonS3 = provideAmazonS3(coreV1ApiProvider, operationRequest);
return TransferManagerBuilder.standard().withS3Client(amazonS3).build();
}
@Override
public String run() throws Exception {
final String fromBucket = this.step.getFromBucket();
final String toBucket = this.step.getToBucket();
final String fromPath = this.step.getFromPath();
final String toPath = this.step.getToPath();
final String kmsId = this.step.getKmsId();
final Map<String, String> metadatas = new HashMap<>();
final CannedAccessControlList acl = this.step.getAcl();
final String cacheControl = this.step.getCacheControl();
final String contentType = this.step.getContentType();
final String sseAlgorithm = this.step.getSseAlgorithm();
final S3ClientOptions s3ClientOptions = this.step.createS3ClientOptions();
final EnvVars envVars = this.getContext().get(EnvVars.class);
if (this.step.getMetadatas() != null && this.step.getMetadatas().length != 0) {
for (String metadata : this.step.getMetadatas()) {
if (metadata.split(":").length == 2) {
metadatas.put(metadata.split(":")[0], metadata.split(":")[1]);
}
}
}
Preconditions.checkArgument(fromBucket != null && !fromBucket.isEmpty(), "From bucket must not be null or empty");
Preconditions.checkArgument(fromPath != null && !fromPath.isEmpty(), "From path must not be null or empty");
Preconditions.checkArgument(toBucket != null && !toBucket.isEmpty(), "To bucket must not be null or empty");
Preconditions.checkArgument(toPath != null && !toPath.isEmpty(), "To path must not be null or empty");
TaskListener listener = Execution.this.getContext().get(TaskListener.class);
listener.getLogger().format("Copying s3://%s/%s to s3://%s/%s%n", fromBucket, fromPath, toBucket, toPath);
CopyObjectRequest request = new CopyObjectRequest(fromBucket, fromPath, toBucket, toPath);
// Add metadata
if (metadatas.size() > 0 || (cacheControl != null && !cacheControl.isEmpty()) || (contentType != null && !contentType.isEmpty()) || (sseAlgorithm != null && !sseAlgorithm.isEmpty())) {
ObjectMetadata metas = new ObjectMetadata();
if (metadatas.size() > 0) {
metas.setUserMetadata(metadatas);
}
if (cacheControl != null && !cacheControl.isEmpty()) {
metas.setCacheControl(cacheControl);
}
if (contentType != null && !contentType.isEmpty()) {
metas.setContentType(contentType);
}
if (sseAlgorithm != null && !sseAlgorithm.isEmpty()) {
metas.setSSEAlgorithm(sseAlgorithm);
}
request.withNewObjectMetadata(metas);
}
// Add acl
if (acl != null) {
request.withCannedAccessControlList(acl);
}
// Add kms
if (kmsId != null && !kmsId.isEmpty()) {
listener.getLogger().format("Using KMS: %s%n", kmsId);
request.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(kmsId));
}
TransferManager mgr = TransferManagerBuilder.standard()
.withS3Client(AWSClientFactory.create(s3ClientOptions.createAmazonS3ClientBuilder(), envVars))
.build();
try {
final Copy copy = mgr.copy(request);
copy.addProgressListener((ProgressListener) progressEvent -> {
if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) {
listener.getLogger().println("Finished: " + copy.getDescription());
}
});
copy.waitForCompletion();
}
finally{
mgr.shutdownNow();
}
listener.getLogger().println("Copy complete");
return String.format("s3://%s/%s", toBucket, toPath);
}
@Override
public Void invoke(File localFile, VirtualChannel channel) throws IOException, InterruptedException {
TransferManager mgr = TransferManagerBuilder.standard()
.withS3Client(AWSClientFactory.create(this.amazonS3ClientOptions.createAmazonS3ClientBuilder(), this.envVars))
.build();
final MultipleFileUpload fileUpload;
ObjectMetadataProvider metadatasProvider = (file, meta) -> {
if (meta != null) {
if (RemoteListUploader.this.metadatas != null && RemoteListUploader.this.metadatas.size() > 0) {
meta.setUserMetadata(RemoteListUploader.this.metadatas);
}
if (RemoteListUploader.this.acl != null) {
meta.setHeader(Headers.S3_CANNED_ACL, RemoteListUploader.this.acl);
}
if (RemoteListUploader.this.cacheControl != null && !RemoteListUploader.this.cacheControl.isEmpty()) {
meta.setCacheControl(RemoteListUploader.this.cacheControl);
}
if (RemoteListUploader.this.contentEncoding != null && !RemoteListUploader.this.contentEncoding.isEmpty()) {
meta.setContentEncoding(RemoteListUploader.this.contentEncoding);
}
if (RemoteListUploader.this.contentType != null && !RemoteListUploader.this.contentType.isEmpty()) {
meta.setContentType(RemoteListUploader.this.contentType);
}
if (RemoteListUploader.this.sseAlgorithm != null && !RemoteListUploader.this.sseAlgorithm.isEmpty()) {
meta.setSSEAlgorithm(RemoteListUploader.this.sseAlgorithm);
}
if (RemoteListUploader.this.kmsId != null && !RemoteListUploader.this.kmsId.isEmpty()) {
final SSEAwsKeyManagementParams sseAwsKeyManagementParams = new SSEAwsKeyManagementParams(RemoteListUploader.this.kmsId);
meta.setSSEAlgorithm(sseAwsKeyManagementParams.getAwsKmsKeyId());
meta.setHeader(
Headers.SERVER_SIDE_ENCRYPTION_AWS_KMS_KEYID,
sseAwsKeyManagementParams.getAwsKmsKeyId()
);
}
}
};
ObjectTaggingProvider objectTaggingProvider =(uploadContext) -> {
List<Tag> tagList = new ArrayList<Tag>();
//add tags
if(tags != null){
for (Map.Entry<String, String> entry : tags.entrySet()) {
Tag tag = new Tag(entry.getKey(), entry.getValue());
tagList.add(tag);
}
}
return new ObjectTagging(tagList);
};
try {
fileUpload = mgr.uploadFileList(this.bucket, this.path, localFile, this.fileList, metadatasProvider, objectTaggingProvider);
for (final Upload upload : fileUpload.getSubTransfers()) {
upload.addProgressListener((ProgressListener) progressEvent -> {
if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) {
RemoteListUploader.this.taskListener.getLogger().println("Finished: " + upload.getDescription());
}
});
}
fileUpload.waitForCompletion();
}
finally {
mgr.shutdownNow();
}
return null;
}
public S3Downloader(AmazonS3Client s3Client) {
this.s3Client = s3Client;
transferManager = TransferManagerBuilder.standard().withS3Client(s3Client).build();
}
TransferManagerBuilder createTransferManagerBuilder() {
return TransferManagerBuilder.standard();
}
@Override
public List<ConfigIssue> init() {
List<ConfigIssue> issues = s3TargetConfigBean.init(getContext(), super.init(), isErrorStage);
elVars = getContext().createELVars();
bucketEval = getContext().createELEval(BUCKET_TEMPLATE);
partitionEval = getContext().createELEval(PARTITION_TEMPLATE);
timeDriverEval = getContext().createELEval(TIME_DRIVER_TEMPLATE);
calendar = Calendar.getInstance(TimeZone.getTimeZone(ZoneId.of(s3TargetConfigBean.timeZoneID)));
transferManager = TransferManagerBuilder
.standard()
.withS3Client(s3TargetConfigBean.s3Config.getS3Client())
.withExecutorFactory(() -> Executors.newFixedThreadPool(s3TargetConfigBean.tmConfig.threadPoolSize))
.withMinimumUploadPartSize(s3TargetConfigBean.tmConfig.minimumUploadPartSize)
.withMultipartUploadThreshold(s3TargetConfigBean.tmConfig.multipartUploadThreshold)
.build();
TimeEL.setCalendarInContext(elVars, calendar);
TimeNowEL.setTimeNowInContext(elVars, new Date());
if (partitionTemplate.contains(EL_PREFIX)) {
ELUtils.validateExpression(partitionTemplate,
getContext(),
Groups.S3.getLabel(),
S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + PARTITION_TEMPLATE,
Errors.S3_03, issues
);
}
if (timeDriverTemplate.contains(EL_PREFIX)) {
ELUtils.validateExpression(timeDriverTemplate,
getContext(),
Groups.S3.getLabel(),
S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + TIME_DRIVER_TEMPLATE,
Errors.S3_04, issues
);
}
if (!isErrorStage && getContext().getService(DataFormatGeneratorService.class).isWholeFileFormat()) {
fileHelper = new WholeFileHelper(getContext(), s3TargetConfigBean, transferManager, issues);
} else {
fileHelper = new DefaultFileHelper(getContext(), s3TargetConfigBean, transferManager, isErrorStage);
}
this.errorRecordHandler = new DefaultErrorRecordHandler(getContext());
return issues;
}
@Override
void putObject(SyncObject obj, String targetKey) {
ObjectMetadata om;
if (options.isSyncMetadata()) om = s3MetaFromSyncMeta(obj.getMetadata());
else om = new ObjectMetadata();
if (obj.getMetadata().isDirectory()) om.setContentType(TYPE_DIRECTORY);
PutObjectRequest req;
File file = (File) obj.getProperty(AbstractFilesystemStorage.PROP_FILE);
S3ProgressListener progressListener = null;
if (obj.getMetadata().isDirectory()) {
req = new PutObjectRequest(config.getBucketName(), targetKey, new ByteArrayInputStream(new byte[0]), om);
} else if (file != null) {
req = new PutObjectRequest(config.getBucketName(), targetKey, file).withMetadata(om);
progressListener = new ByteTransferListener(obj);
} else {
InputStream stream = obj.getDataStream();
if (options.isMonitorPerformance())
stream = new ProgressInputStream(stream, new PerformanceListener(getWriteWindow()));
req = new PutObjectRequest(config.getBucketName(), targetKey, stream, om);
}
if (options.isSyncAcl())
req.setAccessControlList(s3AclFromSyncAcl(obj.getAcl(), options.isIgnoreInvalidAcls()));
TransferManager xferManager = null;
try {
// xfer manager will figure out if MPU is needed (based on threshold), do the MPU if necessary,
// and abort if it fails
xferManager = TransferManagerBuilder.standard()
.withS3Client(s3)
.withExecutorFactory(() -> Executors.newFixedThreadPool(config.getMpuThreadCount()))
.withMultipartUploadThreshold((long) config.getMpuThresholdMb() * 1024 * 1024)
.withMinimumUploadPartSize((long) config.getMpuPartSizeMb() * 1024 * 1024)
.withShutDownThreadPools(true)
.build();
// directly update
final Upload upload = xferManager.upload(req, progressListener);
try {
String eTag = time((Callable<String>) () -> upload.waitForUploadResult().getETag(), OPERATION_MPU);
log.debug("Wrote {}, etag: {}", targetKey, eTag);
} catch (Exception e) {
log.error("upload exception", e);
if (e instanceof RuntimeException) throw (RuntimeException) e;
throw new RuntimeException("upload thread was interrupted", e);
}
} finally {
// NOTE: apparently if we do not reference xferManager again after the upload() call (as in this finally
// block), the JVM will for some crazy reason determine it is eligible for GC and call finalize(), which
// shuts down the thread pool, fails the upload, and gives absolutely no indication of what's going on...
if (xferManager != null) xferManager.shutdownNow(false);
}
}