下面列出了怎么用com.amazonaws.services.s3.model.S3Object的API类实例代码及写法,或者点击链接到github查看源代码。
@Test
void fetchCsvFileTest() throws IOException
{
String objectKey = S3_OBJECT_KEY + ".csv";
byte[] csv = ResourceUtils.loadResourceAsByteArray(CSV_FILE_PATH);
S3Object s3Object = mock(S3Object.class);
S3ObjectInputStream s3ObjectInputStream = new S3ObjectInputStream(new ByteArrayInputStream(csv), null);
when(s3Object.getObjectContent()).thenReturn(s3ObjectInputStream);
when(amazonS3Client.getObject(S3_BUCKET_NAME, objectKey)).thenReturn(s3Object);
Set<VariableScope> scopes = Set.of(VariableScope.SCENARIO);
String variableName = "varName";
steps.fetchCsvObject(objectKey, S3_BUCKET_NAME, scopes, variableName);
verify(amazonS3Client).getObject(S3_BUCKET_NAME, objectKey);
verify(bddVariableContext).putVariable(scopes, variableName, List.of(Map.of("id", "1")));
}
/**
* @param bucketName + key location
* @return {@link S3Object} for the given aws s3 location.
* @throws InvalidS3LocationException for invalid location params
* @throws S3ResourceException for connection and availability errors
*/
public S3Object retriveFrom(String location) {
if (isEmpty(location)) {
throw new InvalidS3LocationException("Location cannot be empty or null");
}
String path = location.startsWith(S3_PROTOCOL_PREFIX) ? location.substring(S3_PROTOCOL_PREFIX.length(), location.length()) : location;
if(!path.contains("/")) {
throw new InvalidS3LocationException("The location must contains the full path of the properties file");
}
String bucketName = path.substring(0, path.indexOf('/'));
String keyName = path.substring(path.indexOf('/') + 1);
try {
return amazonS3.getObject(bucketName, keyName);
} catch (Exception e) {
throw new S3ResourceException("Could not load resource from " + location, e);
}
}
@Override
public boolean isAvailableInDatabase(String state) {
AmazonS3 s3 = this.createS3Client();
S3Object s3Object = getObject(s3, getKey(state));
if (s3Object == null) {
return false;
}
String millisToExpire = null;
try {
millisToExpire = IOUtils.toString(s3Object.getObjectContent());
return Long.valueOf(millisToExpire) > System.currentTimeMillis();
} catch (IOException e) {
log.error("Failed to load a state data for state: {}", state, e);
return false;
} catch (NumberFormatException ne) {
log.error("Invalid state value detected - state: {}, millisToExpire: {}", state, millisToExpire);
return false;
}
}
@BeforeClass
public static void setup() throws IOException {
File remoteFolder = TEMPORARY_FOLDER.newFolder("remote");
File aFile = new File(remoteFolder, "1.test");
FileCopyUtils.copy("Hello".getBytes(), aFile);
File bFile = new File(remoteFolder, "2.test");
FileCopyUtils.copy("Bye".getBytes(), bFile);
File otherFile = new File(remoteFolder, "otherFile");
FileCopyUtils.copy("Other\nOther2".getBytes(), otherFile);
S3_OBJECTS = new ArrayList<>();
for (File file : remoteFolder.listFiles()) {
S3Object s3Object = new S3Object();
s3Object.setBucketName(S3_BUCKET);
s3Object.setKey(file.getName());
s3Object.setObjectContent(new FileInputStream(file));
S3_OBJECTS.add(s3Object);
}
String localFolder = TEMPORARY_FOLDER.newFolder("local").getAbsolutePath();
System.setProperty("s3.localDir", localFolder);
}
/**
* get one component bundle files from s3 server as json String
*/
@Override
public String get2JsonStr(String productName, String version, String component, String locale)
throws DataException {
String filePath = S3Utils.genProductVersionS3Path(productName, version) + component
+ ConstantsChar.BACKSLASH + ResourceFilePathGetter.getLocalizedJSONFileName(locale);
String result = null;
if (s3Client.getS3Client().doesObjectExist(config.getBucketName(), filePath)) {
S3Object o = s3Client.getS3Client().getObject(config.getBucketName(), filePath);
if (o != null) {
try {
result = S3Utils.convertS3Obj2Str(o);
} catch (IOException e) {
logger.warn(e.getMessage(), e);
throw new DataException(S3_NOT_EXIST_STR + filePath);
}
} else {
throw new DataException(S3_NOT_EXIST_STR + filePath);
}
}
if (result == null) {
throw new DataException(S3_NOT_EXIST_STR + filePath);
}
return result;
}
/**
* convert the S3 Object to String
*/
public static String convertS3Obj2Str(S3Object s3Obj) throws IOException {
S3ObjectInputStream s3is = s3Obj.getObjectContent();
ByteArrayOutputStream fos = new ByteArrayOutputStream();
byte[] read_buf = new byte[1024];
int read_len = 0;
try {
while ((read_len = s3is.read(read_buf)) > 0) {
fos.write(read_buf, 0, read_len);
}
return fos.toString(ConstantsUnicode.UTF8);
} finally {
s3is.close();
fos.close();
}
}
@Override
public Properties getProperties(String bucketName, String key, S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto)
{
AmazonS3Client s3Client = getAmazonS3(s3FileTransferRequestParamsDto);
try
{
S3Object s3Object = getS3Object(s3Client, bucketName, key, true);
return javaPropertiesHelper.getProperties(s3Object.getObjectContent());
}
catch (IllegalArgumentException e)
{
throw new IllegalArgumentException("The properties file in S3 bucket '" + bucketName + "' and key '" + key + "' is invalid.", e);
}
finally
{
s3Client.shutdown();
}
}
/**
* This function retrieves large object from S3
* @param bucketName
* @param keyName
* @param objectClass
* @return
*/
public <T> Object retrieveLargeItem(String bucketName, String keyName, Class<T> objectClass) {
try {
S3Object s3Object = s3.getObject(new GetObjectRequest(bucketName.toLowerCase(), keyName));
String objectContent = IOUtils.toString(s3Object.getObjectContent(), StandardCharsets.UTF_8);
if(objectClass == ReCiterFeature.class) {
ReCiterFeature reCiterFeature = OBJECT_MAPPER.readValue(objectContent, ReCiterFeature.class);
return reCiterFeature;
}
} catch (IOException | AmazonServiceException e) {
log.error(e.getMessage());
}
return null;
}
/**
* Fetch error info.
*
* @param datasource the datasource
* @param errorList the error list
*/
private void fetchErrorInfo(List<Map<String,String>> errorList){
if(errorInfo==null){
ObjectMapper objectMapper = new ObjectMapper();
List<Map<String, String>> inventoryErrors = new ArrayList<>();
AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
.withCredentials(new AWSStaticCredentialsProvider(new CredentialProvider().getCredentials(s3Account,s3Role))).withRegion(s3Region).build();
try {
S3Object inventoryErrorData = s3Client.getObject(new GetObjectRequest(bucketName,dataPath+"/"+dataSource+"-loaderror.data"));
try (BufferedReader reader = new BufferedReader(new InputStreamReader(inventoryErrorData.getObjectContent()))) {
inventoryErrors = objectMapper.readValue(reader.lines().collect(Collectors.joining("\n")),new TypeReference<List<Map<String, String>>>() {});
}
} catch (IOException e) {
LOGGER.error("Exception in collecting inventory error data",e);
Map<String,String> errorMap = new HashMap<>();
errorMap.put(ERROR, "Exception in collecting inventory error data");
errorMap.put(ERROR_TYPE, WARN);
errorMap.put(EXCEPTION, e.getMessage());
errorList.add(errorMap);
}
errorInfo = inventoryErrors.parallelStream().collect(Collectors.groupingBy(obj -> obj.get("type")));
}
}
SignedDomain getSignedDomain(AmazonS3 s3, String domainName) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("getSignedDomain with S3: {}", domainName);
}
SignedDomain signedDomain = null;
try {
S3Object object = s3.getObject(s3BucketName, domainName);
try (S3ObjectInputStream s3is = object.getObjectContent()) {
signedDomain = jsonMapper.readValue(s3is, SignedDomain.class);
}
} catch (Exception ex) {
LOGGER.error("AWSS3ChangeLog: getSignedDomain - unable to get domain {} error: {}",
domainName, ex.getMessage());
}
return signedDomain;
}
/**
* Get MD5, size, owner, storage class and last modification time for a desired file in the pointed bucket
*
* @param fileName the file name
*/
@PublicAtsApi
public S3ObjectInfo getFileMetadata( String fileName ) {
try {
S3Object element = s3Client.getObject(bucketName, fileName);
if (element != null) {
ObjectMetadata metaData = element.getObjectMetadata();
S3ObjectInfo s3Info = new S3ObjectInfo();
s3Info.setBucketName(fileName);
s3Info.setLastModified(metaData.getLastModified());
s3Info.setMd5(metaData.getETag());
s3Info.setName(element.getKey());
s3Info.setSize(metaData.getContentLength());
return s3Info;
} else {
throw new NoSuchElementException("File with name '" + fileName + "' does not exist!");
}
} catch (Exception e) {
handleExeption(e, "Could not retrieve metadata for S3 object with key '" + fileName + "'");
}
return null;
}
private static void streamReadAndDownloadObject(
final File workspace,
final S3Object sessionObject,
final String downloadedFileName) throws IOException {
final File outputFile = new File(workspace, downloadedFileName);
try (final S3ObjectInputStream objectContents = sessionObject.getObjectContent();
final OutputStream outputStream = new FileOutputStream(outputFile)) {
final int BUFFER_SIZE = 8192;
final byte[] buffer = new byte[BUFFER_SIZE];
int i;
while ((i = objectContents.read(buffer)) != -1) {
outputStream.write(buffer, 0, i);
}
}
}
@Test
public void testAwsV4SignatureChunkedAnonymous() throws Exception {
client = AmazonS3ClientBuilder.standard()
.withChunkedEncodingDisabled(false)
.withEndpointConfiguration(s3EndpointConfig)
.build();
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(BYTE_SOURCE.size());
client.putObject(containerName, "foo", BYTE_SOURCE.openStream(),
metadata);
S3Object object = client.getObject(containerName, "foo");
assertThat(object.getObjectMetadata().getContentLength()).isEqualTo(
BYTE_SOURCE.size());
try (InputStream actual = object.getObjectContent();
InputStream expected = BYTE_SOURCE.openStream()) {
assertThat(actual).hasContentEqualTo(expected);
}
}
private void downloadAndExtract(
final S3Object sessionObject,
final File workspace,
final String downloadedFileName,
final TaskListener listener) throws IOException {
downloadArtifacts(sessionObject, workspace, downloadedFileName, listener);
final File fullFilePath = new File(workspace, downloadedFileName);
try {
ExtractionTools.decompressFile(fullFilePath, workspace, model.getCompressionType(), listener);
LoggingHelper.log(listener, "Artifact uncompressed successfully");
} finally {
if (fullFilePath != null) {
try {
ExtractionTools.deleteTemporaryCompressedFile(fullFilePath);
} catch (final IOException ex) {
LoggingHelper.log(listener, "Could not delete temporary file: %s", ex.getMessage());
LoggingHelper.log(listener, ex);
}
}
}
}
@Test
public void loadGetsObjectsAndReturnsTrueIfItExistsInS3() throws Exception {
/** Setup **/
buildCacheService = new AwsS3BuildCacheService(s3, "bucketName", null, true);
doReturn(true).when(s3).doesObjectExist("bucketName", "abcdefghijkl123456789");
S3Object s3Object = mock(S3Object.class);
doReturn(s3Object).when(s3).getObject("bucketName", "abcdefghijkl123456789");
S3ObjectInputStream s3ObjectInputStream = mock(S3ObjectInputStream.class);
doReturn(s3ObjectInputStream).when(s3Object).getObjectContent();
/** Run **/
boolean result = buildCacheService.load(key, reader);
/** Check **/
assertTrue(result);
verify(reader).readFrom(s3ObjectInputStream);
}
public static void downloadFromS3(String s3BucketName, List<String> objectKeys,
String downloadDirectory) {
// Initializes the Amazon S3 client.
AmazonS3 s3Client = AmazonS3ClientBuilder.defaultClient();
try {
// Downloads each object to the specified file path.
for (String key : objectKeys) {
S3Object object = s3Client.getObject(s3BucketName, key);
String endpointsFileName = key.substring(key.lastIndexOf("/"));
Path filePath = Paths.get(downloadDirectory + endpointsFileName);
System.out.format("Downloading %s to %s . . .\n",
filePath.getFileName(), filePath.getParent());
writeObjectToFile(filePath, object);
}
System.out.println("Download finished.");
} catch (AmazonServiceException | NullPointerException e) {
System.err.println(e.getMessage());
System.exit(1);
}
}
@Test
public void copyOneFile() throws Exception {
client.putObject("source", "data", inputData);
Path sourceBaseLocation = new Path("s3://source/data");
Path replicaLocation = new Path("s3://target/data2");
List<Path> sourceSubLocations = new ArrayList<>();
S3S3Copier s3s3Copier = newS3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation);
Metrics metrics = s3s3Copier.copy();
assertThat(metrics.getBytesReplicated(), is(7L));
assertThat(metrics.getMetrics().get(S3S3CopierMetrics.Metrics.TOTAL_BYTES_TO_REPLICATE.name()), is(7L));
S3Object object = client.getObject("target", "data2");
String data = IOUtils.toString(object.getObjectContent());
assertThat(data, is("bar foo"));
assertThat(registry.getGauges().containsKey(RunningMetrics.S3S3_CP_BYTES_REPLICATED.name()), is(true));
}
@Test
void getInputStream_existingObject_returnsInputStreamWithContent() throws Exception {
// Arrange
AmazonS3 amazonS3 = mock(AmazonS3.class);
ObjectMetadata objectMetadata = mock(ObjectMetadata.class);
when(amazonS3.getObjectMetadata(any(GetObjectMetadataRequest.class)))
.thenReturn(objectMetadata);
S3Object s3Object = new S3Object();
s3Object.setObjectMetadata(objectMetadata);
s3Object.setObjectContent(new ByteArrayInputStream(new byte[] { 42 }));
when(amazonS3.getObject(any(GetObjectRequest.class))).thenReturn(s3Object);
// Act
SimpleStorageResource simpleStorageResource = new SimpleStorageResource(amazonS3,
"bucket", "object", new SyncTaskExecutor());
// Assert
assertThat(simpleStorageResource.exists()).isTrue();
assertThat(simpleStorageResource.getInputStream().read()).isEqualTo(42);
}
/**
* S3 block read would be achieved through the AmazonS3 client. Following are the steps to achieve:
* (1) Create the objectRequest from bucketName and filePath.
* (2) Set the range to the above created objectRequest.
* (3) Get the object portion through AmazonS3 client API.
* (4) Get the object content from the above object portion.
* @return the block entity
* @throws IOException
*/
@Override
protected Entity readEntity() throws IOException
{
entity.clear();
GetObjectRequest rangeObjectRequest = new GetObjectRequest(
bucketName, filePath);
rangeObjectRequest.setRange(offset, blockMetadata.getLength() - 1);
S3Object objectPortion = s3Client.getObject(rangeObjectRequest);
S3ObjectInputStream wrappedStream = objectPortion.getObjectContent();
byte[] record = ByteStreams.toByteArray(wrappedStream);
entity.setUsedBytes(record.length);
entity.setRecord(record);
wrappedStream.close();
return entity;
}
@Test
public void testAtomicMpuAbort() throws Exception {
String key = "testAtomicMpuAbort";
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(BYTE_SOURCE.size());
client.putObject(containerName, key, BYTE_SOURCE.openStream(),
metadata);
InitiateMultipartUploadRequest initRequest =
new InitiateMultipartUploadRequest(containerName, key);
InitiateMultipartUploadResult initResponse =
client.initiateMultipartUpload(initRequest);
String uploadId = initResponse.getUploadId();
client.abortMultipartUpload(new AbortMultipartUploadRequest(
containerName, key, uploadId));
S3Object object = client.getObject(containerName, key);
assertThat(object.getObjectMetadata().getContentLength()).isEqualTo(
BYTE_SOURCE.size());
try (InputStream actual = object.getObjectContent();
InputStream expected = BYTE_SOURCE.openStream()) {
assertThat(actual).hasContentEqualTo(expected);
}
}
/***
* Download a S3 object to local directory
*
* @param s3ObjectSummary S3 object summary for the object to download
* @param targetDirectory Local target directory to download the object to
* @throws IOException If any errors were encountered in downloading the object
*/
public void downloadS3Object(S3ObjectSummary s3ObjectSummary,
String targetDirectory)
throws IOException {
final AmazonS3 amazonS3 = getS3Client();
final GetObjectRequest getObjectRequest = new GetObjectRequest(
s3ObjectSummary.getBucketName(),
s3ObjectSummary.getKey());
final S3Object s3Object = amazonS3.getObject(getObjectRequest);
final String targetFile = StringUtils.removeEnd(targetDirectory, File.separator) + File.separator + s3Object.getKey();
FileUtils.copyInputStreamToFile(s3Object.getObjectContent(), new File(targetFile));
LOGGER.info("S3 object downloaded to file: " + targetFile);
}
/**
* Reads a spilled block.
*
* @param spillLocation The location to read the spilled Block from.
* @param key The encryption key to use when reading the spilled Block.
* @param schema The Schema to use when deserializing the spilled Block.
* @return The Block stored at the spill location.
*/
protected Block read(S3SpillLocation spillLocation, EncryptionKey key, Schema schema)
{
try {
logger.debug("write: Started reading block from S3");
S3Object fullObject = amazonS3.getObject(spillLocation.getBucket(), spillLocation.getKey());
logger.debug("write: Completed reading block from S3");
Block block = blockCrypto.decrypt(key, ByteStreams.toByteArray(fullObject.getObjectContent()), schema);
logger.debug("write: Completed decrypting block of size.");
return block;
}
catch (IOException ex) {
throw new RuntimeException(ex);
}
}
/**
* Connect to S3 directory to look for new or updated table definitions and then
* update the map.
*/
private void updateTablesFromS3()
{
long now = System.currentTimeMillis();
AmazonS3Client s3client = clientManager.getS3Client();
for (S3ObjectSummary summary : getObjectSummaries()) {
if (!descriptors.containsKey(summary.getKey()) || summary.getLastModified().getTime() >= lastCheck) {
// New or updated file, so we must read from AWS
if (summary.getKey().endsWith("/")) {
continue;
}
log.info("Getting : %s - %s", summary.getBucketName(), summary.getKey());
S3Object object = s3client.getObject(new GetObjectRequest(summary.getBucketName(), summary.getKey()));
try (BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent(), UTF_8))) {
KinesisStreamDescription table = streamDescriptionCodec.fromJson(CharStreams.toString(reader));
descriptors.put(summary.getKey(), table);
log.info("Put table description into the map from %s", summary.getKey());
}
catch (IOException iox) {
log.error("Problem reading input stream from object.", iox);
throwIfUnchecked(iox);
throw new RuntimeException(iox);
}
}
}
log.info("Completed updating table definitions from S3.");
lastCheck = now;
}
@Override
public S3Object getObject(GetObjectRequest getObjectRequest)
{
if (getObjectHttpCode != HTTP_OK) {
AmazonS3Exception exception = new AmazonS3Exception("Failing getObject call with " + getObjectHttpCode);
exception.setStatusCode(getObjectHttpCode);
throw exception;
}
return null;
}
private static S3Object buildS3Object( Bucket bucket, String key, String dataString ) {
S3Object s3Object = new S3Object();
s3Object.setKey( key );
s3Object.setBucketName( bucket.getName() );
s3Object.setObjectContent( new ByteArrayInputStream( dataString.getBytes() ) );
return s3Object;
}
@Test
public void testCopyObject() throws Exception {
String newName = UUID.randomUUID().toString();
AmazonS3ExecutorConfig config = getConfig();
config.taskConfig.taskType = TaskType.COPY_OBJECT;
config.taskConfig.copyTargetLocation = newName;
AmazonS3Executor executor = new AmazonS3Executor(config);
TargetRunner runner = new TargetRunner.Builder(AmazonS3DExecutor.class, executor)
.build();
runner.runInit();
try {
s3client.putObject(new PutObjectRequest(BUCKET_NAME, objectName, IOUtils.toInputStream("content"), new ObjectMetadata()));
runner.runWrite(ImmutableList.of(getTestRecord()));
S3Object object = s3client.getObject(BUCKET_NAME, newName);
S3ObjectInputStream objectContent = object.getObjectContent();
List<String> stringList = IOUtils.readLines(objectContent);
Assert.assertEquals(1, stringList.size());
Assert.assertEquals("content", stringList.get(0));
Assert.assertTrue(s3client.doesObjectExist(BUCKET_NAME, objectName));
Assert.assertEquals(1, runner.getEventRecords().size());
assertEvent(runner.getEventRecords().get(0), newName);
} finally {
runner.runDestroy();
}
}
/**
* A simple set. Reads S3 object and put into the Set.
* @param name String. The name of the object.
* @param object S3Object. The S3 object to read.
* @throws Exception on S3 or I/O options.
*/
public SimpleSet(String name, S3Object object) throws Exception {
InputStream objectData = object.getObjectContent();
BufferedReader br=new BufferedReader(new InputStreamReader(objectData));
message = "Initialize Simple Membership: " + object.getBucketName() + " as " + name;
makeSet(br);
symbols.put(name, this);
}
@Test
public void testGetS3Object() throws Exception {
when( s3ServiceMock.getObject( anyString(), anyString() ) ).thenReturn( new S3Object() );
S3FileObject s3FileObject = new S3FileObject( filename, fileSystemSpy );
S3Object s3Object = s3FileObject.getS3Object();
assertNotNull( s3Object );
}
private cfData readToMemory(AmazonS3 s3Client, String bucket, String key, String aes256key, int retry, int retryseconds) throws Exception {
// Let us run around the number of attempts
int attempts = 0;
while ( attempts < retry ){
try{
GetObjectRequest gor = new GetObjectRequest(bucket, key);
if ( aes256key != null && !aes256key.isEmpty() )
gor.setSSECustomerKey( new SSECustomerKey(aes256key) );
S3Object s3object = s3Client.getObject(gor);
String contentType = s3object.getObjectMetadata().getContentType();
ByteArrayOutputStream baos = new ByteArrayOutputStream( 32000 );
StreamUtil.copyTo(s3object.getObjectContent(), baos, false );
if ( contentType.indexOf("text") != -1 || contentType.indexOf("javascript") != -1 ){
return new cfStringData( baos.toString() );
}else{
return new cfBinaryData( baos.toByteArray() );
}
}catch(Exception e){
cfEngine.log("Failed: AmazonS3Read(bucket=" + bucket + "; key=" + key + "; attempt=" + (attempts+1) + "; exception=" + e.getMessage() + ")");
attempts++;
if ( attempts == retry )
throw e;
else
Thread.sleep( retryseconds*1000 );
}
}
return null; // should never
}
@Override
public InputStream _readFile(String folder, String filename) throws FileNotFoundException {
final S3Object file = s3Client.getObject(bucketName, filename);
InputStream is = file.getObjectContent();
FilterInputStream fos = new FilterInputStream(is) {
@Override
public void close() throws IOException {
super.close();
file.close();
}
};
return fos;
}