下面列出了com.amazonaws.services.s3.model.Tag#com.amazonaws.services.s3.model.AmazonS3Exception 实例代码，或者点击链接到github查看源代码，也可以在右侧发表评论。
@SuppressWarnings({"ResultOfMethodCallIgnored", "OverlyStrongTypeCast", "ConstantConditions"})
@Test
public void testReadRetryCounters()
throws Exception
{
try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
int maxRetries = 2;
MockAmazonS3 s3 = new MockAmazonS3();
s3.setGetObjectHttpErrorCode(HTTP_INTERNAL_ERROR);
Configuration configuration = new Configuration(false);
configuration.set(S3_MAX_BACKOFF_TIME, "1ms");
configuration.set(S3_MAX_RETRY_TIME, "5s");
configuration.setInt(S3_MAX_CLIENT_RETRIES, maxRetries);
fs.initialize(new URI("s3n://test-bucket/"), configuration);
fs.setS3Client(s3);
try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) {
inputStream.read();
}
catch (Throwable expected) {
assertInstanceOf(expected, AmazonS3Exception.class);
assertEquals(((AmazonS3Exception) expected).getStatusCode(), HTTP_INTERNAL_ERROR);
assertEquals(PrestoS3FileSystem.getFileSystemStats().getReadRetries().getTotalCount(), maxRetries);
assertEquals(PrestoS3FileSystem.getFileSystemStats().getGetObjectRetries().getTotalCount(), (maxRetries + 1L) * maxRetries);
}
}
}
private void handleAttachExceptionFallback( String bucket, String keyWithDelimiter, AmazonS3Exception exception )
throws FileSystemException {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
.withBucketName( bucket )
.withPrefix( keyWithDelimiter )
.withDelimiter( DELIMITER );
ObjectListing ol = fileSystem.getS3Client().listObjects( listObjectsRequest );
if ( !( ol.getCommonPrefixes().isEmpty() && ol.getObjectSummaries().isEmpty() ) ) {
injectType( FileType.FOLDER );
} else {
//Folders don't really exist - they will generate a "NoSuchKey" exception
// confirms key doesn't exist but connection okay
String errorCode = exception.getErrorCode();
if ( !errorCode.equals( "NoSuchKey" ) ) {
// bubbling up other connection errors
logger.error( "Could not get information on " + getQualifiedName(),
exception ); // make sure this gets printed for the user
throw new FileSystemException( "vfs.provider/get-type.error", getQualifiedName(), exception );
}
}
}
@Test
public void testAwsV4SignatureBadIdentity() throws Exception {
client = AmazonS3ClientBuilder.standard()
.withCredentials(new AWSStaticCredentialsProvider(
new BasicAWSCredentials(
"bad-access-key", awsCreds.getAWSSecretKey())))
.withEndpointConfiguration(s3EndpointConfig)
.build();
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(BYTE_SOURCE.size());
try {
client.putObject(containerName, "foo",
BYTE_SOURCE.openStream(), metadata);
Fail.failBecauseExceptionWasNotThrown(AmazonS3Exception.class);
} catch (AmazonS3Exception e) {
assertThat(e.getErrorCode()).isEqualTo("InvalidAccessKeyId");
}
}
@Test
public void testHandleAttachExceptionEmptyFolder() throws FileSystemException {
String testKey = BUCKET_NAME + "/" + origKey;
String testBucket = "badBucketName";
AmazonS3Exception exception = new AmazonS3Exception( "NoSuchKey" );
exception.setErrorCode( "NoSuchKey" );
//test the case where the folder exists and contains things; no exception should be thrown
when( s3ServiceMock.getObject( BUCKET_NAME, origKey + "/" ) ).thenThrow( exception );
childObjectListing = mock( ObjectListing.class );
when( childObjectListing.getObjectSummaries() ).thenReturn( new ArrayList<>() );
when( childObjectListing.getCommonPrefixes() ).thenReturn( new ArrayList<>() );
when( s3ServiceMock.listObjects( any( ListObjectsRequest.class ) ) ).thenReturn( childObjectListing );
try {
s3FileObjectFileSpy.handleAttachException( testKey, testBucket );
} catch ( FileSystemException e ) {
fail( "Caught exception " + e.getMessage() );
}
assertEquals( FileType.IMAGINARY, s3FileObjectFileSpy.getType() );
}
@Override
public ObjectStorageMetadataResponse getObjectStorageMetadata(ObjectStorageMetadataRequest request) {
AwsCredentialView awsCredentialView = new AwsCredentialView(request.getCredential());
try {
AmazonS3 s3Client = awsClient.createS3Client(awsCredentialView);
String bucketLocation = fixBucketLocation(s3Client.getBucketLocation(request.getObjectStoragePath()));
return ObjectStorageMetadataResponse.builder()
.withRegion(bucketLocation)
.withStatus(ResponseStatus.OK)
.build();
} catch (AmazonS3Exception e) {
// HACK let's assume that if the user gets back 403 Access Denied it is because s/he does not have the s3:GetBucketLocation permission.
// It is also true though that if the bucket indeed exists but it is in another account or otherwise denied from the requesting user,
// the same error code will be returned. However, this hack is mainly for QAAS.
if (e.getStatusCode() != ACCESS_DENIED_ERROR_CODE) {
throw new CloudConnectorException(String.format("Cannot get object storage location for %s. "
+ "Provider error message: %s", request.getObjectStoragePath(), e.getErrorMessage()), e);
}
return ObjectStorageMetadataResponse.builder()
.withStatus(ResponseStatus.ACCESS_DENIED)
.build();
}
}
@Test
public void testHandleAttachExceptionEmptyFolder() throws FileSystemException {
AmazonS3Exception exception = new AmazonS3Exception( "NoSuchKey" );
exception.setErrorCode( "NoSuchKey" );
//test the case where the folder exists and contains things; no exception should be thrown
when( s3ServiceMock.getObject( BUCKET_NAME, origKey ) ).thenThrow( exception );
when( s3ServiceMock.getObject( BUCKET_NAME, origKey + "/" ) ).thenThrow( exception );
childObjectListing = mock( ObjectListing.class );
when( childObjectListing.getObjectSummaries() ).thenReturn( new ArrayList<>() );
when( childObjectListing.getCommonPrefixes() ).thenReturn( new ArrayList<>() );
when( s3ServiceMock.listObjects( any( ListObjectsRequest.class ) ) ).thenReturn( childObjectListing );
try {
s3FileObjectFileSpy.doAttach();
} catch ( Exception e ) {
fail( "Caught exception " + e.getMessage() );
}
assertEquals( FileType.IMAGINARY, s3FileObjectFileSpy.getType() );
}
@Test
public void testBlobRemove() throws Exception {
String blobName = "blob";
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(BYTE_SOURCE.size());
client.putObject(containerName, blobName, BYTE_SOURCE.openStream(),
metadata);
assertThat(client.getObjectMetadata(containerName, blobName))
.isNotNull();
client.deleteObject(containerName, blobName);
try {
client.getObjectMetadata(containerName, blobName);
Fail.failBecauseExceptionWasNotThrown(AmazonS3Exception.class);
} catch (AmazonS3Exception e) {
assertThat(e.getErrorCode()).isEqualTo("404 Not Found");
}
client.deleteObject(containerName, blobName);
}
@Override
public void downloadFile(final Path localPath, final RemoteObjectReference objectReference) throws Exception {
final GetObjectRequest getObjectRequest = new GetObjectRequest(request.storageLocation.bucket, objectReference.canonicalPath);
Files.createDirectories(localPath.getParent());
final Optional<AmazonClientException> exception = ofNullable(transferManager.download(getObjectRequest,
localPath.toFile(),
new DownloadProgressListener(objectReference)).waitForException());
if (exception.isPresent()) {
if (exception.get() instanceof AmazonS3Exception && ((AmazonS3Exception) exception.get()).getStatusCode() == 404) {
logger.error("Remote object reference {} does not exist.", objectReference);
}
throw exception.get();
}
}
@Test
public void testPartNumberMarker() throws Exception {
String blobName = "foo";
InitiateMultipartUploadResult result = client.initiateMultipartUpload(
new InitiateMultipartUploadRequest(containerName, blobName));
ListPartsRequest request = new ListPartsRequest(containerName,
blobName, result.getUploadId());
client.listParts(request.withPartNumberMarker(0));
try {
client.listParts(request.withPartNumberMarker(1));
Fail.failBecauseExceptionWasNotThrown(AmazonS3Exception.class);
} catch (AmazonS3Exception e) {
assertThat(e.getErrorCode()).isEqualTo("NotImplemented");
}
}
public static boolean checkACLAccess(AmazonS3Client awsS3Client, String s3BucketName, String accessType) {
logger.info("inside the checkACLAccess method");
Boolean openAcces = false;
AccessControlList bucketAcl;
List<Permission> permissionList = null;
try {
bucketAcl = awsS3Client.getBucketAcl(s3BucketName);
List<Grant> grants = bucketAcl.getGrantsAsList();
// Check grants has which permission
if (!CollectionUtils.isNullOrEmpty(grants)) {
permissionList = checkAnyGrantHasOpenToReadOrWriteAccess(grants, accessType);
if (!CollectionUtils.isNullOrEmpty(permissionList)) {
openAcces = true;
}
}
} catch (AmazonS3Exception s3Exception) {
logger.error("error : ", s3Exception);
throw new RuleExecutionFailedExeption(s3Exception.getMessage());
}
return openAcces;
}
@Override
public Object invoke(MethodInvocation invocation) throws Throwable {
try {
return invocation.proceed();
}
catch (AmazonS3Exception e) {
if (301 == e.getStatusCode()) {
AmazonS3 redirectClient = buildAmazonS3ForRedirectLocation(
this.amazonS3, e);
return ReflectionUtils.invokeMethod(invocation.getMethod(),
redirectClient, invocation.getArguments());
}
else {
throw e;
}
}
}
@Override
protected ContainerHolder getUnknownContainer(String containerName) throws IOException {
// Per docs, if invalid security credentials are used to execute
// AmazonS3#doesBucketExist method, the client is not able to distinguish
// between bucket permission errors and invalid credential errors, and the
// method could return an incorrect result.
// Coordinator node gets the new bucket information by overall refresh in the containerMap
// This method is implemented only for the cases when executor is falling behind.
boolean containerFound = false;
try {
// getBucketLocation ensures that given user account has permissions for the bucket.
containerFound = s3.doesBucketExistV2(containerName) &&
s3.getBucketAcl(containerName).getGrantsAsList().stream()
.anyMatch(g -> g.getGrantee().getIdentifier().equals(s3.getS3AccountOwner().getId()));
} catch (AmazonS3Exception e) {
if (e.getMessage().contains("Access Denied")) {
// Ignorable because user doesn't have permissions. We'll omit this case.
logger.info("Ignoring \"" + containerName + "\" because of logged in AWS account doesn't have access rights on this bucket." + e.getMessage());
}
logger.error("Error while looking up for the unknown container " + containerName, e);
}
return containerFound ? new BucketCreator(getConf(), containerName).toContainerHolder() : null;
}
public static <T> T withClientCorrectingRegion(@NotNull final AmazonS3 s3Client,
@NotNull final Map<String, String> settings,
@NotNull final WithS3<T, AmazonS3Exception> withCorrectedClient) {
try {
return withCorrectedClient.run(s3Client);
} catch (AmazonS3Exception awsException) {
final String correctRegion = extractCorrectedRegion(awsException);
if (correctRegion != null) {
LOGGER.debug("Running operation with corrected S3 region [" + correctRegion + "]", awsException);
final HashMap<String, String> correctedSettings = new HashMap<>(settings);
correctedSettings.put(REGION_NAME_PARAM, correctRegion);
return withS3Client(correctedSettings, withCorrectedClient);
} else {
throw awsException;
}
}
}
@Test
public void testAwsV4SignatureBadCredential() throws Exception {
client = AmazonS3ClientBuilder.standard()
.withCredentials(new AWSStaticCredentialsProvider(
new BasicAWSCredentials(
awsCreds.getAWSAccessKeyId(),
"bad-secret-key")))
.withEndpointConfiguration(s3EndpointConfig)
.build();
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(BYTE_SOURCE.size());
try {
client.putObject(containerName, "foo",
BYTE_SOURCE.openStream(), metadata);
Fail.failBecauseExceptionWasNotThrown(AmazonS3Exception.class);
} catch (AmazonS3Exception e) {
assertThat(e.getErrorCode()).isEqualTo("SignatureDoesNotMatch");
}
}
public String getRegionForBucket(String bucket) {
// Just querying for the location for a bucket can be done with the local client
AmazonS3 client = getLocalS3Client();
try {
String region = client.getBucketLocation(bucket);
if ("US".equals(region)) {
// GetBucketLocation requests return null for us-east-1 which the SDK then replaces with "US".
// So change it to the actual region.
region = "us-east-1";
}
return region;
} catch (AmazonS3Exception e) {
if (e.getStatusCode() == Response.Status.NOT_FOUND.getStatusCode()) {
// If the bucket doesn't exist then return null
return null;
}
throw e;
}
}
@Override
protected boolean writeScanCompleteFile(URI fileUri, byte[] contents)
throws IOException {
String bucket = fileUri.getHost();
String key = getKeyFromPath(fileUri);
try {
// The following will throw an exception unless the file already exists
_amazonS3.getObjectMetadata(bucket, key);
return false;
} catch (AmazonS3Exception e) {
if (e.getStatusCode() != Response.Status.NOT_FOUND.getStatusCode()) {
// Expected case is not found, meaning the file does not exist
// All other cases are some unexpected error
throw new IOException(e);
}
}
uploadContents(bucket, key, contents);
return true;
}
@Override
public S3Object getObject(final GetObjectRequest request) throws AmazonClientException {
assertThat(request.getBucketName(), equalTo(bucket));
final String blobName = request.getKey();
final byte[] content = blobs.get(blobName);
if (content == null) {
AmazonS3Exception exception = new AmazonS3Exception("[" + blobName + "] does not exist.");
exception.setStatusCode(404);
throw exception;
}
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(content.length);
S3Object s3Object = new S3Object();
s3Object.setObjectContent(new S3ObjectInputStream(new ByteArrayInputStream(content), null, false));
s3Object.setKey(blobName);
s3Object.setObjectMetadata(metadata);
return s3Object;
}
@VisibleForTesting
MatchResult matchNonGlobPath(S3ResourceId path) {
ObjectMetadata s3Metadata;
try {
s3Metadata = getObjectMetadata(path);
} catch (AmazonClientException e) {
if (e instanceof AmazonS3Exception && ((AmazonS3Exception) e).getStatusCode() == 404) {
return MatchResult.create(MatchResult.Status.NOT_FOUND, new FileNotFoundException());
}
return MatchResult.create(MatchResult.Status.ERROR, new IOException(e));
}
return MatchResult.create(
MatchResult.Status.OK,
ImmutableList.of(
createBeamMetadata(
path.withSize(s3Metadata.getContentLength())
.withLastModified(s3Metadata.getLastModified()),
Strings.nullToEmpty(s3Metadata.getContentEncoding()))));
}
@Test
public void matchNonGlobNotFound() {
S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());
S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/testdirectory/nonexistentfile");
AmazonS3Exception exception = new AmazonS3Exception("mock exception");
exception.setStatusCode(404);
when(s3FileSystem
.getAmazonS3Client()
.getObjectMetadata(
argThat(
new GetObjectMetadataRequestMatcher(
new GetObjectMetadataRequest(path.getBucket(), path.getKey())))))
.thenThrow(exception);
MatchResult result = s3FileSystem.matchNonGlobPath(path);
assertThat(
result,
MatchResultMatcher.create(MatchResult.Status.NOT_FOUND, new FileNotFoundException()));
}
@Test
public void matchNonGlobForbidden() {
S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());
AmazonS3Exception exception = new AmazonS3Exception("mock exception");
exception.setStatusCode(403);
S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/testdirectory/keyname");
when(s3FileSystem
.getAmazonS3Client()
.getObjectMetadata(
argThat(
new GetObjectMetadataRequestMatcher(
new GetObjectMetadataRequest(path.getBucket(), path.getKey())))))
.thenThrow(exception);
assertThat(
s3FileSystem.matchNonGlobPath(path),
MatchResultMatcher.create(MatchResult.Status.ERROR, new IOException(exception)));
}
private FileStatus getFileStatusKeyBased(String key, Path path) throws AmazonS3Exception {
LOG.trace("internal method - get file status by key {}, path {}", key, path);
FileStatus cachedFS = memoryCache.getFileStatus(path.toString());
if (cachedFS != null) {
return cachedFS;
}
ObjectMetadata meta = mClient.getObjectMetadata(mBucket, key);
String sparkOrigin = meta.getUserMetaDataOf("data-origin");
boolean stocatorCreated = false;
if (sparkOrigin != null) {
String tmp = (String) sparkOrigin;
if (tmp.equals("stocator")) {
stocatorCreated = true;
}
}
mCachedSparkOriginated.put(key, Boolean.valueOf(stocatorCreated));
FileStatus fs = createFileStatus(meta.getContentLength(), key, meta.getLastModified(), path);
LOG.trace("getFileStatusKeyBased: key {} fs.path {}", key, fs.getPath());
memoryCache.putFileStatus(path.toString(), fs);
return fs;
}
public Reader reader() throws IOException {
// Will never return null, but an IOException if not found
try {
final InputStream inputStream = this.resource.getInputStream();
if (!StringUtils.isEmptyOrWhitespace(this.characterEncoding)) {
return new BufferedReader(new InputStreamReader(new BufferedInputStream(inputStream), this.characterEncoding));
}
return new BufferedReader(new InputStreamReader(new BufferedInputStream(inputStream)));
} catch (AmazonS3Exception e) {
if (e.getStatusCode() == 404) {
throw new IOException(e);
}
throw e;
}
}
private S3Object getConfigObject() throws Exception
{
try
{
S3Object object = s3Client.getObject(arguments.getBucket(), arguments.getKey());
if ( object.getObjectMetadata().getContentLength() > 0 )
{
return object;
}
}
catch ( AmazonS3Exception e )
{
if ( !isNotFoundError(e) )
{
throw e;
}
}
return null;
}
@Override
public void prepareStorageLocation(final BlobStoreConfiguration blobStoreConfiguration) {
String bucket = getConfiguredBucket(blobStoreConfiguration);
checkPermissions(getConfiguredBucket(blobStoreConfiguration));
if (!s3.doesBucketExistV2(bucket)) {
try {
s3.createBucket(bucket);
}
catch (AmazonS3Exception e) {
if (ACCESS_DENIED_CODE.equals(e.getErrorCode())) {
log.debug("Error creating bucket {}", bucket, e);
throw insufficientCreatePermissionsError();
}
log.info("Error creating bucket {}", bucket, e);
throw unexpectedError("creating bucket");
}
setBucketLifecycleConfiguration(s3, blobStoreConfiguration, null);
}
else {
// bucket exists, we should test that the correct lifecycle config is present
BucketLifecycleConfiguration lifecycleConfiguration = s3.getBucketLifecycleConfiguration(bucket);
if (!isExpirationLifecycleConfigurationPresent(lifecycleConfiguration, blobStoreConfiguration)) {
setBucketLifecycleConfiguration(s3, blobStoreConfiguration, lifecycleConfiguration);
}
}
}
@Override
public ObjectMetadata getObjectMetadata(GetObjectMetadataRequest getObjectMetadataRequest)
{
this.getObjectMetadataRequest = getObjectMetadataRequest;
if (getObjectMetadataHttpCode != HTTP_OK) {
AmazonS3Exception exception = new AmazonS3Exception("Failing getObjectMetadata call with " + getObjectMetadataHttpCode);
exception.setStatusCode(getObjectMetadataHttpCode);
throw exception;
}
return null;
}
@Override
public S3Object getObject(GetObjectRequest getObjectRequest)
{
if (getObjectHttpCode != HTTP_OK) {
AmazonS3Exception exception = new AmazonS3Exception("Failing getObject call with " + getObjectHttpCode);
exception.setStatusCode(getObjectHttpCode);
throw exception;
}
return null;
}
protected void putFileWithUserMetadata(String key, File file, Map<String, String> userMetadata) throws AmazonS3Exception, FileNotFoundException {
ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.setUserMetadata(userMetadata);
PutObjectRequest putRequest = new PutObjectRequest(BUCKET_NAME, key, new FileInputStream(file), objectMetadata);
client.putObject(putRequest);
}
/**
* {@inheritDoc}
*/
@Override
public void putFile(
@NotBlank(message = "Source local path cannot be empty.") final String srcLocalPath,
@NotBlank(message = "Destination remote path cannot be empty") final String dstRemotePath
) throws GenieException {
final long start = System.nanoTime();
final Set<Tag> tags = Sets.newHashSet();
try {
log.debug("Called with src path {} and destination path {}", srcLocalPath, dstRemotePath);
final AmazonS3URI s3Uri = getS3Uri(dstRemotePath);
try {
this.s3ClientFactory
.getClient(s3Uri)
.putObject(s3Uri.getBucket(), s3Uri.getKey(), new File(srcLocalPath));
} catch (final AmazonS3Exception ase) {
log.error("Error posting file {} to s3 due to exception {}", dstRemotePath, ase.toString());
throw new GenieServerException("Error uploading file to s3. Filename: " + dstRemotePath, ase);
}
MetricsUtils.addSuccessTags(tags);
} catch (Throwable t) {
MetricsUtils.addFailureTagsWithException(tags, t);
throw t;
} finally {
this.registry.timer(UPLOAD_TIMER_NAME, tags).record(System.nanoTime() - start, TimeUnit.NANOSECONDS);
}
}
@Test
public void testContainerExists() throws Exception {
client.headBucket(new HeadBucketRequest(containerName));
try {
client.headBucket(new HeadBucketRequest(
createRandomContainerName()));
Fail.failBecauseExceptionWasNotThrown(AmazonS3Exception.class);
} catch (AmazonS3Exception e) {
assertThat(e.getErrorCode()).isEqualTo("404 Not Found");
}
}
@Test
public void testHandleAttachException() throws FileSystemException {
String testKey = BUCKET_NAME + "/" + origKey;
String testBucket = "badBucketName";
AmazonS3Exception exception = new AmazonS3Exception( "NoSuchKey" );
//test the case where the folder exists and contains things; no exception should be thrown
when( s3ServiceMock.getObject( BUCKET_NAME, origKey + "/" ) ).thenThrow( exception );
try {
s3FileObjectFileSpy.handleAttachException( testKey, testBucket );
} catch ( FileSystemException e ) {
fail( "Caught exception " + e.getMessage() );
}
assertEquals( FileType.FOLDER, s3FileObjectFileSpy.getType() );
}