com.amazonaws.services.s3.model.GetObjectMetadataRequest#com.amazonaws.services.s3.model.Bucket源码实例Demo

/**
 * Maps a DBInstance into a row in our Apache Arrow response block(s).
 *
 * @param bucket The S3 Bucket to map.
 * @param spiller The BlockSpiller to use when we want to write a matching row to the response.
 * @note The current implementation is rather naive in how it maps fields. It leverages a static
 * list of fields that we'd like to provide and then explicitly filters and converts each field.
 */
private void toRow(Bucket bucket,
        BlockSpiller spiller)
{
    spiller.writeRows((Block block, int row) -> {
        boolean matched = true;
        matched &= block.offerValue("bucket_name", row, bucket.getName());
        matched &= block.offerValue("create_date", row, bucket.getCreationDate());

        Owner owner = bucket.getOwner();
        if (owner != null) {
            matched &= block.offerValue("owner_name", row, bucket.getOwner().getDisplayName());
            matched &= block.offerValue("owner_id", row, bucket.getOwner().getId());
        }

        return matched ? 1 : 0;
    });
}
 

protected List<String> getS3ObjectsFromVirtualFolder( String key, String bucketName ) {
  List<String> childrenList = new ArrayList<>();

  // fix cases where the path doesn't include the final delimiter
  String realKey = key;
  if ( !realKey.endsWith( DELIMITER ) ) {
    realKey += DELIMITER;
  }

  if ( "".equals( key ) && "".equals( bucketName ) ) {
    //Getting buckets in root folder
    List<Bucket> bucketList = fileSystem.getS3Client().listBuckets();
    for ( Bucket bucket : bucketList ) {
      childrenList.add( bucket.getName() + DELIMITER );
    }
  } else {
    getObjectsFromNonRootFolder( key, bucketName, childrenList, realKey );
  }
  return childrenList;
}
 

private static String fetchS3EncryptInfo(Bucket bucket, AmazonS3 s3Client) {
	
	String bucketEncryp = null;
	try{
		GetBucketEncryptionResult buckectEncry = s3Client.getBucketEncryption(bucket.getName());
		if (buckectEncry != null) {
			ServerSideEncryptionConfiguration sseBucketEncryp = buckectEncry.getServerSideEncryptionConfiguration();
			if (sseBucketEncryp != null && sseBucketEncryp.getRules() != null) {
				bucketEncryp = sseBucketEncryp.getRules().get(0).getApplyServerSideEncryptionByDefault()
						.getSSEAlgorithm();
			}
		}
	}catch(Exception e){
		// Exception thrown when there is no bucket encryption available.
	}
	return bucketEncryp;
}
 

/**
 * Fetch S 3 info test test exception.
 *
 * @throws Exception the exception
 */
@SuppressWarnings("static-access")
@Test
public void fetchS3InfoTestTest_Exception() throws Exception {
    
    mockStatic(AmazonS3ClientBuilder.class);
    AmazonS3 amazonS3Client = PowerMockito.mock(AmazonS3.class);
    AmazonS3ClientBuilder amazonRDSClientBuilder = PowerMockito.mock(AmazonS3ClientBuilder.class);
    AWSStaticCredentialsProvider awsStaticCredentialsProvider = PowerMockito.mock(AWSStaticCredentialsProvider.class);
    PowerMockito.whenNew(AWSStaticCredentialsProvider.class).withAnyArguments().thenReturn(awsStaticCredentialsProvider);
    when(amazonRDSClientBuilder.standard()).thenReturn(amazonRDSClientBuilder);
    when(amazonRDSClientBuilder.withCredentials(anyObject())).thenReturn(amazonRDSClientBuilder);
    when(amazonRDSClientBuilder.withRegion(anyString())).thenReturn(amazonRDSClientBuilder);
    when(amazonRDSClientBuilder.build()).thenReturn(amazonS3Client);
    
    List<Bucket> s3buckets = new ArrayList<>();
    Bucket bucket = new Bucket();
    bucket.setName("name");
    s3buckets.add(bucket);
    when(amazonS3Client.listBuckets()).thenReturn(s3buckets);
    
    when(amazonS3Client.getBucketLocation(anyString())).thenThrow(new AmazonServiceException("Error"));
    assertThat(inventoryUtil.fetchS3Info(new BasicSessionCredentials("awsAccessKey", "awsSecretKey", "sessionToken"), 
            "skipRegions", "account","accountName").size(), is(0));
}
 

public synchronized void createS3BucketIfNotExists(String p_bucket_name) {
    _logger.debug("Searching for bucket " + p_bucket_name);
    if (!s3Client.doesBucketExist(p_bucket_name)) {
        Bucket bucket = s3Client.createBucket(p_bucket_name);
       _logger.info("Created bucket: " + bucket.getName());
    } else {
        _logger.debug("Bucket detected. Verifying permissions.");
        try {
            s3Client.getBucketAcl(p_bucket_name);
        } catch (AmazonClientException ex) {
            _logger.warn("Permission check failed. Randomizing.");
            ConfigFacade.set(Configuration.S3_BUCKET_FORMAT, p_bucket_name + "-" + Security.getRandomHash(8));
            _logger.debug("Reiterating with: " + p_bucket_name);
            createS3BucketIfNotExists(getConfiguredBucketName());
        }
    }
}
 

private LambdaConfig() {
    try (InputStream is = getClass().getClassLoader().getResourceAsStream("env.properties")) {
        this.props.load(is);
        this.repository = new FileRepository(new File(System.getProperty("java.io.tmpdir"), "s3"));
    }
    catch (IOException e) {
        throw new IllegalArgumentException(e);
    }
    overwriteWithSystemProperty(ENV_BRANCH);
    overwriteWithSystemProperty(ENV_BUCKET);
    overwriteWithSystemProperty(ENV_GITHUB);

    this.remote = new Remote(Constants.DEFAULT_REMOTE_NAME);
    this.branch = new Branch(props.getProperty(ENV_BRANCH, Constants.MASTER));
    this.authentication = new SecureShellAuthentication(new Bucket(props.getProperty(ENV_BUCKET)), client);
}
 

private LambdaConfig() {
    try (InputStream is = getClass().getClassLoader().getResourceAsStream("env.properties")) {
        this.props.load(is);
        this.repository = new FileRepository(new File(System.getProperty("java.io.tmpdir"), "s3"));
    }
    catch (IOException e) {
        throw new IllegalArgumentException(e);
    }
    overwriteWithSystemProperty(ENV_BRANCH);
    overwriteWithSystemProperty(ENV_BUCKET);
    overwriteWithSystemProperty(ENV_GITHUB);

    this.remote = new Remote(Constants.DEFAULT_REMOTE_NAME);
    this.branch = new Branch(props.getProperty(ENV_BRANCH, Constants.MASTER));
    this.authentication = new SecureShellAuthentication(new Bucket(props.getProperty(ENV_BUCKET)), client);
}
 

public static void main(String[] args) {
    final String USAGE = "\n" +
            "CreateBucket - create an S3 bucket\n\n" +
            "Usage: CreateBucket <bucketname>\n\n" +
            "Where:\n" +
            "  bucketname - the name of the bucket to create.\n\n" +
            "The bucket name must be unique, or an error will result.\n";

    if (args.length < 1) {
        System.out.println(USAGE);
        System.exit(1);
    }

    String bucket_name = args[0];

    System.out.format("\nCreating S3 bucket: %s\n", bucket_name);
    Bucket b = createBucket(bucket_name);
    if (b == null) {
        System.out.println("Error creating bucket!\n");
    } else {
        System.out.println("Done!\n");
    }
}
 

@Test
public void s3TestOverBridgeNetwork() throws IOException {
    AmazonS3 s3 = AmazonS3ClientBuilder
        .standard()
        .withEndpointConfiguration(localstack.getEndpointConfiguration(S3))
        .withCredentials(localstack.getDefaultCredentialsProvider())
        .build();

    final String bucketName = "foo";
    s3.createBucket(bucketName);
    s3.putObject(bucketName, "bar", "baz");

    final List<Bucket> buckets = s3.listBuckets();
    final Optional<Bucket> maybeBucket = buckets.stream().filter(b -> b.getName().equals(bucketName)).findFirst();
    assertTrue("The created bucket is present", maybeBucket.isPresent());
    final Bucket bucket = maybeBucket.get();

    assertEquals("The created bucket has the right name", bucketName, bucket.getName());

    final ObjectListing objectListing = s3.listObjects(bucketName);
    assertEquals("The created bucket has 1 item in it", 1, objectListing.getObjectSummaries().size());

    final S3Object object = s3.getObject(bucketName, "bar");
    final String content = IOUtils.toString(object.getObjectContent(), Charset.forName("UTF-8"));
    assertEquals("The object can be retrieved", "baz", content);
}
 

public Bucket createBucketForInstance(String instanceId, ServiceDefinition service, String planId,
        String organizationGuid, String spaceGuid) {
    String bucketName = getBucketNameForInstance(instanceId);
    logger.info("Creating bucket '{}' for serviceInstanceId '{}'", bucketName, instanceId);
    Bucket bucket = s3.createBucket(bucketName, Region.fromValue(region));

    // TODO allow for additional, custom tagging options
    BucketTaggingConfiguration bucketTaggingConfiguration = new BucketTaggingConfiguration();
    TagSet tagSet = new TagSet();
    tagSet.setTag("serviceInstanceId", instanceId);
    tagSet.setTag("serviceDefinitionId", service.getId());
    tagSet.setTag("planId", planId);
    tagSet.setTag("organizationGuid", organizationGuid);
    tagSet.setTag("spaceGuid", spaceGuid);
    bucketTaggingConfiguration.withTagSets(tagSet);
    s3.setBucketTaggingConfiguration(bucket.getName(), bucketTaggingConfiguration);

    return bucket;
}
 

protected Collection<String> getSiteListFromBucketNames(String bucketNameRegex) {
    List<String> siteNames = new ArrayList<>();
    AmazonS3 client = clientBuilder.getClient();

    List<Bucket> buckets = client.listBuckets();
    if (CollectionUtils.isNotEmpty(buckets)) {
        for (Bucket bucket : buckets) {
            Matcher bucketNameMatcher = Pattern.compile(bucketNameRegex).matcher(bucket.getName());
            if (bucketNameMatcher.matches()) {
                siteNames.add(bucketNameMatcher.group(1));
            }
        }
    }

    return siteNames;
}
 

@Before
public void setup()
{
    logger.info("setUpBefore - enter");

    bucketNames = Arrays.asList("bucket1", "bucket2", "bucket3");
    List<Bucket> buckets = createBuckets(bucketNames);
    AmazonS3 mockS3 = createMockS3(buckets);
    spyVerifier = spy(new SpillLocationVerifier(mockS3));

    logger.info("setUpBefore - exit");
}
 

private List<Bucket> createBuckets(List<String> names)
{
    List<Bucket> buckets = new ArrayList();
    for (String name : names) {
        Bucket bucket = mock(Bucket.class);
        when(bucket.getName()).thenReturn(name);
        buckets.add(bucket);
    }

    return buckets;
}
 

/**
 * Calls DescribeDBInstances on the AWS RDS Client returning all DB Instances that match the supplied predicate and attempting
 * to push down certain predicates (namely queries for specific DB Instance) to EC2.
 *
 * @See TableProvider
 */
@Override
public void readWithConstraint(BlockSpiller spiller, ReadRecordsRequest recordsRequest, QueryStatusChecker queryStatusChecker)
{
    for (Bucket next : amazonS3.listBuckets()) {
        toRow(next, spiller);
    }
}
 

@Override
protected void setUpRead()
{
    when(mockS3.listBuckets()).thenAnswer((InvocationOnMock invocation) -> {
        List<Bucket> values = new ArrayList<>();
        values.add(makeBucket(getIdValue()));
        values.add(makeBucket(getIdValue()));
        values.add(makeBucket("fake-id"));
        return values;
    });
}
 

private Bucket makeBucket(String id)
{
    Bucket bucket = new Bucket();
    bucket.setName(id);
    Owner owner = new Owner();
    owner.setDisplayName("owner_name");
    owner.setId("owner_id");
    bucket.setOwner(owner);
    bucket.setCreationDate(new Date(100_000));
    return bucket;
}
 

private List<Bucket> createBuckets() {
  List<Bucket> buckets = new ArrayList<>();
  buckets.add( new Bucket( "bucket1" ) );
  buckets.add( new Bucket( "bucket2" ) );
  buckets.add( new Bucket( "bucket3" ) );
  buckets.add( new Bucket( "bucket4" ) );
  childBucketNameListComp.addAll(
    buckets.stream().map( bucket -> "/" + bucket.getName() ).collect( Collectors.toList() ) );
  return buckets;
}
 

private List<Bucket> createBuckets() {
  List<Bucket> buckets = new ArrayList<>();
  buckets.add( new Bucket( "bucket1" ) );
  buckets.add( new Bucket( "bucket2" ) );
  buckets.add( new Bucket( "bucket3" ) );
  buckets.add( new Bucket( "bucket4" ) );
  childBucketNameListComp.addAll(
    buckets.stream().map( bucket -> "/" + bucket.getName() ).collect( Collectors.toList() ) );
  return buckets;
}
 

@Test
public void testS3() throws Exception {
    AmazonS3 client = amazonDockerClientsHolder.amazonS3();

    client.createBucket("test-bucket");
    List<Bucket> bucketList = client.listBuckets();

    assertThat(bucketList.size(), is(1));

    File file = File.createTempFile("localstack", "s3");
    file.deleteOnExit();

    try (FileOutputStream stream = new FileOutputStream(file)) {
        String content = "HELLO WORLD!";
        stream.write(content.getBytes());
    }

    PutObjectRequest request = new PutObjectRequest("test-bucket", "testData", file);
    client.putObject(request);

    ObjectListing listing = client.listObjects("test-bucket");
    assertThat(listing.getObjectSummaries().size(), is(1));

    S3Object s3Object = client.getObject("test-bucket", "testData");
    String resultContent = IOUtils.toString(s3Object.getObjectContent());

    assertThat(resultContent, is("HELLO WORLD!"));
}
 

/**
 * Instantiates a new bucket VH.
 *
 * @param bucket the bucket
 * @param location the location
 * @param versionConfig the version config
 * @param tags the tags
 */
public BucketVH(Bucket bucket,String location,BucketVersioningConfiguration versionConfig, List<Tag> tags, String bucketEncryp, boolean websiteConfiguration,BucketLoggingConfiguration bucketLoggingConfiguration){
	this.bucket = bucket;
	this.location = location;
	this.versionStatus = versionConfig==null?"":versionConfig.getStatus();
	this.mfaDelete =  versionConfig==null?null:versionConfig.isMfaDeleteEnabled();
	this.tags = tags;
	this.bucketEncryp = bucketEncryp;
	this.websiteConfiguration = websiteConfiguration;
       this.isLoggingEnabled = bucketLoggingConfiguration==null?null:bucketLoggingConfiguration.isLoggingEnabled();
       this.destinationBucketName = bucketLoggingConfiguration==null?"":bucketLoggingConfiguration.getDestinationBucketName();
       this.logFilePrefix = bucketLoggingConfiguration==null?"":bucketLoggingConfiguration.getLogFilePrefix();

}
 

/**
 * Fetch S 3 info test.
 *
 * @throws Exception the exception
 */
@SuppressWarnings({ "static-access"})
@Test
public void fetchS3InfoTest() throws Exception {
    
    mockStatic(AmazonS3ClientBuilder.class);
    AmazonS3 amazonS3Client = PowerMockito.mock(AmazonS3.class);
    AmazonS3ClientBuilder amazonRDSClientBuilder = PowerMockito.mock(AmazonS3ClientBuilder.class);
    AWSStaticCredentialsProvider awsStaticCredentialsProvider = PowerMockito.mock(AWSStaticCredentialsProvider.class);
    PowerMockito.whenNew(AWSStaticCredentialsProvider.class).withAnyArguments().thenReturn(awsStaticCredentialsProvider);
    when(amazonRDSClientBuilder.standard()).thenReturn(amazonRDSClientBuilder);
    when(amazonRDSClientBuilder.withCredentials(anyObject())).thenReturn(amazonRDSClientBuilder);
    when(amazonRDSClientBuilder.withRegion(anyString())).thenReturn(amazonRDSClientBuilder);
    when(amazonRDSClientBuilder.build()).thenReturn(amazonS3Client);
    
    List<Bucket> s3buckets = new ArrayList<>();
    Bucket bucket = new Bucket();
    bucket.setName("name");
    s3buckets.add(bucket);
    when(amazonS3Client.listBuckets()).thenReturn(s3buckets);
    when(amazonS3Client.getBucketLocation(anyString())).thenReturn("bucketLocation");
    mockStatic(com.amazonaws.services.s3.model.Region.class);
    com.amazonaws.services.s3.model.Region value = null;
    when(com.amazonaws.services.s3.model.Region.fromValue(anyString())).thenReturn(value.US_West);
    when(value.US_West.toAWSRegion()).thenReturn(getRegions().get(0));
    when(amazonS3Client.getBucketVersioningConfiguration(anyString())).thenReturn(new BucketVersioningConfiguration());
    BucketTaggingConfiguration tagConfig = new BucketTaggingConfiguration();
    List<TagSet> tagSets = new ArrayList<>();
    TagSet tagSet = new TagSet();
    tagSet.setTag("key", "value");
    tagSets.add(tagSet);
    tagSets.add(tagSet);
    tagConfig.setTagSets(tagSets);
    when(amazonS3Client.getBucketTaggingConfiguration(anyString())).thenReturn(tagConfig);
    
    assertThat(inventoryUtil.fetchS3Info(new BasicSessionCredentials("awsAccessKey", "awsSecretKey", "sessionToken"), 
            "skipRegions", "account","accountName").size(), is(1));
}
 

@Override
public S3Bucket createBucket(String bucketName) {
	S3Bucket bucket = new S3Bucket();
	Bucket s3Bucket = s3Client.createBucket(bucketName);
	bucket.setName(s3Bucket.getName());
	bucket.setOwner(new Owner(s3Bucket.getOwner().getId(), s3Bucket.getOwner().getDisplayName()));
	bucket.setCreationDate(s3Bucket.getCreationDate());
	log.info("Created s3 bucket: {}", bucketName);
	return bucket;
}
 

public SecureShellAuthentication(Bucket bucket, AmazonS3 client) {
    factory = new JschConfigSessionFactory() {

        @Override
        public synchronized RemoteSession getSession(URIish uri, CredentialsProvider credentialsProvider, FS fs, int tms) throws TransportException {
            // Do not check for default ssh user config
            fs.setUserHome(null);
            return super.getSession(uri, credentialsProvider, fs, tms);
        }

        @Override
        protected void configure(OpenSshConfig.Host host, Session session) {
            session.setConfig("HashKnownHosts", "no");
            if ("localhost".equalsIgnoreCase(host.getHostName())) {
                session.setConfig("StrictHostKeyChecking", "no");
            }
        }

        @Override
        protected void configureJSch(JSch jsch) {
            S3Object file;
            file = client.getObject(bucket.getName(), ".ssh/known_hosts");
            try (InputStream is = file.getObjectContent()) {
                jsch.setKnownHosts(is);
            } catch (IOException | JSchException e) {
                throw new IllegalArgumentException("Missing known hosts file on s3: .ssh/known_hosts", e);
            }
            file = client.getObject(bucket.getName(), ".ssh/id_rsa");
            try (InputStream is = file.getObjectContent()) {
                jsch.addIdentity("git", IOUtils.toByteArray(is), null, new byte[0]);
            } catch (IOException | JSchException e) {
                throw new IllegalArgumentException("Missing key file on s3: .ssh/id_rsa", e);
            }
        }
    };
}
 

public RepositoryS3(Bucket bucket, Repository repository, AmazonS3 s3, Branch branch) {
    this.s3 = s3;
    this.bucket = bucket;
    this.repository = repository;
    this.branch = branch;
    this.uri = new URIish().setScheme("amazon-s3").setHost(bucket.getName()).setPath(Constants.DOT_GIT);
}
 

@Override
protected Stream<ContainerCreator> getContainerCreators() throws IOException {
  Stream<String> buckets = getBucketNamesFromConfigurationProperty(S3StoragePlugin.EXTERNAL_BUCKETS);
  if (!NONE_PROVIDER.equals(getConf().get(Constants.AWS_CREDENTIALS_PROVIDER))) {
    if (!useWhitelistedBuckets) {
      // if we have authentication to access S3, add in owner buckets.
      buckets = Stream.concat(buckets, s3.listBuckets().stream().map(Bucket::getName));
    } else {
      // Only add the buckets provided in the configuration.
      buckets = Stream.concat(buckets, getBucketNamesFromConfigurationProperty(S3StoragePlugin.WHITELISTED_BUCKETS));
    }
  }
  return buckets.distinct() // Remove duplicate bucket names.
      .map(input -> new BucketCreator(getConf(), input));
}
 

@Override
public Content getContent(final AmazonS3 s3Client, final Map<String, String> parameters) {
  AWSCommonParams.validate(parameters, true);
  final Element bucketsElement = new Element("buckets");
  for (Bucket bucket : withClientCorrectingRegion(s3Client, new HashMap<>(parameters), AmazonS3::listBuckets)) {
    final Element bucketElement = new Element("bucket");
    final String bucketName = bucket.getName();
    bucketElement.setText(bucketName);
    bucketsElement.addContent(bucketElement);
  }
  return bucketsElement;
}
 

public SecureShellAuthentication(Bucket bucket, AmazonS3 client) {
    factory = new JschConfigSessionFactory() {

        @Override
        public synchronized RemoteSession getSession(URIish uri, CredentialsProvider credentialsProvider, FS fs, int tms) throws TransportException {
            // Do not check for default ssh user config
            fs.setUserHome(null);
            return super.getSession(uri, credentialsProvider, fs, tms);
        }

        @Override
        protected void configure(OpenSshConfig.Host host, Session session) {
            session.setConfig("HashKnownHosts", "no");
            if ("localhost".equalsIgnoreCase(host.getHostName())) {
                session.setConfig("StrictHostKeyChecking", "no");
            }
        }

        @Override
        protected void configureJSch(JSch jsch) {
            S3Object file;
            file = client.getObject(bucket.getName(), ".ssh/known_hosts");
            try (InputStream is = file.getObjectContent()) {
                jsch.setKnownHosts(is);
            } catch (IOException | JSchException e) {
                throw new IllegalArgumentException("Missing known hosts file on s3: .ssh/known_hosts", e);
            }
            file = client.getObject(bucket.getName(), ".ssh/id_rsa");
            try (InputStream is = file.getObjectContent()) {
                jsch.addIdentity("git", IOUtils.toByteArray(is), null, new byte[0]);
            } catch (IOException | JSchException e) {
                throw new IllegalArgumentException("Missing key file on s3: .ssh/id_rsa", e);
            }
        }
    };
}
 

public RepositoryS3(Bucket bucket, Repository repository, AmazonS3 s3, Branch branch) {
    this.s3 = s3;
    this.bucket = bucket;
    this.repository = repository;
    this.branch = branch;
    this.uri = new URIish().setScheme("amazon-s3").setHost(bucket.getName()).setPath(Constants.DOT_GIT);
}
 

public static Bucket getBucket(String bucket_name) {
    final AmazonS3 s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
    Bucket named_bucket = null;
    List<Bucket> buckets = s3.listBuckets();
    for (Bucket b : buckets) {
        if (b.getName().equals(bucket_name)) {
            named_bucket = b;
        }
    }
    return named_bucket;
}
 

public static Bucket createBucket(String bucket_name) {
    final AmazonS3 s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
    Bucket b = null;
    if (s3.doesBucketExistV2(bucket_name)) {
        System.out.format("Bucket %s already exists.\n", bucket_name);
        b = getBucket(bucket_name);
    } else {
        try {
            b = s3.createBucket(bucket_name);
        } catch (AmazonS3Exception e) {
            System.err.println(e.getErrorMessage());
        }
    }
    return b;
}