下面列出了com.amazonaws.services.s3.model.ListObjectsRequest#setMarker ( ) 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
@Override
public void move(String folderPath, String newFolderPath, AuthenticationInfo subject) throws IOException {
try {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
.withBucketName(bucketName)
.withPrefix(rootFolder + folderPath + "/");
ObjectListing objectListing;
do {
objectListing = s3client.listObjects(listObjectsRequest);
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
if (objectSummary.getKey().endsWith(".zpln")) {
String noteId = getNoteId(objectSummary.getKey());
String notePath = getNotePath(rootFolder, objectSummary.getKey());
String newNotePath = newFolderPath + notePath.substring(folderPath.length());
move(noteId, notePath, newNotePath, subject);
}
}
listObjectsRequest.setMarker(objectListing.getNextMarker());
} while (objectListing.isTruncated());
} catch (AmazonClientException ace) {
throw new IOException("Fail to move folder: " + folderPath + " to " + newFolderPath + " in S3" , ace);
}
}
public List<String> listCommonPrefixesS3Objects(final String bucketName, final String prefix) {
final List<String> commonPrefixes = Lists.newArrayList();
try {
log.debug("Listing objects in bucket '{}' with prefix '{}'", bucketName, prefix);
final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
.withDelimiter("/")
.withBucketName(bucketName)
.withPrefix(prefix);
ObjectListing objectListing;
do {
objectListing = s3client.listObjects(listObjectsRequest);
objectListing.getCommonPrefixes().stream().map(S3Service::urlDecode).forEach(commonPrefixes::add);
listObjectsRequest.setMarker(objectListing.getNextMarker());
} while (objectListing.isTruncated());
} catch (final AmazonServiceException e) {
log.error("Could not list common prefixes in S3", e);
}
return commonPrefixes;
}
/**
* Call S3 to get the most recent object list.
* <p>
* This is an object list request to AWS in the given "directory".
*/
private List<S3ObjectSummary> getObjectSummaries()
{
AmazonS3Client s3client = clientManager.getS3Client();
AmazonS3URI directoryURI = new AmazonS3URI(bucketUrl.get());
List<S3ObjectSummary> result = new ArrayList<>();
try {
log.info("Getting the listing of objects in the S3 table config directory: bucket %s prefix %s :", directoryURI.getBucket(), directoryURI.getKey());
ListObjectsRequest request = new ListObjectsRequest()
.withBucketName(directoryURI.getBucket())
.withPrefix(directoryURI.getKey() + "/")
.withDelimiter("/")
.withMaxKeys(25);
ObjectListing response;
do {
response = s3client.listObjects(request);
result.addAll(response.getObjectSummaries());
request.setMarker(response.getNextMarker());
}
while (response.isTruncated());
log.info("Completed getting S3 object listing.");
}
catch (AmazonClientException e) {
log.error("Skipping update as faced error fetching table descriptions from S3 " + e.toString());
}
return result;
}
@Override
public List<OldNoteInfo> list(AuthenticationInfo subject) throws IOException {
List<OldNoteInfo> infos = new LinkedList<>();
OldNoteInfo info;
try {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
.withBucketName(bucketName)
.withPrefix(user + "/" + "notebook");
ObjectListing objectListing;
do {
objectListing = s3client.listObjects(listObjectsRequest);
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
if (objectSummary.getKey().endsWith("note.json")) {
info = getNoteInfo(objectSummary.getKey());
if (info != null) {
infos.add(info);
} else {
LOG.debug("Unable to get notebook info for key: " + objectSummary.getKey());
}
}
}
listObjectsRequest.setMarker(objectListing.getNextMarker());
} while (objectListing.isTruncated());
} catch (AmazonClientException ace) {
throw new IOException("Unable to list objects in S3: " + ace, ace);
}
return infos;
}
@Override
public Map<String, NoteInfo> list(AuthenticationInfo subject) throws IOException {
Map<String, NoteInfo> notesInfo = new HashMap<>();
try {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
.withBucketName(bucketName)
.withPrefix(user + "/" + "notebook");
ObjectListing objectListing;
do {
objectListing = s3client.listObjects(listObjectsRequest);
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
if (objectSummary.getKey().endsWith(".zpln")) {
try {
NoteInfo info = getNoteInfo(objectSummary.getKey());
notesInfo.put(info.getId(), info);
} catch (IOException e) {
LOGGER.warn(e.getMessage());
}
}
}
listObjectsRequest.setMarker(objectListing.getNextMarker());
} while (objectListing.isTruncated());
} catch (AmazonClientException ace) {
throw new IOException("Fail to list objects in S3", ace);
}
return notesInfo;
}
public List<String> listS3Objects(final String bucketName, final String prefix) {
final List<String> s3Objects = Lists.newArrayList();
try {
log.debug("Listing objects");
final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
.withDelimiter("/")
.withBucketName(bucketName)
.withPrefix(prefix);
ObjectListing objectListing;
do {
objectListing = s3client.listObjects(listObjectsRequest);
for (final S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
if (objectSummary.getKey().equals(prefix)) {
continue;
}
s3Objects.add(objectSummary.getKey());
}
listObjectsRequest.setMarker(objectListing.getNextMarker());
} while (objectListing.isTruncated());
} catch (final AmazonServiceException e) {
log.error("Error Message: " + e.getMessage());
}
return s3Objects;
}
@Override
public void list(String keyPrefix, FileListing callback)
{
checkArgument(keyPrefix != null, "keyPrefix is null");
String errorMessage = "listing files on bucket " + bucket + " prefix " + keyPrefix;
ListObjectsRequest req = new ListObjectsRequest();
req.setBucketName(bucket);
req.setPrefix(keyPrefix);
ObjectListing listing;
do {
try {
listing = getWithRetry(errorMessage, () -> client.listObjects(req));
}
catch (StorageFileNotFoundException ex) {
throw Throwables.propagate(ex.getCause());
}
callback.accept(Lists.transform(
listing.getObjectSummaries(),
(summary) -> StorageObjectSummary.builder()
.key(summary.getKey())
.contentLength(summary.getSize())
.lastModified(summary.getLastModified().toInstant())
.build()
));
req.setMarker(listing.getNextMarker());
}
while (listing.isTruncated());
}
public static List<S3ObjectSummary> listDirectory(final ClientOptions clientOptions, final String bucketName, final String directory) {
LOGGER.debug(format("Listing S3 directory %1$s in bucket %2$s", directory, bucketName));
List<S3ObjectSummary> objects = new ArrayList<>();
ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
listObjectsRequest.withBucketName(bucketName);
listObjectsRequest.withPrefix(directory);
ObjectListing ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest);
if(ol.isTruncated()) {
do {
objects.addAll(ol.getObjectSummaries());
listObjectsRequest.setMarker(ol.getNextMarker());
ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest);
} while (ol.isTruncated());
}
else {
objects.addAll(ol.getObjectSummaries());
}
if (objects.isEmpty()) {
return emptyList();
}
return unmodifiableList(objects);
}
@Override
public List<S3ObjectSummary> listDirectory(final S3FileTransferRequestParamsDto params, boolean ignoreZeroByteDirectoryMarkers)
{
Assert.isTrue(!isRootKeyPrefix(params.getS3KeyPrefix()), "Listing of S3 objects from root directory is not allowed.");
AmazonS3Client s3Client = getAmazonS3(params);
List<S3ObjectSummary> s3ObjectSummaries = new ArrayList<>();
try
{
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(params.getS3BucketName()).withPrefix(params.getS3KeyPrefix());
ObjectListing objectListing;
do
{
objectListing = s3Operations.listObjects(listObjectsRequest, s3Client);
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries())
{
// Ignore 0 byte objects that represent S3 directories.
if (!(ignoreZeroByteDirectoryMarkers && objectSummary.getKey().endsWith("/") && objectSummary.getSize() == 0L))
{
s3ObjectSummaries.add(objectSummary);
}
}
listObjectsRequest.setMarker(objectListing.getNextMarker());
}
while (objectListing.isTruncated());
}
catch (AmazonS3Exception amazonS3Exception)
{
if (S3Operations.ERROR_CODE_NO_SUCH_BUCKET.equals(amazonS3Exception.getErrorCode()))
{
throw new IllegalArgumentException("The specified bucket '" + params.getS3BucketName() + "' does not exist.", amazonS3Exception);
}
throw new IllegalStateException("Error accessing S3", amazonS3Exception);
}
catch (AmazonClientException e)
{
throw new IllegalStateException(String
.format("Failed to list keys with prefix \"%s\" from bucket \"%s\". Reason: %s", params.getS3KeyPrefix(), params.getS3BucketName(),
e.getMessage()), e);
}
finally
{
// Shutdown the AmazonS3Client instance to release resources.
s3Client.shutdown();
}
return s3ObjectSummaries;
}
/**
* Lists objects from AmazonS3 in lexicographical order
*
* @param s3Client
* @param s3ConfigBean
* @param pathMatcher glob patterns to match file name against
* @param s3Offset current offset which provides the key name of the previous object
* @param fetchSize number of objects to fetch in one go
* @return
* @throws AmazonClientException
*/
static List<S3ObjectSummary> listObjectsLexicographically(
AmazonS3 s3Client,
S3ConfigBean s3ConfigBean,
AntPathMatcher pathMatcher,
S3Offset s3Offset,
int fetchSize
) {
// Incrementally scan objects after the marker (s3Offset).
List<S3ObjectSummary> list = new ArrayList<>(fetchSize);
ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
listObjectsRequest.setBucketName(s3ConfigBean.s3Config.bucket);
listObjectsRequest.setPrefix(s3ConfigBean.s3Config.commonPrefix);
listObjectsRequest.setMaxKeys(BATCH_SIZE);
if (s3Offset.getKey() != null) {
if (!s3Offset.getKey().isEmpty() && parseOffset(s3Offset) != -1) {
S3ObjectSummary currentObjectSummary = getObjectSummary(s3Client, s3ConfigBean.s3Config.bucket, s3Offset.getKey());
list.add(currentObjectSummary);
}
listObjectsRequest.setMarker(s3Offset.getKey());
}
ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);
while (true) {
for (S3ObjectSummary s : objectListing.getObjectSummaries()) {
String fullPrefix = s.getKey();
String remainingPrefix = fullPrefix.substring(s3ConfigBean.s3Config.commonPrefix.length(), fullPrefix.length());
if (!remainingPrefix.isEmpty()) {
if (pathMatcher.match(s3ConfigBean.s3FileConfig.prefixPattern, remainingPrefix)) {
list.add(s);
}
// We've got enough objects.
if (list.size() == fetchSize) {
return list;
}
}
}
// Listing is complete. No more objects to be listed.
if (!objectListing.isTruncated()) {
break;
}
objectListing = s3Client.listNextBatchOfObjects(objectListing);
}
return list;
}
public cfData execute( cfSession _session, cfArgStructData argStruct ) throws cfmRunTimeException {
AmazonKey amazonKey = getAmazonKey( _session, argStruct );
AmazonS3 s3Client = getAmazonS3( amazonKey );
String bucket = getNamedStringParam( argStruct, "bucket", null );
String prefix = getNamedStringParam( argStruct, "prefix", "" );
if ( bucket == null )
throwException( _session, "Please specify a bucket" );
try {
// Create the results
cfQueryResultData qD = new cfQueryResultData( new String[] { "key", "size", "modified", "etag" }, null );
qD.setQuerySource( "AmazonS3." + amazonKey.getDataSource() );
ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
.withBucketName( bucket )
.withDelimiter( "/" )
.withPrefix( prefix );
ObjectListing objectListing;
do {
objectListing = s3Client.listObjects( listObjectsRequest );
java.util.List<String> prefixes = objectListing.getCommonPrefixes();
// first add the prefixes
for ( String nextPrefix : prefixes ) {
qD.addRow( 1 );
qD.setCurrentRow( qD.getSize() );
qD.setCell( 1, new cfStringData( nextPrefix ) );
qD.setCell( 2, new cfNumberData( 0 ) );
qD.setCell( 3, cfNullData.NULL );
qD.setCell( 4, cfNullData.NULL );
}
for ( S3ObjectSummary objectSummary : objectListing.getObjectSummaries() ) {
// don't include the prefix being listed
if ( objectSummary.getKey().equals( prefix ) ) {
continue;
}
qD.addRow( 1 );
qD.setCurrentRow( qD.getSize() );
qD.setCell( 1, new cfStringData( objectSummary.getKey() ) );
qD.setCell( 2, new cfNumberData( objectSummary.getSize() ) );
qD.setCell( 3, new cfDateData( objectSummary.getLastModified() ) );
qD.setCell( 4, new cfStringData( objectSummary.getETag() ) );
}
listObjectsRequest.setMarker( objectListing.getNextMarker() );
} while ( objectListing.isTruncated() );
return qD;
} catch ( Exception e ) {
throwException( _session, "AmazonS3: " + e.getMessage() );
return cfBooleanData.FALSE;
}
}
private void scanBucket(Set<TocInfo> toc, Queue<TocInfo> tocQueue) throws Exception {
ListObjectsRequest listRequest = new ListObjectsRequest();
listRequest.setBucketName(s3BucketName);
// listRequest.setGeneralProgressListener(this);
listRequest.setMaxKeys(1000);
String nextMarker = null;
ObjectListing objectListing = null;
while(true) {
objectListing = s3Client.listObjects(listRequest);
List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
for (S3ObjectSummary objSummary : objectSummaries) {
String key = objSummary.getKey();
TocInfo tocInfo = new TocInfo(key, objSummary.getSize());
// is it a "dir/" ?
if (key.lastIndexOf("/") == (key.length() - 1)) {
tocInfo.isDirectory = true;
} else {
tocInfo.isDirectory = false;
}
toc.add(tocInfo);
tocQueue.add(tocInfo);
tocInfosGenerated++; // increment for logging
}
// for pagination
nextMarker = objectListing.getNextMarker();
if (nextMarker == null) {
break;
} else {
listRequest.setMarker(nextMarker);
logger.debug("scanBucket() nextMarker we will request listing for => " + nextMarker);
}
}
}
String getObjectInfo(String key, String access_key,
String secret_key, String bucket,
String endpoint, String process
) {
AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
AmazonS3 s3Client = new AmazonS3Client(credentials,
new ClientConfiguration());
if (endpoint.contains("amazonaws.com")) {
String aws_endpoint = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket));
if (aws_endpoint.contains("US")) {
s3Client.setEndpoint("https://s3.amazonaws.com");
} else if (aws_endpoint.contains("us-west")) {
s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
} else if (aws_endpoint.contains("eu-west")) {
s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
} else if (aws_endpoint.contains("ap-")) {
s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
} else if (aws_endpoint.contains("sa-east-1")) {
s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
} else {
s3Client.setEndpoint("https://s3." + aws_endpoint + ".amazonaws.com");
}
} else {
s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
s3Client.setEndpoint(endpoint);
}
objectlist = null;
try {
ObjectListing current = s3Client.listObjects((bucket));
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket);
ObjectListing objectListing;
do {
objectListing = s3Client.listObjects(listObjectsRequest);
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
if (process.contains("checkmd5")) {
if (objectSummary.getKey().contains(key)) {
objectlist = objectSummary.getETag();
break;
}
}
if (process.contains("objectsize")) {
if (objectSummary.getKey().contains(key)) {
objectlist = String.valueOf(objectSummary.getSize());
break;
}
}
if (process.contains("objectdate")) {
if (objectSummary.getKey().contains(key)) {
objectlist = String.valueOf(objectSummary.getLastModified());
break;
}
}
}
listObjectsRequest.setMarker(objectListing.getNextMarker());
} while (objectListing.isTruncated());
} catch (Exception listBucket) {
// mainFrame.jTextArea1.append("\n" + listBucket.getMessage());
}
return objectlist;
}