下面列出了com.amazonaws.services.s3.model.ListObjectsRequest#setPrefix ( ) 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
@Override
protected List<String> getFileNames(String lockPrefix) throws Exception
{
ListObjectsRequest request = new ListObjectsRequest();
request.setBucketName(bucket);
request.setPrefix(lockPrefix);
ObjectListing objectListing = client.listObjects(request);
return Lists.transform
(
objectListing.getObjectSummaries(),
new Function<S3ObjectSummary, String>()
{
@Override
public String apply(S3ObjectSummary summary)
{
return summary.getKey();
}
}
);
}
@Override
public void list(String keyPrefix, FileListing callback)
{
checkArgument(keyPrefix != null, "keyPrefix is null");
String errorMessage = "listing files on bucket " + bucket + " prefix " + keyPrefix;
ListObjectsRequest req = new ListObjectsRequest();
req.setBucketName(bucket);
req.setPrefix(keyPrefix);
ObjectListing listing;
do {
try {
listing = getWithRetry(errorMessage, () -> client.listObjects(req));
}
catch (StorageFileNotFoundException ex) {
throw Throwables.propagate(ex.getCause());
}
callback.accept(Lists.transform(
listing.getObjectSummaries(),
(summary) -> StorageObjectSummary.builder()
.key(summary.getKey())
.contentLength(summary.getSize())
.lastModified(summary.getLastModified().toInstant())
.build()
));
req.setMarker(listing.getNextMarker());
}
while (listing.isTruncated());
}
/**
* Lists objects from AmazonS3 in lexicographical order
*
* @param s3Client
* @param s3ConfigBean
* @param pathMatcher glob patterns to match file name against
* @param s3Offset current offset which provides the key name of the previous object
* @param fetchSize number of objects to fetch in one go
* @return
* @throws AmazonClientException
*/
static List<S3ObjectSummary> listObjectsLexicographically(
AmazonS3 s3Client,
S3ConfigBean s3ConfigBean,
AntPathMatcher pathMatcher,
S3Offset s3Offset,
int fetchSize
) {
// Incrementally scan objects after the marker (s3Offset).
List<S3ObjectSummary> list = new ArrayList<>(fetchSize);
ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
listObjectsRequest.setBucketName(s3ConfigBean.s3Config.bucket);
listObjectsRequest.setPrefix(s3ConfigBean.s3Config.commonPrefix);
listObjectsRequest.setMaxKeys(BATCH_SIZE);
if (s3Offset.getKey() != null) {
if (!s3Offset.getKey().isEmpty() && parseOffset(s3Offset) != -1) {
S3ObjectSummary currentObjectSummary = getObjectSummary(s3Client, s3ConfigBean.s3Config.bucket, s3Offset.getKey());
list.add(currentObjectSummary);
}
listObjectsRequest.setMarker(s3Offset.getKey());
}
ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);
while (true) {
for (S3ObjectSummary s : objectListing.getObjectSummaries()) {
String fullPrefix = s.getKey();
String remainingPrefix = fullPrefix.substring(s3ConfigBean.s3Config.commonPrefix.length(), fullPrefix.length());
if (!remainingPrefix.isEmpty()) {
if (pathMatcher.match(s3ConfigBean.s3FileConfig.prefixPattern, remainingPrefix)) {
list.add(s);
}
// We've got enough objects.
if (list.size() == fetchSize) {
return list;
}
}
}
// Listing is complete. No more objects to be listed.
if (!objectListing.isTruncated()) {
break;
}
objectListing = s3Client.listNextBatchOfObjects(objectListing);
}
return list;
}
/**
* Calculate the total size of all objects in the indicated bucket
*
* @param path to use
* @return calculated size
* @throws IOException
*/
@Override
public long calculateSize(Path path) throws IOException {
long totalBucketSize = 0L;
if (s3Enabled) {
String key = pathToKey(path);
final FileStatus fileStatus = fs.getFileStatus(path);
if (fileStatus.isDirectory()) {
if (!key.isEmpty()) {
key = key + "/";
}
ListObjectsRequest request = new ListObjectsRequest();
request.setBucketName(uri.getHost());
request.setPrefix(key);
request.setMaxKeys(maxKeys);
if (LOG.isDebugEnabled()) {
LOG.debug("listStatus: doing listObjects for directory " + key);
}
ObjectListing objects = s3.listObjects(request);
while (true) {
for (S3ObjectSummary summary : objects.getObjectSummaries()) {
Path keyPath = keyToPath(summary.getKey()).makeQualified(uri, fs.getWorkingDirectory());
// Skip over keys that are ourselves and old S3N _$folder$ files
if (keyPath.equals(path) || summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Ignoring: " + keyPath);
}
continue;
}
if (!objectRepresentsDirectory(summary.getKey(), summary.getSize())) {
totalBucketSize += summary.getSize();
}
}
if (objects.isTruncated()) {
if (LOG.isDebugEnabled()) {
LOG.debug("listStatus: list truncated - getting next batch");
}
objects = s3.listNextBatchOfObjects(objects);
} else {
break;
}
}
} else {
return fileStatus.getLen();
}
} else {
totalBucketSize = fs.getContentSummary(path).getLength();
}
return totalBucketSize;
}