下面列出了javax.ws.rs.core.UriBuilderException#com.microsoft.azure.storage.blob.ListBlobItem 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
private void deleteStaleBlobs() throws Exception {
final Date expiryDate = Date.from(ZonedDateTime.now().minusWeeks(1).toInstant());
final CloudBlobDirectory directoryReference = blobContainer.getDirectoryReference(request.storageLocation.clusterId + "/" + request.storageLocation.datacenterId);
for (final ListBlobItem blob : directoryReference.listBlobs(null, true, EnumSet.noneOf(BlobListingDetails.class), null, null)) {
if (!(blob instanceof CloudBlob)) {
continue;
}
final BlobProperties properties = ((CloudBlob) blob).getProperties();
if (properties == null || properties.getLastModified() == null) {
continue;
}
if (properties.getLastModified().before(expiryDate)) {
((CloudBlob) blob).delete();
}
}
}
@Override
public Path downloadFileToDir(final Path destinationDir, final Path remotePrefix, final Predicate<String> keyFilter) throws Exception {
final Iterable<ListBlobItem> blobItemsIterable = list(remotePrefix);
final List<ListBlobItem> blobItems = new ArrayList<>();
for (final ListBlobItem listBlobItem : blobItemsIterable) {
if (keyFilter.test(listBlobItem.getUri().getPath())) {
blobItems.add(listBlobItem);
}
}
if (blobItems.size() != 1) {
throw new IllegalStateException(format("There is not one key which satisfies key filter: %s", blobItems.toString()));
}
final String blobItemPath = blobItems.get(0).getUri().getPath();
final String fileName = blobItemPath.split("/")[blobItemPath.split("/").length - 1];
final Path destination = destinationDir.resolve(fileName);
downloadFile(destination, objectKeyToRemoteReference(remotePrefix.resolve(fileName)));
return destination;
}
@Before
public void addMockRules() {
CloudStorageAccount cloudStorageAccount = mock(CloudStorageAccount.class);
CloudBlobClient cloudBlobClient = mock(CloudBlobClient.class);
CloudBlobContainer cloudBlobContainer = mock(CloudBlobContainer.class);
ListBlobItem listBlobItem = mock(ListBlobItem.class);
List<ListBlobItem> lst = new ArrayList<>();
lst.add(listBlobItem);
PowerMockito.mockStatic(CloudStorageAccount.class);
try {
doReturn(cloudStorageAccount).when(CloudStorageAccount.class, "parse", Mockito.anyString());
doReturn(cloudBlobClient).when(cloudStorageAccount).createCloudBlobClient();
doReturn(cloudBlobContainer).when(cloudBlobClient).getContainerReference(Mockito.anyString());
doReturn(true).when(cloudBlobContainer).exists();
when(cloudBlobContainer.listBlobs()).thenReturn(lst);
when(listBlobItem.getUri()).thenReturn(new URI("http://www.google.com"));
} catch (Exception e) {
Assert.fail("Could not initalize mocks, underlying reason " + e.getLocalizedMessage());
}
}
private Map<String, Long> getSnapshotFileKeys(CloudBlobContainer container, String keyPrefix) {
Map<String, Long> snapshotFiles = new HashMap<>();
try {
for (ListBlobItem item : container.listBlobs(keyPrefix, true)) {
if (item instanceof CloudPageBlob) {
CloudPageBlob cloudBlob = (CloudPageBlob) item;
snapshotFiles.put(cloudBlob.getName(), getOriginalFileSize(cloudBlob));
}
}
} catch (StorageException e) {
logger.error("Unable to retrieve metadata.", e);
// all or none
snapshotFiles = new HashMap<>();
}
return snapshotFiles;
}
/**
* For a set of remote storage objects under a remote location and a given prefix/path
* returns their properties wrapped in ObjectSummary objects
*
* @param remoteStorageLocation location, i.e. container for Azure
* @param prefix the prefix/path to list under
* @return a collection of storage summary objects
* @throws StorageProviderException Azure storage exception
*/
@Override
public StorageObjectSummaryCollection listObjects(String remoteStorageLocation, String prefix)
throws StorageProviderException
{
StorageObjectSummaryCollection storageObjectSummaries;
try
{
CloudBlobContainer container = azStorageClient.getContainerReference(remoteStorageLocation);
Iterable<ListBlobItem> listBlobItemIterable = container.listBlobs(
prefix, // List the BLOBs under this prefix
true // List the BLOBs as a flat list, i.e. do not list directories
);
storageObjectSummaries = new StorageObjectSummaryCollection(listBlobItemIterable);
}
catch (URISyntaxException | StorageException ex)
{
logger.debug("Failed to list objects: {}", ex);
throw new StorageProviderException(ex);
}
return storageObjectSummaries;
}
@Override
public void deleteByTenant(final String tenant) {
try {
final CloudBlobContainer container = getContainer();
final CloudBlobDirectory tenantDirectory = container.getDirectoryReference(sanitizeTenant(tenant));
LOG.info("Deleting Azure Storage blob folder (tenant) from container {} for tenant {}", container.getName(),
tenant);
final ResultSegment<ListBlobItem> blobs = tenantDirectory.listBlobsSegmented();
ResultContinuation token = null;
do {
token = blobs.getContinuationToken();
blobs.getResults().stream().filter(CloudBlob.class::isInstance).map(CloudBlob.class::cast)
.forEach(this::deleteBlob);
} while (token != null);
} catch (final URISyntaxException | StorageException e) {
throw new ArtifactStoreException("Failed to delete tenant directory from Azure storage", e);
}
}
public void deleteIfExist(RuntimeContainer runtimeContainer) {
try {
for (RemoteBlob rmtb : createRemoteBlobFilter()) {
for (ListBlobItem blob : azureStorageBlobService.listBlobs(containerName, rmtb.prefix, rmtb.include)) {
if (blob instanceof CloudBlockBlob) {
// FIXME - problem with blobs with space in name...
boolean successfulyDeleted = azureStorageBlobService.deleteBlobBlockIfExist((CloudBlockBlob) blob);
if (!successfulyDeleted) {
LOGGER.warn(messages.getMessage("warn.FaildDelete", ((CloudBlockBlob) blob).getName()));
}
}
}
}
} catch (StorageException | URISyntaxException | InvalidKeyException e) {
LOGGER.error(e.getLocalizedMessage());
if (dieOnError) {
throw new ComponentException(e);
}
}
}
@Test
public void testStartAsNonStartable() {
try {
when(blobService.listBlobs(anyString(), anyString(), anyBoolean())).thenReturn(new Iterable<ListBlobItem>() {
@Override
public Iterator<ListBlobItem> iterator() {
return new DummyListBlobItemIterator(new ArrayList<CloudBlockBlob>());
}
});
properties.remoteBlobs = new RemoteBlobsTable("RemoteBlobsTable");
properties.remoteBlobs.include.setValue(Arrays.asList(true));
properties.remoteBlobs.prefix.setValue(Arrays.asList("dummyFilter"));
boolean startable = reader.start();
assertFalse(startable);
assertFalse(reader.advance());
} catch (InvalidKeyException | URISyntaxException | StorageException | IOException e) {
fail("should not throw " + e.getMessage());
}
}
public void testDownloadBlob() {
try {
// Loop through each blob item in the container.
for (ListBlobItem blobItem : container.listBlobs()) {
// If the item is a blob, not a virtual directory.
if (blobItem instanceof CloudBlob) {
// Download the item and save it to a file with the same
// name.
CloudBlob blob = (CloudBlob) blobItem;
blob.download(new FileOutputStream("C:\\mydownloads\\" + blob.getName()));
}
}
} catch (Exception e) {
// Output the stack trace.
e.printStackTrace();
}
}
/**
* List all logs
*
* @throws URISyntaxException
* @throws StorageException
* @throws IOException
* @throws InterruptedException
*/
public void testCloudAnalyticsClientListLogs() throws URISyntaxException, StorageException, IOException {
this.container.create();
this.client.LogContainer = this.container.getName();
int numBlobs = 13;
Calendar now = new GregorianCalendar();
now.add(GregorianCalendar.MONTH, -13);
List<String> blobNames = AnalyticsTestHelper.CreateLogs(this.container, StorageService.BLOB, 13, now,
Granularity.MONTH);
assertEquals(numBlobs, blobNames.size());
for (ListBlobItem blob : this.client.listLogBlobs(StorageService.BLOB)) {
assertEquals(CloudBlockBlob.class, blob.getClass());
assertTrue(blobNames.remove(((CloudBlockBlob) blob).getName()));
}
assertTrue(blobNames.size() == 0);
}
/**
* List Logs with open ended time range
*
* @throws URISyntaxException
* @throws StorageException
* @throws IOException
* @throws InterruptedException
*/
@Test
public void testCloudAnalyticsClientListLogsStartTime() throws URISyntaxException, StorageException, IOException {
this.container.create();
this.client.LogContainer = this.container.getName();
int numBlobs = 48;
Calendar now = new GregorianCalendar();
now.add(GregorianCalendar.DAY_OF_MONTH, -2);
List<String> blobNames = AnalyticsTestHelper.CreateLogs(this.container, StorageService.BLOB, 48, now,
Granularity.HOUR);
assertEquals(numBlobs, blobNames.size());
Calendar start = new GregorianCalendar();
start.add(GregorianCalendar.DAY_OF_MONTH, -1);
for (ListBlobItem blob : this.client.listLogBlobs(StorageService.BLOB, start.getTime(), null, null, null, null,
null)) {
assertEquals(CloudBlockBlob.class, blob.getClass());
assertTrue(blobNames.remove(((CloudBlockBlob) blob).getName()));
}
assertTrue(blobNames.size() == 24);
}
/**
* List Logs with well defined time range
*
* @throws URISyntaxException
* @throws StorageException
* @throws IOException
* @throws InterruptedException
*/
@Test
public void testCloudAnalyticsClientListLogsStartEndTime() throws URISyntaxException, StorageException, IOException {
this.container.create();
this.client.LogContainer = this.container.getName();
int numBlobs = 72;
Calendar now = new GregorianCalendar();
now.add(GregorianCalendar.DAY_OF_MONTH, -3);
List<String> blobNames = AnalyticsTestHelper.CreateLogs(this.container, StorageService.BLOB, 72, now,
Granularity.HOUR);
assertEquals(numBlobs, blobNames.size());
Calendar start = new GregorianCalendar();
start.add(GregorianCalendar.DAY_OF_MONTH, -2);
Calendar end = new GregorianCalendar();
end.add(GregorianCalendar.DAY_OF_MONTH, -1);
for (ListBlobItem blob : this.client.listLogBlobs(StorageService.BLOB, start.getTime(), end.getTime(), null,
null, null, null)) {
assertEquals(CloudBlockBlob.class, blob.getClass());
assertTrue(blobNames.remove(((CloudBlockBlob) blob).getName()));
}
assertTrue(blobNames.size() == 48);
}
/**
* Validates that the log given is of the correct log type.
*
* @param current
* the current log
* @return whether or not the log is of the correct type.
*/
private boolean isCorrectLogType(ListBlobItem current) {
HashMap<String, String> metadata = ((CloudBlob) current).getMetadata();
String logType = metadata.get("LogType");
if (logType == null) {
return true;
}
if (this.operations.contains(LoggingOperations.READ) && logType.contains("read")) {
return true;
}
if (this.operations.contains(LoggingOperations.WRITE) && logType.contains("write")) {
return true;
}
if (this.operations.contains(LoggingOperations.DELETE) && logType.contains("delete")) {
return true;
}
return false;
}
private void deleteBlobsInDirectory(CloudBlobContainer cloudBlobContainer, String directoryName)
throws URISyntaxException, StorageException {
CloudBlobDirectory blobDirectory = cloudBlobContainer.getDirectoryReference(directoryName);
for (ListBlobItem blobItem : blobDirectory.listBlobs()) {
if (blobItem instanceof CloudBlobDirectory) {
deleteBlobsInDirectory(cloudBlobContainer, ((CloudBlobDirectory) blobItem).getPrefix());
} else if (blobItem instanceof CloudPageBlob) {
CloudPageBlob cloudPageBlob = cloudBlobContainer.getPageBlobReference(((CloudPageBlob) blobItem).getName());
cloudPageBlob.deleteIfExists();
} else if (blobItem instanceof CloudBlockBlob) {
CloudBlockBlob cloudBlockBlob = cloudBlobContainer.getBlockBlobReference(((CloudBlockBlob) blobItem).getName());
cloudBlockBlob.deleteIfExists();
}
}
}
private void listBlobsInDirectory(CloudBlobContainer cloudBlobContainer, String directoryName)
throws URISyntaxException, StorageException {
CloudBlobDirectory blobDirectory = cloudBlobContainer.getDirectoryReference(directoryName);
for (ListBlobItem blobItem : blobDirectory.listBlobs()) {
if (blobItem instanceof CloudBlobDirectory) {
listBlobsInDirectory(cloudBlobContainer, ((CloudBlobDirectory) blobItem).getPrefix());
} else if (blobItem instanceof CloudPageBlob) {
Log.log(LOGGER, format(" Azure Adls Gen 2 Cloud Page Blob is present with Name: [%s] and with bytes of content: [%d] at URI: [%s] ",
((CloudPageBlob) blobItem).getName(), ((CloudPageBlob) blobItem).getProperties().getLength(), blobItem.getUri().getPath()));
} else if (blobItem instanceof CloudBlockBlob) {
Log.log(LOGGER, format(" Azure Adls Gen 2 Cloud Block Blob is present with Name: [%s] and with bytes of content: [%d] at URI: [%s] ",
((CloudBlockBlob) blobItem).getName(), ((CloudBlockBlob) blobItem).getProperties().getLength(), blobItem.getUri().getPath()));
} else {
LOGGER.error("Azure Adls Gen 2 Cloud Storage Item that is present at URI: [{}] cannot be classify as CloudBlob, CloudPageBlob and " +
"CloudBlockBlob. ", blobItem.getUri().getPath());
throw new TestFailException(String.format("Azure Adls Gen 2 Cloud Storage Item that is present at URI: [%s] cannot be classify as" +
" CloudBlob, CloudPageBlob and CloudBlockBlob. ", blobItem.getUri().getPath()));
}
}
}
private void listBlobsInDirectoryWithValidation(CloudBlobContainer cloudBlobContainer, String directoryName, Boolean zeroContent)
throws URISyntaxException, StorageException {
CloudBlobDirectory blobDirectory = cloudBlobContainer.getDirectoryReference(directoryName);
Set<String> blobsWithZeroLength = new HashSet<>();
for (ListBlobItem blobItem : blobDirectory.listBlobs()) {
if (blobItem instanceof CloudBlobDirectory) {
listBlobsInDirectoryWithValidation(cloudBlobContainer, ((CloudBlobDirectory) blobItem).getPrefix(), zeroContent);
} else if (blobItem instanceof CloudPageBlob) {
validateBlobItemLength(blobItem, zeroContent, blobsWithZeroLength);
} else if (blobItem instanceof CloudBlockBlob) {
validateBlobItemLength(blobItem, zeroContent, blobsWithZeroLength);
} else {
LOGGER.error("Azure Adls Gen 2 Cloud Storage Item that is present at URI: {} cannot be classify as CloudBlob, CloudPageBlob and " +
"CloudBlockBlob. ", blobItem.getUri().getPath());
throw new TestFailException(String.format("Azure Adls Gen 2 Cloud Storage Item that is present at URI: %s cannot be classify as" +
" CloudBlob, CloudPageBlob and CloudBlockBlob. ", blobItem.getUri().getPath()));
}
}
}
public SdxTestDto deleteAllFolders(TestContext testContext, SdxTestDto sdxTestDto, SdxClient sdxClient) {
String containerName = getContainerName(sdxTestDto.getRequest().getCloudStorage().getBaseLocation());
CloudBlobContainer cloudBlobContainer = getCloudBlobContainer(containerName);
try {
for (ListBlobItem blob : cloudBlobContainer.listBlobs()) {
String blobName = blob.getUri().getPath().split("/", 3)[2];
String blobUriPath = blob.getUri().getPath();
if (blob instanceof CloudBlob) {
((CloudBlob) blob).deleteIfExists();
} else {
if (blobName.endsWith("/")) {
blobName = blobName.replaceAll(".$", "");
}
CloudBlobDirectory blobDirectory = cloudBlobContainer.getDirectoryReference(blobName);
deleteBlobsInDirectory(cloudBlobContainer, blobDirectory.getPrefix());
}
}
} catch (StorageException | URISyntaxException e) {
LOGGER.error("Azure Adls Gen 2 Blob couldn't process the call. So it has been returned with error!", e);
throw new TestFailException(String.format("Azure Adls Gen 2 Blob couldn't process the call. So it has been returned the error: %s", e));
}
return sdxTestDto;
}
public void deleteAllFolders() throws StorageException, URISyntaxException {
String containerName = getContainerName();
CloudBlobContainer cloudBlobContainer = getCloudBlobContainer(containerName);
try {
for (ListBlobItem blob : cloudBlobContainer.listBlobs()) {
String blobName = blob.getUri().getPath().split("/", 3)[2];
String blobUriPath = blob.getUri().getPath();
if (blob instanceof CloudBlob) {
((CloudBlob) blob).deleteIfExists();
} else {
if (blobName.endsWith("/")) {
blobName = blobName.replaceAll(".$", "");
}
CloudBlobDirectory blobDirectory = cloudBlobContainer.getDirectoryReference(blobName);
deleteBlobsInDirectory(cloudBlobContainer, blobDirectory.getPrefix());
}
}
} catch (StorageException | URISyntaxException e) {
LOGGER.error("Azure Adls Gen 2 Blob couldn't process the call. So it has been returned with error!", e);
throw e;
}
}
public Set<String> children(String account, String container, BlobPath path) throws URISyntaxException, StorageException {
final var blobsBuilder = new HashSet<String>();
final Tuple<CloudBlobClient, Supplier<OperationContext>> client = client();
final CloudBlobContainer blobContainer = client.v1().getContainerReference(container);
final String keyPath = path.buildAsString();
final EnumSet<BlobListingDetails> enumBlobListingDetails = EnumSet.of(BlobListingDetails.METADATA);
for (ListBlobItem blobItem : blobContainer.listBlobs(keyPath, false, enumBlobListingDetails, null, client.v2().get())) {
if (blobItem instanceof CloudBlobDirectory) {
final URI uri = blobItem.getUri();
LOGGER.trace(() -> new ParameterizedMessage("blob url [{}]", uri));
// uri.getPath is of the form /container/keyPath.* and we want to strip off the /container/
// this requires 1 + container.length() + 1, with each 1 corresponding to one of the /.
// Lastly, we add the length of keyPath to the offset to strip this container's path.
final String uriPath = uri.getPath();
blobsBuilder.add(uriPath.substring(1 + container.length() + 1 + keyPath.length(), uriPath.length() - 1));
}
}
return Set.copyOf(blobsBuilder);
}
@Override
public void consumeFiles(final RemoteObjectReference prefix, final Consumer<RemoteObjectReference> consumer) throws Exception {
final AzureRemoteObjectReference azureRemoteObjectReference = (AzureRemoteObjectReference) prefix;
final Iterable<ListBlobItem> blobItemsIterable = list(azureRemoteObjectReference.getObjectKey());
for (final ListBlobItem listBlobItem : blobItemsIterable) {
try {
consumer.accept(objectKeyToRemoteReference(removeNodePrefix(listBlobItem)));
} catch (StorageException | URISyntaxException ex) {
logger.error("Error occurred while trying to consume {}", listBlobItem.getUri().toString(), ex);
throw ex;
}
}
}
private Path removeNodePrefix(final ListBlobItem listBlobItem) {
final String pattern = format("^/%s/%s/%s/%s/",
request.storageLocation.bucket,
request.storageLocation.clusterId,
request.storageLocation.datacenterId,
request.storageLocation.nodeId);
final Pattern containerPattern = Pattern.compile(pattern);
return Paths.get(containerPattern.matcher(listBlobItem.getUri().getPath()).replaceFirst(""));
}
private Iterable<ListBlobItem> list(final Path prefix) {
final String blobPrefix = Paths.get(request.storageLocation.clusterId)
.resolve(request.storageLocation.datacenterId)
.resolve(request.storageLocation.nodeId)
.resolve(prefix).toString();
return blobContainer.listBlobs(blobPrefix, true, EnumSet.noneOf(BlobListingDetails.class), null, null);
}
public static List<String> listAllBlobbs(String containerName) {
List<String> blobsList = new ArrayList<>();
CloudBlobContainer container = AzureConnectionManager.getContainer(containerName, true);
// Loop over blobs within the container and output the URI to each of them.
if (container != null) {
for (ListBlobItem blobItem : container.listBlobs()) {
blobsList.add(blobItem.getUri().toString());
}
}
return blobsList;
}
public static Iterable<ListBlobItem> wrap(
final Iterable<ListBlobItem> present) {
return new Iterable<ListBlobItem>() {
@Override
public Iterator<ListBlobItem> iterator() {
return new WrappingIterator(present.iterator());
}
};
}
@Override
public ListBlobItem next() {
ListBlobItem unwrapped = present.next();
if (unwrapped instanceof CloudBlobDirectory) {
return new CloudBlobDirectoryWrapperImpl((CloudBlobDirectory) unwrapped);
} else if (unwrapped instanceof CloudBlockBlob) {
return new CloudBlockBlobWrapperImpl((CloudBlockBlob) unwrapped);
} else if (unwrapped instanceof CloudPageBlob) {
return new CloudPageBlobWrapperImpl((CloudPageBlob) unwrapped);
} else {
return unwrapped;
}
}
@Override
public Iterable<ListBlobItem> listBlobs(String prefix,
boolean useFlatBlobListing, EnumSet<BlobListingDetails> listingDetails,
BlobRequestOptions options, OperationContext opContext)
throws URISyntaxException, StorageException {
return WrappingIterator.wrap(directory.listBlobs(prefix,
useFlatBlobListing, listingDetails, options, opContext));
}
@Override
public void purge(String prefix) throws IOException {
try {
// Attempts to purge may occur before opening any streams so first,
// check if a session exists, if not create a session with the Azure
// storage server.
if (null == storageInteractionLayer) {
final String errMsg = String.format(
"Storage session expected for URI '%s' but does not exist.",
sessionUri);
throw new AssertionError(errMsg);
}
if (checkContainer(ContainerAccessType.ReadThenWrite) == ContainerState.DoesntExist) {
// Container doesn't exist, no need to do anything.
return;
}
// Get all blob items with the given prefix from the container and delete
// them.
Iterable<ListBlobItem> objects = listRootBlobs(prefix, false);
for (ListBlobItem blobItem : objects) {
((CloudBlob) blobItem).delete(DeleteSnapshotsOption.NONE, null, null,
getInstrumentedContext());
}
} catch (Exception e) {
// Re-throw as an Azure storage exception.
//
throw new AzureException(e);
}
}
public static Iterable<ListBlobItem> wrap(
final Iterable<ListBlobItem> present) {
return new Iterable<ListBlobItem>() {
@Override
public Iterator<ListBlobItem> iterator() {
return new WrappingIterator(present.iterator());
}
};
}
@Override
public ListBlobItem next() {
ListBlobItem unwrapped = present.next();
if (unwrapped instanceof CloudBlobDirectory) {
return new CloudBlobDirectoryWrapperImpl((CloudBlobDirectory) unwrapped);
} else if (unwrapped instanceof CloudBlockBlob) {
return new CloudBlockBlobWrapperImpl((CloudBlockBlob) unwrapped);
} else if (unwrapped instanceof CloudPageBlob) {
return new CloudPageBlobWrapperImpl((CloudPageBlob) unwrapped);
} else {
return unwrapped;
}
}
@Override
public Iterable<ListBlobItem> listBlobs(String prefix,
boolean useFlatBlobListing, EnumSet<BlobListingDetails> listingDetails,
BlobRequestOptions options, OperationContext opContext)
throws URISyntaxException, StorageException {
return WrappingIterator.wrap(directory.listBlobs(prefix,
useFlatBlobListing, listingDetails, options, opContext));
}