java.util.concurrent.ConcurrentHashMap.KeySetView#com.google.api.services.storage.Storage源码实例Demo

public static InputStream downloadObject(
    Storage storage,
    String bucketName,
    String objectName,
    String base64CseKey,
    String base64CseKeyHash)
    throws Exception {

  // Set the CSEK headers
  final HttpHeaders httpHeaders = new HttpHeaders();
  httpHeaders.set("x-goog-encryption-algorithm", "AES256");
  httpHeaders.set("x-goog-encryption-key", base64CseKey);
  httpHeaders.set("x-goog-encryption-key-sha256", base64CseKeyHash);
  Storage.Objects.Get getObject = storage.objects().get(bucketName, objectName);
  getObject.setRequestHeaders(httpHeaders);

  try {

    return getObject.executeMediaAsInputStream();
  } catch (GoogleJsonResponseException e) {
    LOG.info("Error downloading: " + e.getContent());
    System.exit(1);
    return null;
  }
}
 

@Test
public void initGeneration_hasGenerationId() throws IOException {
  StorageObject storageObject = newStorageObject(BUCKET_NAME, OBJECT_NAME);
  MockHttpTransport transport = mockTransport(jsonDataResponse(storageObject));

  List<HttpRequest> requests = new ArrayList<>();

  Storage storage = new Storage(transport, JSON_FACTORY, requests::add);

  GoogleCloudStorageReadOptions options =
      GoogleCloudStorageReadOptions.builder().setFastFailOnNotFound(false).build();

  GoogleCloudStorageReadChannel readChannel = createReadChannel(storage, options);
  // initialize metadata
  readChannel.size();
  assertThat(readChannel.generation()).isEqualTo(storageObject.getGeneration());
}
 

@Test
public void newBatchHelper_throwsException_whenTotalRequestsZero() {
  Storage storage = new Storage(HTTP_TRANSPORT, JSON_FACTORY, httpRequestInitializer);

  IllegalArgumentException e =
      assertThrows(
          IllegalArgumentException.class,
          () ->
              batchFactory.newBatchHelper(
                  httpRequestInitializer,
                  storage,
                  /* maxRequestsPerBatch= */ 1,
                  /* totalRequests= */ 0,
                  /* maxThreads= */ 1));

  assertThat(e).hasMessageThat().startsWith("totalRequests should be greater than 0");
}
 

/**
 * Constructs an instance of GoogleCloudStorageReadChannel.
 *
 * @param gcs storage object instance
 * @param bucketName name of the bucket containing the object to read
 * @param objectName name of the object to read
 * @param requestHelper a ClientRequestHelper used to set any extra headers
 * @param readOptions fine-grained options specifying things like retry settings, buffering, etc.
 *     Could not be null.
 * @throws IOException on IO error
 */
public GoogleCloudStorageReadChannel(
    Storage gcs,
    String bucketName,
    String objectName,
    ApiErrorExtractor errorExtractor,
    ClientRequestHelper<StorageObject> requestHelper,
    @Nonnull GoogleCloudStorageReadOptions readOptions)
    throws IOException {
  this(
      gcs,
      new StorageResourceId(bucketName, objectName),
      errorExtractor,
      requestHelper,
      readOptions);
}
 

private GcsUtil(
    Storage storageClient,
    HttpRequestInitializer httpRequestInitializer,
    ExecutorService executorService,
    Boolean shouldUseGrpc,
    @Nullable Integer uploadBufferSizeBytes) {
  this.storageClient = storageClient;
  this.httpRequestInitializer = httpRequestInitializer;
  this.uploadBufferSizeBytes = uploadBufferSizeBytes;
  this.executorService = executorService;
  this.maxBytesRewrittenPerCall = null;
  this.numRewriteTokensUsed = null;
  this.shouldUseGrpc = shouldUseGrpc;
  googleCloudStorageOptions =
      GoogleCloudStorageOptions.newBuilder()
          .setAppName("Beam")
          .setGrpcEnabled(shouldUseGrpc)
          .build();
  googleCloudStorage = new GoogleCloudStorageImpl(googleCloudStorageOptions, storageClient);
}
 

/** Lists {@link Objects} given the {@code bucket}, {@code prefix}, {@code pageToken}. */
public Objects listObjects(String bucket, String prefix, @Nullable String pageToken)
    throws IOException {
  // List all objects that start with the prefix (including objects in sub-directories).
  Storage.Objects.List listObject = storageClient.objects().list(bucket);
  listObject.setMaxResults(MAX_LIST_ITEMS_PER_CALL);
  listObject.setPrefix(prefix);

  if (pageToken != null) {
    listObject.setPageToken(pageToken);
  }

  try {
    return ResilientOperation.retry(
        ResilientOperation.getGoogleRequestCallable(listObject),
        createBackOff(),
        RetryDeterminer.SOCKET_ERRORS,
        IOException.class);
  } catch (Exception e) {
    throw new IOException(
        String.format("Unable to match files in bucket %s, prefix %s.", bucket, prefix), e);
  }
}
 

@Test
public void testWithMultipleTraces() throws Exception {
  String[] args = new String[] {STORAGE_GET_AND_LIST_TRACE};
  GcsOptions options = PipelineOptionsFactory.fromArgs(args).as(GcsOptions.class);
  options.setGcpCredential(new TestCredential());

  assertNotNull(options.getGoogleApiTrace());

  Storage.Objects.Get getRequest =
      Transport.newStorageClient(options).build().objects().get("testBucketId", "testObjectId");
  assertEquals("GetTraceDestination", getRequest.get("$trace"));

  Storage.Objects.List listRequest =
      Transport.newStorageClient(options).build().objects().list("testProjectId");
  assertEquals("ListTraceDestination", listRequest.get("$trace"));
}
 

@Test
public void testMatchingAllCalls() throws Exception {
  String[] args = new String[] {STORAGE_TRACE};
  GcsOptions options = PipelineOptionsFactory.fromArgs(args).as(GcsOptions.class);
  options.setGcpCredential(new TestCredential());

  assertNotNull(options.getGoogleApiTrace());

  Storage.Objects.Get getRequest =
      Transport.newStorageClient(options).build().objects().get("testBucketId", "testObjectId");
  assertEquals("TraceDestination", getRequest.get("$trace"));

  Storage.Objects.List listRequest =
      Transport.newStorageClient(options).build().objects().list("testProjectId");
  assertEquals("TraceDestination", listRequest.get("$trace"));
}
 

@Test
public void testMatchingAgainstClient() throws Exception {
  GcsOptions options = PipelineOptionsFactory.as(GcsOptions.class);
  options.setGcpCredential(new TestCredential());
  options.setGoogleApiTrace(
      new GoogleApiTracer()
          .addTraceFor(Transport.newStorageClient(options).build(), "TraceDestination"));

  Storage.Objects.Get getRequest =
      Transport.newStorageClient(options).build().objects().get("testBucketId", "testObjectId");
  assertEquals("TraceDestination", getRequest.get("$trace"));

  Delete deleteRequest =
      GcpOptions.GcpTempLocationFactory.newCloudResourceManagerClient(
              options.as(CloudResourceManagerOptions.class))
          .build()
          .projects()
          .delete("testProjectId");
  assertNull(deleteRequest.get("$trace"));
}
 

@Test
public void testBasicOperation() throws IOException {
  when(mockLowLevelRequest.execute()).thenReturn(mockLowLevelResponse);
  when(mockLowLevelResponse.getStatusCode()).thenReturn(200);

  Storage.Buckets.Get result = storage.buckets().get("test");
  HttpResponse response = result.executeUnparsed();
  assertNotNull(response);

  verify(mockHttpResponseInterceptor).interceptResponse(any(HttpResponse.class));
  verify(mockLowLevelRequest, atLeastOnce()).addHeader(anyString(), anyString());
  verify(mockLowLevelRequest).setTimeout(anyInt(), anyInt());
  verify(mockLowLevelRequest).setWriteTimeout(anyInt());
  verify(mockLowLevelRequest).execute();
  verify(mockLowLevelResponse).getStatusCode();
  expectedLogs.verifyNotLogged("Request failed");
}
 

/** Tests that a non-retriable error is not retried. */
@Test
public void testErrorCodeForbidden() throws IOException {
  when(mockLowLevelRequest.execute()).thenReturn(mockLowLevelResponse);
  when(mockLowLevelResponse.getStatusCode())
      .thenReturn(403) // Non-retryable error.
      .thenReturn(200); // Shouldn't happen.

  try {
    Storage.Buckets.Get result = storage.buckets().get("test");
    HttpResponse response = result.executeUnparsed();
    assertNotNull(response);
  } catch (HttpResponseException e) {
    assertThat(e.getMessage(), Matchers.containsString("403"));
  }

  verify(mockHttpResponseInterceptor).interceptResponse(any(HttpResponse.class));
  verify(mockLowLevelRequest, atLeastOnce()).addHeader(anyString(), anyString());
  verify(mockLowLevelRequest).setTimeout(anyInt(), anyInt());
  verify(mockLowLevelRequest).setWriteTimeout(anyInt());
  verify(mockLowLevelRequest).execute();
  verify(mockLowLevelResponse).getStatusCode();
  expectedLogs.verifyWarn("Request failed with code 403");
}
 

/**
 * Constructs an instance of GoogleCloudStorageReadChannel.
 *
 * @param gcs storage object instance
 * @param resourceId contains information about a specific resource
 * @param requestHelper a ClientRequestHelper used to set any extra headers
 * @param readOptions fine-grained options specifying things like retry settings, buffering, etc.
 *     Could not be null.
 * @throws IOException on IO error
 */
public GoogleCloudStorageReadChannel(
    Storage gcs,
    StorageResourceId resourceId,
    ApiErrorExtractor errorExtractor,
    ClientRequestHelper<StorageObject> requestHelper,
    @Nonnull GoogleCloudStorageReadOptions readOptions)
    throws IOException {
  this.gcs = gcs;
  this.clientRequestHelper = requestHelper;
  this.errorExtractor = errorExtractor;
  this.readOptions = readOptions;
  this.resourceId = resourceId;

  // Initialize metadata if available.
  GoogleCloudStorageItemInfo info = getInitialMetadata();

  if (info != null) {
    initMetadata(info);
  }
}
 

@Test
public void initMetadata_throwsException_whenReadConsistencyEnabledAndGenerationIsNull()
    throws IOException {
  Storage storage = new Storage(HTTP_TRANSPORT, JSON_FACTORY, r -> {});

  GoogleCloudStorageReadOptions options = newLazyReadOptionsBuilder().build();

  GoogleCloudStorageReadChannel readChannel = createReadChannel(storage, options);

  IllegalStateException e =
      assertThrows(
          IllegalStateException.class,
          () ->
              readChannel.initMetadata("gzip", /* sizeFromMetadata= */ 1, UNKNOWN_GENERATION_ID));

  assertThat(e).hasMessageThat().contains("Generation parameter of -1 is invalid");
}
 

@Override
public void uploadCache(JobOutputProcessor outputProcessor, String cachePath) {
    try {
            outputProcessor.output("Uploading cache file " + cacheFileName + " to google storage\n");

            Storage client = createClient();
            File uploadFile = new File(cachePath);
            InputStreamContent contentStream = new InputStreamContent(
                    null, new FileInputStream(uploadFile));
            contentStream.setLength(uploadFile.length());
            StorageObject objectMetadata = new StorageObject()
                    .setName(cacheFileName);

            Storage.Objects.Insert insertRequest = client.objects().insert(
                    settings.bucketName, objectMetadata, contentStream);

            insertRequest.execute();

            outputProcessor.output("Cache uploaded\n");
    } catch (GeneralSecurityException | IOException e) {
        outputProcessor.output("Error upload cache: " + e.getMessage() + "\n");
    }
}
 

@Test
public void eagerReadFileAtSpecificGeneration_succeeds_whenReadConsistencyStrict()
    throws IOException {
  long generation = 5L;
  MockHttpTransport transport =
      mockTransport(
          jsonDataResponse(newStorageObject(BUCKET_NAME, OBJECT_NAME).setGeneration(generation)));

  List<HttpRequest> requests = new ArrayList<>();

  Storage storage = new Storage(transport, JSON_FACTORY, requests::add);

  GoogleCloudStorageReadOptions options =
      GoogleCloudStorageReadOptions.builder().setFastFailOnNotFound(true).build();

  GoogleCloudStorageReadChannel readChannel = createReadChannel(storage, options, generation);
  assertThat(readChannel.generation()).isEqualTo(generation);
}
 

@Test
public void testBucketAccessible() throws IOException {
  GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
  GcsUtil gcsUtil = pipelineOptions.getGcsUtil();

  Storage mockStorage = Mockito.mock(Storage.class);
  gcsUtil.setStorageClient(mockStorage);

  Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class);
  Storage.Buckets.Get mockStorageGet = Mockito.mock(Storage.Buckets.Get.class);

  BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff());

  when(mockStorage.buckets()).thenReturn(mockStorageObjects);
  when(mockStorageObjects.get("testbucket")).thenReturn(mockStorageGet);
  when(mockStorageGet.execute())
      .thenThrow(new SocketTimeoutException("SocketException"))
      .thenReturn(new Bucket());

  assertTrue(
      gcsUtil.bucketAccessible(
          GcsPath.fromComponents("testbucket", "testobject"),
          mockBackOff,
          new FastNanoClockAndSleeper()));
}
 

@Test
public void testMakeRewriteOps() throws IOException {
  GcsOptions gcsOptions = gcsOptionsWithTestCredential();
  GcsUtil gcsUtil = gcsOptions.getGcsUtil();

  LinkedList<RewriteOp> rewrites =
      gcsUtil.makeRewriteOps(makeStrings("s", 1), makeStrings("d", 1));
  assertEquals(1, rewrites.size());

  RewriteOp rewrite = rewrites.pop();
  assertTrue(rewrite.getReadyToEnqueue());
  Storage.Objects.Rewrite request = rewrite.rewriteRequest;
  assertNull(request.getMaxBytesRewrittenPerCall());
  assertEquals("bucket", request.getSourceBucket());
  assertEquals("s0", request.getSourceObject());
  assertEquals("bucket", request.getDestinationBucket());
  assertEquals("d0", request.getDestinationObject());
}
 

private static Bucket createBucket(
    Storage storage, String projectId, String locationId, String bucketId) {
  try {
    Bucket bucket =
        new Bucket()
            .setLocation(locationId)
            .setName(bucketId)
            .setVersioning(new Bucket.Versioning().setEnabled(true));

    if (!StringUtils.isEmpty(locationId)) {
      bucket.setLocation(locationId);
    }

    return storage.buckets().insert(projectId, bucket).execute();
  } catch (IOException e) {
    throw new RuntimeException("Unable to create bucket", e);
  }
}
 

@Override
protected InputStream downloadRemoteFile(EncryptedSecret encryptedSecret) {

  String bucket = encryptedSecret.getParams().get(STORAGE_BUCKET);
  String objName = encryptedSecret.getParams().get(STORAGE_FILE_URI);

  log.info("Getting contents of object {} from bucket {}", objName, bucket);

  try {
    Storage storage = getStorage();

    return storage.objects().get(bucket, objName).executeMediaAsInputStream();
  } catch (IOException e) {
    throw new SecretException(
        String.format(
            "Error reading contents of GCS. Bucket: %s, Object: %s.\nError: %s",
            bucket, objName, e.toString()));
  }
}
 

@Test
public void lazyInitGeneration_succeeds_whenReadConsistencyStrict() throws IOException {
  MockHttpTransport transport =
      mockTransport(
          jsonDataResponse(newStorageObject(BUCKET_NAME, OBJECT_NAME).setGeneration(5L)));

  List<HttpRequest> requests = new ArrayList<>();

  Storage storage = new Storage(transport, JSON_FACTORY, requests::add);

  GoogleCloudStorageReadOptions options =
      GoogleCloudStorageReadOptions.builder().setFastFailOnNotFound(false).build();

  GoogleCloudStorageReadChannel readChannel = createReadChannel(storage, options);
  // initialize metadata
  readChannel.size();
  assertThat(readChannel.generation()).isEqualTo(5L);
}
 

/**
 * Method to create the service or get the service if is already created.
 *
 * @author <a href="mailto:[email protected]"> João Felipe de Medeiros Moreira </a>
 * @since 13/10/2015
 *
 * @return the Storage service already created.
 *
 * @throws IOException in case a IO problem.
 * @throws GeneralSecurityException in case a security problem.
 */
private static Storage getService() throws IOException, GeneralSecurityException {
  logger.finest("###### Getting the storage service");
  if (null == storageService) {
    HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport();
    GoogleCredential credential = GoogleCredential.getApplicationDefault();
    // Depending on the environment that provides the default credentials (e.g. Compute Engine,
    // App Engine), the credentials may require us to specify the scopes we need explicitly.
    // Check for this case, and inject the Cloud Storage scope if required.
    if (credential.createScopedRequired()) {
      credential = credential.createScoped(StorageScopes.all());
    }
    storageService = new Storage.Builder(httpTransport, JSON_FACTORY, credential)
        .setApplicationName(APPLICATION_NAME).build();
  }
  return storageService;
}
 

@Test
public void metadataInitialization_eager() throws IOException {
  StorageObject object = newStorageObject(BUCKET_NAME, OBJECT_NAME);
  MockHttpTransport transport = mockTransport(jsonDataResponse(object));

  List<HttpRequest> requests = new ArrayList<>();

  Storage storage = new Storage(transport, JSON_FACTORY, requests::add);

  GoogleCloudStorageReadOptions options =
      GoogleCloudStorageReadOptions.builder().setFastFailOnNotFound(true).build();

  GoogleCloudStorageReadChannel readChannel = createReadChannel(storage, options);

  assertThat(requests).hasSize(1);
  assertThat(readChannel.size()).isEqualTo(object.getSize().longValue());
  assertThat(requests).hasSize(1);
}
 

public Storage getStorage() throws IOException {
  HttpTransport httpTransport = buildHttpTransport();
  JsonFactory jsonFactory = JacksonFactory.getDefaultInstance();
  GoogleCredentials credentials = getCredentials(StorageScopes.all());
  HttpRequestInitializer reqInit = setHttpTimeout(credentials);
  String applicationName = "Spinnaker/" + applicationVersion;

  return new Storage.Builder(httpTransport, jsonFactory, reqInit)
      .setApplicationName(applicationName)
      .build();
}
 

private <T> T deserialize(Storage storage, StorageObject object, TypeReference<T> typeReference)
    throws IOException {
  ByteArrayOutputStream output = new java.io.ByteArrayOutputStream();
  Storage.Objects.Get getter = storage.objects().get(object.getBucket(), object.getName());
  getter.executeMediaAndDownloadTo(output);
  String json = output.toString("UTF8");

  return kayentaObjectMapper.readValue(json, typeReference);
}
 

public static ReaderAndIndex openBAMAndExposeIndex(Storage.Objects storageClient, String gcsStoragePath, ValidationStringency stringency) throws IOException {
  ReaderAndIndex result = new ReaderAndIndex();
  result.index = openIndexForPath(storageClient, gcsStoragePath);
  result.reader = openBAMReader(
      openBAMFile(storageClient, gcsStoragePath,result.index), stringency, false, 0);
  return result;
}
 

@Test
public void lazyReadFileAtSpecificGeneration_fails_whenReadConsistencyStrict()
    throws IOException {
  long generation = 5L;
  MockHttpTransport transport = mockTransport(jsonErrorResponse(ErrorResponses.NOT_FOUND));

  List<HttpRequest> requests = new ArrayList<>();

  Storage storage = new Storage(transport, JSON_FACTORY, requests::add);

  GoogleCloudStorageReadOptions options =
      GoogleCloudStorageReadOptions.builder().setFastFailOnNotFound(false).build();
  GoogleCloudStorageReadChannel readChannel = createReadChannel(storage, options, generation);
  assertThrows(FileNotFoundException.class, () -> readChannel.size());
}
 

public static Storage buildStorage(CloudCredential gcpCredential, String name) {
    try {
        HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport();
        GoogleCredential credential = buildCredential(gcpCredential, httpTransport);
        return new Builder(
                httpTransport, JSON_FACTORY, null).setApplicationName(name)
                .setHttpRequestInitializer(credential)
                .build();
    } catch (Exception e) {
        LOGGER.warn("Error occurred while building Google Storage access.", e);
    }
    return null;
}
 

private static void checkGcsUrlExists(String url) throws IOException {
  // Ensure data is accessible.
  // If we can read the size, then surely we can read the file.
  GcsPath fn = GcsPath.fromUri(url);
  Storage.Objects storageClient = GCSOptions.Methods.createStorageClient(pipelineOptions, auth);
  Storage.Objects.Get getter = storageClient.get(fn.getBucket(), fn.getObject());
  StorageObject object = getter.execute();
  BigInteger size = object.getSize();
}
 

/**
 * Returns an instance of {@link GcsUtil} based on the {@link PipelineOptions}.
 *
 * <p>If no instance has previously been created, one is created and the value stored in {@code
 * options}.
 */
@Override
public GcsUtil create(PipelineOptions options) {
  LOG.debug("Creating new GcsUtil");
  GcsOptions gcsOptions = options.as(GcsOptions.class);
  Storage.Builder storageBuilder = Transport.newStorageClient(gcsOptions);
  return new GcsUtil(
      storageBuilder.build(),
      storageBuilder.getHttpRequestInitializer(),
      gcsOptions.getExecutorService(),
      hasExperiment(options, "use_grpc_for_gcs"),
      gcsOptions.getGcsUploadBufferSizeBytes());
}
 

/** Returns an instance of {@link GcsUtil} based on the given parameters. */
public static GcsUtil create(
    PipelineOptions options,
    Storage storageClient,
    HttpRequestInitializer httpRequestInitializer,
    ExecutorService executorService,
    @Nullable Integer uploadBufferSizeBytes) {
  return new GcsUtil(
      storageClient,
      httpRequestInitializer,
      executorService,
      hasExperiment(options, "use_grpc_for_gcs"),
      uploadBufferSizeBytes);
}