public ExampleRecordCursor(List<ExampleColumnHandle> columnHandles, ByteSource byteSource)
{
    this.columnHandles = columnHandles;

    fieldToColumnIndex = new int[columnHandles.size()];
    for (int i = 0; i < columnHandles.size(); i++) {
        ExampleColumnHandle columnHandle = columnHandles.get(i);
        fieldToColumnIndex[i] = columnHandle.getOrdinalPosition();
    }

    try (CountingInputStream input = new CountingInputStream(byteSource.openStream())) {
        lines = byteSource.asCharSource(UTF_8).readLines().iterator();
        totalBytes = input.getCount();
    }
    catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}
 

/**
 * Adds a watch on the specified file. The file must exist, otherwise a FileNotFoundException
 * is returned. If the file is deleted after a watch is established, the watcher will log errors
 * but continue to monitor it, and resume watching if it is recreated.
 *
 * @param filePath path to the file to watch.
 * @param onUpdate function to call when a change is detected to the file. The entire contents
 *                 of the file will be passed in to the function. Note that onUpdate will be
 *                 called once before this call completes, which facilities initial load of data.
 *                 This callback is executed synchronously on the watcher thread - it is
 *                 important that the function be non-blocking.
 */
public synchronized void addWatch(String filePath, Function<byte[], Void> onUpdate)
    throws IOException {
  MorePreconditions.checkNotBlank(filePath);
  Preconditions.checkNotNull(onUpdate);

  // Read the file and make the initial onUpdate call.
  File file = new File(filePath);
  ByteSource byteSource = Files.asByteSource(file);
  onUpdate.apply(byteSource.read());

  // Add the file to our map if it isn't already there, and register the new change watcher.
  ConfigFileInfo configFileInfo = watchedFileMap.get(filePath);
  if (configFileInfo == null) {
    configFileInfo = new ConfigFileInfo(file.lastModified(), byteSource.hash(HASH_FUNCTION));
    watchedFileMap.put(filePath, configFileInfo);
  }
  configFileInfo.changeWatchers.add(onUpdate);
}
 

private void generatePutTempURL() throws IOException {
   System.out.format("Generate PUT Temp URL%n");

   // Create the Payload
   String data = "This object will be public for 10 minutes.";
   ByteSource source = ByteSource.wrap(data.getBytes());
   Payload payload = Payloads.newByteSourcePayload(source);

   // Create the Blob
   Blob blob = blobStore.blobBuilder(FILENAME).payload(payload).contentType("text/plain").build();
   HttpRequest request = blobStoreContext.getSigner(REGION).signPutBlob(CONTAINER, blob, TEN_MINUTES);

   System.out.format("  %s %s%n", request.getMethod(), request.getEndpoint());

   // PUT the file using jclouds
   HttpResponse response = blobStoreContext.utils().http().invoke(request);
   int statusCode = response.getStatusCode();

   if (statusCode >= 200 && statusCode < 299) {
      System.out.format("  PUT Success (%s)%n", statusCode);
   }
   else {
      throw new HttpResponseException(null, response);
   }
}
 

/**
 * Uploads a byte array using FileResource and ExternalFileResource
 *
 * @param name  name of the file to be stored
 * @param bytes the byte array representing the file to be stored
 * @return url pointing to the uploaded resource
 */
private String uploadImage( String name, byte[] bytes )
    throws IOException
{
    FileResource fileResource = new FileResource(
        name,
        MimeTypeUtils.IMAGE_PNG.toString(), // All files uploaded from PushAnalysis is PNG.
        bytes.length,
        ByteSource.wrap( bytes ).hash( Hashing.md5() ).toString(),
        FileResourceDomain.PUSH_ANALYSIS
    );

    String accessToken = saveFileResource( fileResource, bytes );

    return dhisConfigurationProvider.getServerBaseUrl() + "/api/externalFileResources/" + accessToken;

}
 

/**
 * Upload a large object from a File using the BlobStore API.
 *
 * @throws ExecutionException
 * @throws InterruptedException
 */
private void uploadLargeObjectFromFile(File largeFile) throws InterruptedException, ExecutionException {
   System.out.format("Upload Large Object From File%n");

   ByteSource source = Files.asByteSource(largeFile);
   // create the payload and set the content length
   Payload payload = Payloads.newByteSourcePayload(source);
   payload.getContentMetadata().setContentLength(largeFile.length());

   Blob blob = blobStore.blobBuilder(largeFile.getName())
         .payload(payload)
         .build();

   // configure the blobstore to use multipart uploading of the file
   String eTag = blobStore.putBlob(CONTAINER, blob, multipart());

   System.out.format("  Uploaded %s eTag=%s", largeFile.getName(), eTag);
}
 

/**
 * This method will put a plain text object into the container.
 */
private void createObjectFromFile() throws IOException {
   System.out.format("Create Object From File%n");

   File tempFile = File.createTempFile(FILENAME, SUFFIX);

   try {
      Files.write("Hello Cloud Files", tempFile, Charsets.UTF_8);

      ObjectApi objectApi = cloudFiles.getObjectApi(REGION, CONTAINER_PUBLISH);

      ByteSource byteSource = Files.asByteSource(tempFile);
      Payload payload = Payloads.newByteSourcePayload(byteSource);

      objectApi.put(FILENAME + SUFFIX, payload);
   } finally {
      tempFile.delete();
   }
}
 

/**
 * Given a list of {@link ByteSource} computes the GZIP size increments attributed to each stream.
 */
public ImmutableList<Long> calculateGZipSizeForEntries(List<ByteSource> byteSources)
    throws IOException {
  ImmutableList.Builder<Long> gzipSizeIncrements = ImmutableList.builder();

  try (ApkGzipDeflater deflater = deflaterSupplier.get()) {
    // matches the {@code ByteStreams} buffer size
    byte[] inputBuffer = new byte[INPUT_BUFFER_SIZE];

    for (ByteSource byteSource : byteSources) {
      try (InputStream is = byteSource.openStream()) {
        while (true) {
          int r = is.read(inputBuffer);
          if (r == -1) {
            gzipSizeIncrements.add(
                Math.max(0, deflater.entryComplete() - DEFLATER_SYNC_OVERHEAD_BYTES));
            break;
          }
          deflater.handleInput(inputBuffer, r);
        }
      }
    }
  }
  return gzipSizeIncrements.build();
}
 

/**
 * Adds a watch on the specified file. The file must exist, otherwise a FileNotFoundException
 * is returned. If the file is deleted after a watch is established, the watcher will log errors
 * but continue to monitor it, and resume watching if it is recreated.
 *
 * @param filePath path to the file to watch.
 * @param onUpdate function to call when a change is detected to the file. The entire contents
 *                 of the file will be passed in to the function. Note that onUpdate will be
 *                 called once before this call completes, which facilities initial load of data.
 *                 This callback is executed synchronously on the watcher thread - it is
 *                 important that the function be non-blocking.
 */
public synchronized void addWatch(String filePath, Function<byte[], Void> onUpdate)
    throws IOException {
  MorePreconditions.checkNotBlank(filePath);
  Preconditions.checkNotNull(onUpdate);

  // Read the file and make the initial onUpdate call.
  File file = new File(filePath);
  ByteSource byteSource = Files.asByteSource(file);
  onUpdate.apply(byteSource.read());

  // Add the file to our map if it isn't already there, and register the new change watcher.
  ConfigFileInfo configFileInfo = watchedFileMap.get(filePath);
  if (configFileInfo == null) {
    configFileInfo = new ConfigFileInfo(file.lastModified(), byteSource.hash(HASH_FUNCTION));
    watchedFileMap.put(filePath, configFileInfo);
  }
  configFileInfo.changeWatchers.add(onUpdate);
}
 

public void registerSource(String identifier, String version, ByteSource byteSource)
        throws SchemaSourceException, IOException, YangSyntaxErrorException {
    YangTextSchemaSource source = YangTextSchemaSource.delegateForByteSource(
            RevisionSourceIdentifier.create(identifier, Revision.ofNullable(version)), byteSource);
    ASTSchemaSource ast = TextToASTTransformer.transformText(source);
    SourceIdentifier actualIdentifier = ast.getIdentifier();

    // Fixup YANG source identifier if the provided YANG model has a different actual identifier
    if (!source.getIdentifier().equals(actualIdentifier))
        source = YangTextSchemaSource.delegateForByteSource(actualIdentifier, byteSource);

    cache.schemaSourceEncountered(ast);

    sources.put(source.getIdentifier(), source);
    repository.registerSchemaSource(this, PotentialSchemaSource.create(
            source.getIdentifier(), YangTextSchemaSource.class, PotentialSchemaSource.Costs.IMMEDIATE.getValue()));
}
 

/**
 * Upload an object from a String with metadata using the BlobStore API.
 */
private void uploadObjectFromStringWithMetadata() {
   System.out.format("Upload Object From String With Metadata%n");

   String filename = "uploadObjectFromStringWithMetadata.txt";

   Map<String, String> userMetadata = new HashMap<String, String>();
   userMetadata.put("key1", "value1");

   ByteSource source = ByteSource.wrap("uploadObjectFromString".getBytes());

   Blob blob = blobStore.blobBuilder(filename)
         .payload(Payloads.newByteSourcePayload(source))
         .userMetadata(userMetadata)
         .build();

   blobStore.putBlob(CONTAINER, blob);

   System.out.format("  %s%n", filename);
}
 

private void sendMessage( final WebSocketSession session, final Object message )
{
    if ( session == null )
    {
        return;
    }

    if ( message == null )
    {
        return;
    }

    if ( message instanceof ByteSource )
    {
        session.send( (ByteSource) message );
    }
    else
    {
        session.send( message.toString() );
    }
}
 

@Test
public void shouldWrapOverOldBlocks() throws Exception {
    // given
    // use random text so that the lzf compressed text is also large and forces wrapping
    Random random = new Random();
    StringBuilder sb = new StringBuilder();
    for (int i = 0; i < 600; i++) {
        sb.append((char) ('a' + random.nextInt(26)));
    }
    String text = sb.toString();
    long cappedId = cappedDatabase.write(ByteSource.wrap(text.getBytes(UTF_8)), "test");
    cappedDatabase.write(ByteSource.wrap(text.getBytes(UTF_8)), "test");

    // when
    cappedDatabase.write(ByteSource.wrap(text.getBytes(UTF_8)), "test");

    // then
    String exceptionClassName = null;
    try {
        cappedDatabase.read(cappedId).read();
    } catch (Exception e) {
        exceptionClassName = e.getClass().getName();
    }
    assertThat(exceptionClassName).isEqualTo("org.glowroot.agent.embedded.util.CappedDatabase"
            + "$CappedBlockRolledOverMidReadException");
}
 

@Test
public void testHandleErrorResponse_fails() throws Exception {
  RawReportDownloadResponse rawResponse =
      new RawReportDownloadResponse(
          500,
          ByteSource.wrap(ERROR_XML.getBytes(REPORT_CHARSET)).openStream(),
          REPORT_CHARSET,
          "CSV");
  thrown.expect(DetailedReportDownloadResponseException.class);
  thrown.expect(Matchers.hasProperty("fieldPath", Matchers.equalTo("foobar")));
  thrown.expect(Matchers.hasProperty("trigger", Matchers.equalTo("AdFormatt")));
  thrown.expect(
      Matchers.hasProperty(
          "type", Matchers.equalTo("ReportDefinitionError.INVALID_FIELD_NAME_FOR_REPORT")));
  helper.handleResponse(rawResponse, exceptionBuilder);
}
 

@Test
public void testCreateMetadataHeader() throws IOException {
  String base64EncodedData =
      "AAAAAQAgMDAwMDAwMDAwMTAwMDAwMDAwMDAwMDgwMDAwMDAwMDAAAAABAANrZXkAAAAFdmFsdWVc/GBY";
  RuleKey ruleKey = new RuleKey("00000000010000000000008000000000");
  String data = "data";
  byte[] metadata =
      HttpArtifactCacheBinaryProtocol.createMetadataHeader(
          ImmutableSet.of(ruleKey),
          ImmutableMap.of("key", "value"),
          ByteSource.wrap(data.getBytes(Charsets.UTF_8)));
  assertThat(metadata, Matchers.equalTo(BaseEncoding.base64().decode(base64EncodedData)));
}
 

@Override
public OutputStream openOutputStream() throws IOException {
	return new ByteArrayOutputStream() {
		@Override
		public void close() throws IOException {
			super.close();
			data = Optional.of(ByteSource.wrap(toByteArray()));
			lastModified = System.currentTimeMillis();
		}
	};
}
 

private static ModuleEntry createEntryForAbi(Abi abi) {
  return ModuleEntry.builder()
      .setPath(
          BundleModule.LIB_DIRECTORY
              .resolve(AbiName.fromProto(abi.getAlias()).getPlatformName())
              .resolve("libplaceholder.so"))
      .setContent(ByteSource.wrap(new byte[0]))
      .build();
}
 

@Test
public void testStoreRequest() throws IOException {
  RuleKey ruleKey = new RuleKey("00000000010000000000008000000000");
  RuleKey ruleKey2 = new RuleKey("90000000000000000000008000000005");
  String data = "data";
  ImmutableMap<String, String> metadata = ImmutableMap.of("metaKey", "metaValue");

  HttpArtifactCacheBinaryProtocol.StoreRequest storeRequest =
      new HttpArtifactCacheBinaryProtocol.StoreRequest(
          ArtifactInfo.builder().addRuleKeys(ruleKey, ruleKey2).setMetadata(metadata).build(),
          new ByteSource() {
            @Override
            public InputStream openStream() {
              return new ByteArrayInputStream(data.getBytes(Charsets.UTF_8));
            }
          });

  ByteArrayOutputStream storeRequestOutputStream = new ByteArrayOutputStream();
  storeRequest.write(storeRequestOutputStream);

  ByteArrayOutputStream storeRequestPayloadStream = new ByteArrayOutputStream();
  HttpArtifactCacheBinaryProtocol.StoreResponseReadResult readStoreRequest =
      HttpArtifactCacheBinaryProtocol.readStoreRequest(
          new DataInputStream(new ByteArrayInputStream(storeRequestOutputStream.toByteArray())),
          storeRequestPayloadStream);

  assertThat(readStoreRequest.getRuleKeys(), Matchers.containsInAnyOrder(ruleKey, ruleKey2));
  assertThat(readStoreRequest.getMetadata(), Matchers.equalTo(metadata));
  assertThat(
      storeRequestPayloadStream.toByteArray(), Matchers.equalTo(data.getBytes(Charsets.UTF_8)));
}
 

@Test
default void saveShouldThrowWhenNullByteSource() {
    DumbBlobStore store = testee();

    assertThatThrownBy(() -> Mono.from(store.save(TEST_BUCKET_NAME, TEST_BLOB_ID, (ByteSource) null)).block())
        .isInstanceOf(NullPointerException.class);
}
 

@Override
public OutputStream openOutputStream() throws IOException {
  return new ByteArrayOutputStream() {
    @Override
    public void close() throws IOException {
      super.close();
      data = Optional.of(ByteSource.wrap(toByteArray()));
      lastModified = System.currentTimeMillis();
    }
  };
}
 

/**
 * Decorates a map using the provided algorithms.
 * <p>Takes a salt and secretKey so that it can work with a distributed cache.
 *
 * @param decoratedMap the map to decorate.  CANNOT be NULL.
 * @param hashAlgorithm the algorithm to use for hashing.  CANNOT BE NULL.
 * @param salt the salt, as a String. Gets converted to bytes.   CANNOT be NULL.
 * @param secretKeyAlgorithm the encryption algorithm. CANNOT BE NULL.
 * @param secretKey the secret to use.  CANNOT be NULL.
 * @throws RuntimeException if the algorithm cannot be found or the iv size cant be determined.
 */
public EncryptedMapDecorator(final Map<String, String> decoratedMap, final String hashAlgorithm, final byte[] salt,
        final String secretKeyAlgorithm, final Key secretKey) {
    try {
        this.decoratedMap = decoratedMap;
        this.key = secretKey;
        this.salt = ByteSource.wrap(salt);
        this.secretKeyAlgorithm = secretKeyAlgorithm;
        this.messageDigest = MessageDigest.getInstance(hashAlgorithm);
        this.ivSize = getIvSize();
    } catch (final Exception e) {
        throw new RuntimeException(e);
    }
}
 

@Test
public void testFetchOK() throws Exception {
  Path output = Paths.get("output/file");
  String data = "test";
  RuleKey ruleKey = new RuleKey("00000000000000000000000000000000");
  FakeProjectFilesystem filesystem = new FakeProjectFilesystem();
  List<Response> responseList = new ArrayList<>();
  argsBuilder.setProjectFilesystem(filesystem);
  argsBuilder.setFetchClient(
      withMakeRequest(
          (path, requestBuilder) -> {
            Request request = requestBuilder.url(SERVER + path).build();
            Response response =
                new Response.Builder()
                    .request(request)
                    .protocol(Protocol.HTTP_1_1)
                    .code(HttpURLConnection.HTTP_OK)
                    .body(
                        createResponseBody(
                            ImmutableSet.of(ruleKey),
                            ImmutableMap.of(),
                            ByteSource.wrap(data.getBytes(Charsets.UTF_8)),
                            data))
                    .message("")
                    .build();
            responseList.add(response);
            return new OkHttpResponseWrapper(response);
          }));

  HttpArtifactCache cache = new HttpArtifactCache(argsBuilder.build());
  CacheResult result =
      Futures.getUnchecked(cache.fetchAsync(null, ruleKey, LazyPath.ofInstance(output)));
  assertEquals(result.cacheError().orElse(""), CacheResultType.HIT, result.getType());
  assertEquals(Optional.of(data), filesystem.readFileIfItExists(output));
  assertEquals(result.artifactSizeBytes(), Optional.of(filesystem.getFileSize(output)));
  assertTrue("response wasn't fully read!", responseList.get(0).body().source().exhausted());
  cache.close();
}
 

@ParameterizedTest(name = "[{index}] {0}")
@MethodSource("blobs")
default void saveInputStreamShouldBeIdempotent(String description, byte[] bytes) {
    DumbBlobStore store = testee();
    Mono.from(store.save(TEST_BUCKET_NAME, TEST_BLOB_ID, ByteSource.wrap(bytes))).block();
    Mono.from(store.save(TEST_BUCKET_NAME, TEST_BLOB_ID, new ByteArrayInputStream(bytes))).block();

    byte[] read = Mono.from(store.readBytes(TEST_BUCKET_NAME, TEST_BLOB_ID)).block();

    assertThat(read).isEqualTo(bytes);
}
 

@Override
public MultipartPart uploadMultipartPart(MultipartUpload mpu,
        int partNumber, Payload payload) {
    long length;
    try (InputStream is = payload.openStream()) {
        length = ByteStreams.copy(is, ByteStreams.nullOutputStream());
    } catch (IOException ioe) {
        throw new RuntimeException(ioe);
    }

    byte[] array = Longs.toByteArray(length);
    ByteSourcePayload newPayload = new ByteSourcePayload(
            ByteSource.wrap(array));
    newPayload.setContentMetadata(payload.getContentMetadata());
    newPayload.getContentMetadata().setContentLength((long) array.length);
    newPayload.getContentMetadata().setContentMD5((HashCode) null);

    // create a single-part object which contains the logical length which
    // list and complete will read later
    Blob blob = blobBuilder(mpu.id() + "-" + partNumber)
            .payload(newPayload)
            .build();
    super.putBlob(mpu.containerName(), blob);

    MultipartPart part = super.uploadMultipartPart(mpu, partNumber,
            newPayload);
    return MultipartPart.create(part.partNumber(), length, part.partETag(),
            part.lastModified());
}
 

@Test
default void saveInputStreamShouldNotOverwritePreviousDataOnFailingInputStream() {
    DumbBlobStore store = testee();

    Mono.from(store.save(TEST_BUCKET_NAME, TEST_BLOB_ID, ByteSource.wrap(ELEVEN_KILOBYTES))).block();
    Mono.from(store.save(TEST_BUCKET_NAME, TEST_BLOB_ID, getThrowingInputStream()))
        .onErrorResume(throwable -> Mono.empty()).block();

    byte[] read = Mono.from(store.readBytes(TEST_BUCKET_NAME, TEST_BLOB_ID)).block();

    assertThat(read).isEqualTo(ELEVEN_KILOBYTES);
}
 

/**
 * Copis bytes from the source to target path.
 */
private static void copySourceToPath(ByteSource source, Path dest) {
    try (InputStream is = source.openBufferedStream()) {
        Files.copy(is, dest, StandardCopyOption.REPLACE_EXISTING);
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}
 

@Test
public void contingenciesProviderPreprocessor() {
    ContingenciesProvider provider = Mockito.mock(ContingenciesProvider.class);
    ContingenciesProviderFactory providerFactory = () -> provider;
    SecurityAnalysisPreprocessorFactory factory = new ContingenciesProviderPreprocessorFactory(providerFactory);

    assertEquals("default", factory.getName());
    SecurityAnalysisPreprocessor preprocessor = factory.newPreprocessor(ByteSource.wrap("".getBytes()));

    SecurityAnalysisInput input = new SecurityAnalysisInput(Mockito.mock(Network.class), "variant");
    preprocessor.preprocess(input);

    assertSame(provider, input.getContingenciesProvider());
}
 

private ByteSource toByteSource( final Object value )
{
    if ( value instanceof ByteSource )
    {
        return (ByteSource) value;
    }

    return newStream( value.toString() );
}
 

@Override
public ImmutableList<Step> getBuildSteps(
    BuildContext context, BuildableContext buildableContext) {
  buildableContext.recordArtifact(output);
  ProjectFilesystem projectFilesystem = getProjectFilesystem();
  return ImmutableList.of(
      MkdirStep.of(
          BuildCellRelativePath.fromCellRelativePath(
              context.getBuildCellRootPath(), getProjectFilesystem(), output.getParent())),
      new WriteFileStep(projectFilesystem, ByteSource.wrap(fileContents), output, executable));
}
 

static ByteSource asByteSource(final JavaFileObject javaFileObject) {
  return new ByteSource() {
    @Override public InputStream openStream() throws IOException {
      return javaFileObject.openInputStream();
    }
  };
}
 

@Test
public void testFetchMetadata() {
  Path output = Paths.get("output/file");
  String data = "test";
  RuleKey ruleKey = new RuleKey("00000000000000000000000000000000");
  ImmutableMap<String, String> metadata = ImmutableMap.of("some", "metadata");
  argsBuilder.setFetchClient(
      withMakeRequest(
          ((path, requestBuilder) -> {
            Request request = requestBuilder.url(SERVER + path).build();
            Response response =
                new Response.Builder()
                    .request(request)
                    .protocol(Protocol.HTTP_1_1)
                    .code(HttpURLConnection.HTTP_OK)
                    .message("")
                    .body(
                        createResponseBody(
                            ImmutableSet.of(ruleKey),
                            metadata,
                            ByteSource.wrap(data.getBytes(Charsets.UTF_8)),
                            data))
                    .build();
            return new OkHttpResponseWrapper(response);
          })));
  HttpArtifactCache cache = new HttpArtifactCache(argsBuilder.build());
  CacheResult result =
      Futures.getUnchecked(cache.fetchAsync(null, ruleKey, LazyPath.ofInstance(output)));
  assertEquals(CacheResultType.HIT, result.getType());
  assertEquals(metadata, result.getMetadata());
  cache.close();
}