下面列出了com.google.protobuf.ByteString#newInput ( ) 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
@Test
public void nonEmptyDigestIsDelegated() throws IOException, InterruptedException {
ByteString content = ByteString.copyFromUtf8("Hello, World");
Digest contentDigest = DIGEST_UTIL.compute(content);
EmptyInputStreamFactory emptyFactory =
new EmptyInputStreamFactory(
new InputStreamFactory() {
@Override
public InputStream newInput(Digest digest, long offset) throws IOException {
if (digest.equals(contentDigest)) {
return content.newInput();
}
throw new IOException("invalid");
}
});
InputStream in = emptyFactory.newInput(contentDigest, /* offset=*/ 0);
assertThat(ByteString.readFrom(in)).isEqualTo(content);
}
@Override
public OperationContext getClone() throws IOException, ClassNotFoundException{
ByteArrayOutputStream baos = new ByteArrayOutputStream();
ObjectOutputStream oos = new ObjectOutputStream(baos);
oos.writeObject(this);
oos.flush();
oos.close();
ByteString bs = ZeroCopyLiteralByteString.wrap(baos.toByteArray());
// Deserialize activation to clone it
try (InputStream is = bs.newInput();
ObjectInputStream ois = new ObjectInputStream(is)) {
SparkLeanOperationContext operationContext = (SparkLeanOperationContext) ois.readObject();
BroadcastedActivation broadcastedActivation = operationContext.broadcastedActivation;
BroadcastedActivation.ActivationHolderAndBytes activationHolderAndBytes = broadcastedActivation.readActivationHolder();
broadcastedActivation.setActivationHolder(activationHolderAndBytes.getActivationHolder());
operationContext.op = broadcastedActivation.getActivationHolder().getOperationsMap().get(op.resultSetNumber());
operationContext.activation = operationContext.broadcastedActivation.getActivationHolder().getActivation();
return operationContext;
}
}
@Override
public OperationContext getClone() throws IOException, ClassNotFoundException{
ByteArrayOutputStream baos = new ByteArrayOutputStream();
ObjectOutputStream oos = new ObjectOutputStream(baos);
oos.writeObject(this);
oos.flush();
oos.close();
ByteString bs = ZeroCopyLiteralByteString.wrap(baos.toByteArray());
// Deserialize activation to clone it
try (InputStream is = bs.newInput();
ObjectInputStream ois = new ObjectInputStream(is)) {
SparkOperationContext operationContext = (SparkOperationContext) ois.readObject();
BroadcastedActivation broadcastedActivation = operationContext.broadcastedActivation;
BroadcastedActivation.ActivationHolderAndBytes activationHolderAndBytes = broadcastedActivation.readActivationHolder();
broadcastedActivation.setActivationHolder(activationHolderAndBytes.getActivationHolder());
operationContext.op = broadcastedActivation.getActivationHolder().getOperationsMap().get(op.resultSetNumber());
operationContext.activation = operationContext.broadcastedActivation.getActivationHolder().getActivation();
return operationContext;
}
}
/**
* Reads a list of protos, using the provided parser, from the provided {@link ByteString}.
* @throws IOException if the proto list could not be parsed.
*/
public static <T extends MessageLite> List<T> readMessageList(
ByteString bytes,
Parser<T> parser)
throws IOException {
InputStream stream = bytes.newInput();
return readMessageList(stream, parser);
}
private void advance() throws IOException {
ByteString data = ByteString.EMPTY;
while (hasNext() && data.isEmpty()) {
try {
data = queue.take();
} catch (InterruptedException e) {
throw new IOException(e);
}
}
input = data.newInput();
}
@Override
public ListenableFuture<Void> uploadBlob(Digest digest, ByteString data) {
try (InputStream in = data.newInput()) {
uploadBlocking(digest.getHash(), digest.getSizeBytes(), in, /* casUpload= */ true);
} catch (IOException | InterruptedException e) {
return Futures.immediateFailedFuture(e);
}
return Futures.immediateFuture(null);
}
@Override
public void uploadActionResult(ActionKey actionKey, ActionResult actionResult)
throws IOException, InterruptedException {
ByteString serialized = actionResult.toByteString();
try (InputStream in = serialized.newInput()) {
uploadBlocking(actionKey.getDigest().getHash(), serialized.size(), in, false);
}
}
@Override
public ListenableFuture<Void> uploadBlob(Digest digest, ByteString data) {
try (InputStream in = data.newInput()) {
saveFile(digest.getHash(), in);
} catch (IOException e) {
return Futures.immediateFailedFuture(e);
}
return Futures.immediateFuture(null);
}
@Override
public ListenableFuture<Void> uploadBlob(Digest digest, ByteString data) {
try (InputStream in = data.newInput()) {
cas.put(digest, data.toByteArray());
} catch (IOException e) {
return Futures.immediateFailedFuture(e);
}
return Futures.immediateFuture(null);
}
/**
* Deserialize from a byte string
*/
public UtxoTrieNode(ByteString bs)
{
try
{
DataInputStream din=new DataInputStream(bs.newInput());
long ver = din.readLong();
Assert.assertEquals(serialVersionUID, ver);
prefix = SerialUtil.readString(din);
int count = din.readInt();
springs = new TreeMap<String, Sha256Hash>();
for(int i=0; i<count; i++)
{
byte hash_bytes[]=new byte[32];
String sub = SerialUtil.readString(din);
din.readFully(hash_bytes);
Sha256Hash hash = new Sha256Hash(hash_bytes);
if (hash.equals(hash_null))
{
hash=null;
springs.put(sub, null);
}
else
{
springs.put(sub, hash);
}
}
}
catch(java.io.IOException e)
{
throw new RuntimeException(e);
}
}
private void saveFile(String key, ByteString data, int cache_seconds)
{
ObjectMetadata omd = new ObjectMetadata();
omd.setCacheControl("max-age=" + cache_seconds);
omd.setContentLength(data.size());
PutObjectRequest put = new PutObjectRequest(bucket, key, data.newInput(), omd);
put.setCannedAcl(CannedAccessControlList.PublicRead);
put.setStorageClass(com.amazonaws.services.s3.model.StorageClass.StandardInfrequentAccess.toString());
s3.putObject(put);
}
/**
* Convert compressed byte string to a Configuration object using protocol
* buffer
*
* @param byteString
* :compressed conf in Protocol buffer
* @return Configuration
* @throws IOException
*/
public static Configuration createConfFromByteString(ByteString byteString) throws IOException {
Preconditions.checkNotNull(byteString, "ByteString must be specified");
// SnappyInputStream uncompressIs = new
// SnappyInputStream(byteString.newInput());
InflaterInputStream uncompressIs = new InflaterInputStream(byteString.newInput());
ConfigurationProto confProto = ConfigurationProto.parseFrom(uncompressIs);
Configuration conf = new Configuration(false);
readConfFromPB(confProto, conf);
return conf;
}
@SuppressWarnings("unchecked")
public static <R extends Serializable> R decode(ByteString commandBytes) throws IOException{
InputStream is = commandBytes.newInput();
ObjectInputStream ois = new ObjectInputStream(is);
try{
return (R)ois.readObject(); //shouldn't be a problem with any IOExceptions
}catch(ClassNotFoundException e){
throw new IOException(e); //shouldn't happen
}
}
@Private
public static byte[] decompressByteStringToByteArray(ByteString byteString, Inflater inflater) throws IOException {
inflater.reset();
try (InflaterInputStream inflaterInputStream = new InflaterInputStream(byteString.newInput(), inflater)) {
return IOUtils.toByteArray(inflaterInputStream);
}
}
/**
* Convert a byte string to a Configuration object
*
* @param byteString byteString representation of the conf created using {@link
* #createByteStringFromConf(org.apache.hadoop.conf.Configuration)}
* @return Configuration
* @throws java.io.IOException
*/
public static Configuration createConfFromByteString(ByteString byteString) throws IOException {
Objects.requireNonNull(byteString, "ByteString must be specified");
try(SnappyInputStream uncompressIs = new SnappyInputStream(byteString.newInput());) {
CodedInputStream in = CodedInputStream.newInstance(uncompressIs);
in.setSizeLimit(Integer.MAX_VALUE);
DAGProtos.ConfigurationProto confProto = DAGProtos.ConfigurationProto.parseFrom(in);
Configuration conf = new Configuration(false);
readConfFromPB(confProto, conf);
return conf;
}
}
public static Configuration createConfFromBaseConfAndPayload(TaskContext context)
throws IOException {
Configuration baseConf = context.getContainerConfiguration();
Configuration configuration = new Configuration(baseConf);
UserPayload payload = context.getUserPayload();
ByteString byteString = ByteString.copyFrom(payload.getPayload());
try(SnappyInputStream uncompressIs = new SnappyInputStream(byteString.newInput())) {
DAGProtos.ConfigurationProto confProto = DAGProtos.ConfigurationProto.parseFrom(uncompressIs);
readConfFromPB(confProto, configuration);
return configuration;
}
}
public static void addToConfFromByteString(Configuration configuration, ByteString byteString)
throws IOException {
try(SnappyInputStream uncompressIs = new SnappyInputStream(byteString.newInput())) {
DAGProtos.ConfigurationProto confProto = DAGProtos.ConfigurationProto.parseFrom(uncompressIs);
readConfFromPB(confProto, configuration);
}
}
@Test
public void getArtifactFromCloudTest() {
LOGGER.info("get artifact from cloud test start................................");
try {
ArtifactStoreBlockingStub artifactStoreBlockingStub =
ArtifactStoreGrpc.newBlockingStub(channel);
StoreArtifact storeArtifact =
StoreArtifact.newBuilder()
.setKey("verta_logo.png")
.setPath(
"https://www.verta.ai/static/logo-landing-424af27a5fc184c64225f604232db39e.png")
.build();
StoreArtifact.Response response = artifactStoreBlockingStub.storeArtifact(storeArtifact);
String cloudFileKey = response.getArtifactStoreKey();
String cloudFilePath = response.getArtifactStorePath();
LOGGER.log(
Level.INFO,
"StoreArtifact.Response : \n cloudFileKey - "
+ cloudFileKey
+ " \n cloudFilePath - "
+ cloudFilePath);
assumeTrue(cloudFileKey != null && !cloudFileKey.isEmpty());
assumeTrue(cloudFilePath != null && !cloudFilePath.isEmpty());
GetArtifact getArtifactRequest = GetArtifact.newBuilder().setKey(cloudFileKey).build();
GetArtifact.Response getArtifactResponse =
artifactStoreBlockingStub.getArtifact(getArtifactRequest);
ByteString responseByteString = getArtifactResponse.getContents();
InputStream inputStream = responseByteString.newInput();
String rootPath = System.getProperty("user.dir");
FileOutputStream fileOutputStream =
new FileOutputStream(new File(rootPath + File.separator + cloudFileKey));
IOUtils.copy(inputStream, fileOutputStream);
fileOutputStream.close();
inputStream.close();
File downloadedFile = new File(rootPath + File.separator + cloudFileKey);
if (!downloadedFile.exists()) {
fail("File not fount at download destination");
}
downloadedFile.delete();
DeleteArtifact deleteArtifact = DeleteArtifact.newBuilder().setKey(cloudFileKey).build();
DeleteArtifact.Response deleteArtifactResponse =
artifactStoreBlockingStub.deleteArtifact(deleteArtifact);
assertTrue(deleteArtifactResponse.getStatus());
} catch (Exception e) {
e.printStackTrace();
Status status = Status.fromThrowable(e);
LOGGER.warning(
"Error Code : " + status.getCode() + " Description : " + status.getDescription());
fail();
}
LOGGER.info("get artifact from cloud test stop................................");
}
@Private
public static byte[] decompressByteStringToByteArray(ByteString byteString) throws IOException {
InflaterInputStream in = new InflaterInputStream(byteString.newInput());
byte[] bytes = IOUtils.toByteArray(in);
return bytes;
}