com.google.protobuf.DynamicMessage#newBuilder ( )源码实例Demo

@Nonnull
@Override
public byte[] serialize(@Nonnull RecordMetaData metaData,
                        @Nonnull RecordType recordType,
                        @Nonnull Message record,
                        @Nullable StoreTimer timer) {
    long startTime = System.nanoTime();
    try {
        // Wrap in union message, if needed.
        Message storedRecord = record;
        Descriptors.Descriptor unionDescriptor = metaData.getUnionDescriptor();
        if (unionDescriptor != null) {
            DynamicMessage.Builder unionBuilder = DynamicMessage.newBuilder(unionDescriptor);
            Descriptors.FieldDescriptor unionField = metaData.getUnionFieldForRecordType(recordType);
            unionBuilder.setField(unionField, record);
            storedRecord = unionBuilder.build();
        }
        return serializeToBytes(storedRecord);
    } finally {
        if (timer != null) {
            timer.recordSinceNanoTime(Events.SERIALIZE_PROTOBUF_RECORD, startTime);
        }
    }
}
 

@Override
public DynamicMessage encode(RowData rowData) {
    RowDataValueSource value = new RowDataValueSource();
    DynamicMessage.Builder builder = DynamicMessage.newBuilder(messageType);
    for (int i = 0; i < fields.length; i++) {
        value.bind(rowDef.getFieldDef(i), rowData);
        if (value.isNull()) {
            if (nullFields != null) {
                FieldDescriptor nullField = nullFields[i];
                if (nullField != null) {
                    builder.setField(nullField, Boolean.TRUE);
                }
            }
        }
        else {
            conversions[i].setValue(builder, fields[i], value);
        }
    }
    return builder.build();
}
 

@Override
public DynamicMessage encode(Row row) {
    DynamicMessage.Builder builder = DynamicMessage.newBuilder(messageType);
    for (int i = 0; i < fields.length; i++) {
        if (row.value(i).isNull()) {
            if (nullFields != null) {
                FieldDescriptor nullField = nullFields[i];
                if (nullField != null) {
                    builder.setField(nullField, Boolean.TRUE);
                }
            }
        } else {
            conversions[i].setValue(builder, fields[i], row.value(i));
        }
    }
    return builder.build();
}
 

public ProtobufDataParser(
    ProtoConfigurableEntity.Context context,
    String messageId,
    Descriptors.Descriptor descriptor,
    Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
    ExtensionRegistry extensionRegistry,
    InputStream inputStream,
    String readerOffset,
    int maxObjectLength,
    boolean isDelimited
) throws IOException, Descriptors.DescriptorValidationException, DataParserException {
  this.context = context;
  this.inputStream = new OverrunInputStream(inputStream, maxObjectLength, true);
  this.messageId = messageId;
  this.messageTypeToExtensionMap = messageTypeToExtensionMap;
  this.extensionRegistry = extensionRegistry;
  this.descriptor = descriptor;
  this.builder = DynamicMessage.newBuilder(descriptor);
  this.isDelimited = isDelimited;

  // skip to the required location
  if (readerOffset != null && !readerOffset.isEmpty() && !readerOffset.equals("0")) {
    int offset = Integer.parseInt(readerOffset);
    this.inputStream.skip(offset);
  }
}
 

private ByteString convData(DataDstWriterNode node) throws ConvException {
    // Descriptors.Descriptor msg_desc = (Descriptors.Descriptor) node.privateData;

    DynamicMessage.Builder root = DynamicMessage.newBuilder(currentMsgDesc);
    boolean valid_data = dumpMessage(root, node);
    // 过滤空项
    if (!valid_data) {
        return null;
    }

    try {
        return root.build().toByteString();
    } catch (Exception e) {
        this.logErrorMessage("serialize failed. %s", e.getMessage());
        return null;
    }
}
 

public static List<DynamicMessage> parseToMessages(TypeRegistry registry, Descriptor descriptor,
        List<String> jsonTexts) {
    Parser parser = JsonFormat.parser().usingTypeRegistry(registry);
    List<DynamicMessage> messages = new ArrayList<>();
    try {
        for (String jsonText : jsonTexts) {
            DynamicMessage.Builder messageBuilder = DynamicMessage.newBuilder(descriptor);
            parser.merge(jsonText, messageBuilder);
            messages.add(messageBuilder.build());
        }
        return messages;
    } catch (InvalidProtocolBufferException e) {
        throw new IllegalArgumentException("Unable to parse json text", e);
    }
}
 

/** Parses all the messages and returns them in a list. */
public ImmutableList<DynamicMessage> read() {
  ImmutableList.Builder<DynamicMessage> resultBuilder = ImmutableList.builder();
  try {
    String line;
    boolean wasLastLineEmpty = false;
    while (true) {
      line = bufferedReader.readLine();

      // Two consecutive empty lines mark the end of the stream.
      if (Strings.isNullOrEmpty(line)) {
        if (wasLastLineEmpty) {
          return resultBuilder.build();
        }
        wasLastLineEmpty = true;
        continue;
      } else {
        wasLastLineEmpty = false;
      }

      // Read the next full message.
      StringBuilder stringBuilder = new StringBuilder();
      while (!Strings.isNullOrEmpty(line)) {
        stringBuilder.append(line);
        line = bufferedReader.readLine();
      }
      wasLastLineEmpty = true;

      DynamicMessage.Builder nextMessage = DynamicMessage.newBuilder(descriptor);
      jsonParser.merge(stringBuilder.toString(), nextMessage);

      // Clean up and prepare for next message.
      resultBuilder.add(nextMessage.build());
    }
  } catch (Exception e) {
    throw new IllegalArgumentException("Unable to read messages from: " + source, e);
  }
}
 

public Message toRecord(@Nonnull Descriptors.Descriptor recordDescriptor, @Nonnull IndexEntry kv) {
    Message.Builder recordBuilder = DynamicMessage.newBuilder(recordDescriptor);
    for (Copier copier : copiers) {
        copier.copy(recordDescriptor, recordBuilder, kv);
    }
    return recordBuilder.build();
}
 

protected static DynamicMessage buildRecord2Dynamic(@Nonnull String name, @Nullable Integer intValue, @Nullable String stringValue) {
    final Descriptors.Descriptor descriptor = TestRecordsNulls2Proto.MyNullRecord.getDescriptor();
    DynamicMessage.Builder builder = DynamicMessage.newBuilder(descriptor);
    builder.setField(descriptor.findFieldByNumber(TestRecordsNulls3Proto.MyNullRecord.NAME_FIELD_NUMBER), name);
    if (intValue != null) {
        builder.setField(descriptor.findFieldByNumber(TestRecordsNulls3Proto.MyNullRecord.INT_VALUE_FIELD_NUMBER), intValue);
    }
    if (stringValue != null) {
        builder.setField(descriptor.findFieldByNumber(TestRecordsNulls3Proto.MyNullRecord.STRING_VALUE_FIELD_NUMBER), stringValue);
    }
    return builder.build();
}
 

protected static DynamicMessage buildRecord3Dynamic(@Nonnull String name, @Nullable Integer intValue, @Nullable String stringValue) {
    final Descriptors.Descriptor descriptor = TestRecordsNulls3Proto.MyNullRecord.getDescriptor();
    DynamicMessage.Builder builder = DynamicMessage.newBuilder(descriptor);
    builder.setField(descriptor.findFieldByNumber(TestRecordsNulls3Proto.MyNullRecord.NAME_FIELD_NUMBER), name);
    if (intValue != null) {
        builder.setField(descriptor.findFieldByNumber(TestRecordsNulls3Proto.MyNullRecord.INT_VALUE_FIELD_NUMBER), intValue);
    }
    if (stringValue != null) {
        builder.setField(descriptor.findFieldByNumber(TestRecordsNulls3Proto.MyNullRecord.STRING_VALUE_FIELD_NUMBER), stringValue);
    }
    return builder.build();
}
 

@Override
public DynamicMessage.Builder invokeNewBuilder() {
  if (descriptor == null) {
    descriptor = domain.getDescriptor(messageName);
  }
  return DynamicMessage.newBuilder(descriptor);
}
 

@Override
void setOnProtoMessage(Message.Builder message, Object value) {
  if (value != null) {
    DynamicMessage.Builder builder =
        DynamicMessage.newBuilder(getFieldDescriptor(message).getMessageType());
    valueConvert.setOnProtoMessage(builder, value);
    message.setField(getFieldDescriptor(message), builder.build());
  }
}
 

@Override
public DynamicMessage encode(Row row) {
    Integer tableId = row.rowType().table().getTableId();
    DynamicMessage inside = 
        tableConvertersByTableId.get(tableId).encode(row);
    DynamicMessage.Builder builder = DynamicMessage.newBuilder(messageType);
    builder.setField(groupFieldsByTabelId.get(tableId), inside);
    return builder.build();
}
 

public static List<DynamicMessage> getMessages(
  Descriptors.Descriptor md,
  ExtensionRegistry extensionRegistry,
  byte[] data
) throws Exception {
  DynamicMessage.Builder builder = DynamicMessage.newBuilder(md);
  ByteArrayInputStream in = new ByteArrayInputStream(data);
  List<DynamicMessage> messages = new ArrayList<>();
  while (builder.mergeDelimitedFrom(in, extensionRegistry)) {
    messages.add(builder.build());
    builder.clear();
  }
  return messages;
}
 

@Test
public void testOneofProtoToSdc() throws DataParserException, IOException, DataGeneratorException {

  Descriptors.Descriptor descriptor = OneofProto.getDescriptor().findMessageTypeByName("Oneof");
  ByteArrayOutputStream bOut = new ByteArrayOutputStream();
  OneofProto.Oneof.Builder builder = OneofProto.Oneof.newBuilder();

  OneofProto.Oneof build = builder.setOneofInt(5).build();
  build.writeDelimitedTo(bOut);
  bOut.close();

  DynamicMessage.Builder dynBldr = DynamicMessage.newBuilder(descriptor);
  dynBldr.mergeDelimitedFrom(new ByteArrayInputStream(bOut.toByteArray()), null);
  Record record = RecordCreator.create();
  Field field = ProtobufTypeUtil.protobufToSdcField(record, "", descriptor, typeToExtensionMap, dynBldr.build());
  Assert.assertNotNull(field);
  Assert.assertEquals("", field.getValueAsMap().get("oneofString").getValue());
  Assert.assertEquals(Field.Type.INTEGER, field.getValueAsListMap().get("oneofInt").getType());
  Assert.assertEquals(5, field.getValueAsMap().get("oneofInt").getValueAsInteger());

  bOut.reset();
  builder.clear();
  build = builder.setOneofString("Hello").build();
  build.writeDelimitedTo(bOut);
  bOut.close();

  dynBldr = DynamicMessage.newBuilder(descriptor);
  dynBldr.mergeDelimitedFrom(new ByteArrayInputStream(bOut.toByteArray()), null);
  record = RecordCreator.create();
  field = ProtobufTypeUtil.protobufToSdcField(record, "", descriptor, typeToExtensionMap, dynBldr.build());
  Assert.assertNotNull(field);
  Assert.assertEquals(0, field.getValueAsMap().get("oneofInt").getValue());
  Assert.assertEquals(Field.Type.STRING, field.getValueAsListMap().get("oneofString").getType());
  Assert.assertEquals("Hello", field.getValueAsMap().get("oneofString").getValueAsString());

}
 

public DynamicMessage parsePlainDataMessage(String[] inputs, IdentifyDescriptor ident,
        DataDstWriterNode.DataDstFieldDescriptor field) throws ConvException {
    if (field.getTypeDescriptor() == null || inputs == null || inputs.length == 0) {
        return null;
    }

    Descriptors.FieldDescriptor fd = (Descriptors.FieldDescriptor) field.getRawDescriptor();
    if (null == fd || fd.getJavaType() != MESSAGE) {
        // 不需要提示，如果从其他方式解包协议描述的时候可能有可选字段丢失的
        return null;
    }

    ArrayList<DataDstWriterNode.DataDstFieldDescriptor> children = field.getTypeDescriptor().getSortedFields();
    DynamicMessage.Builder ret = DynamicMessage.newBuilder(fd.getMessageType());
    boolean hasData = false;
    HashSet<String> dumpedOneof = null;
    if (field.getTypeDescriptor().getSortedOneofs().size() > 0) {
        dumpedOneof = new HashSet<String>();
    }

    int usedInputIdx = 0;
    for (int i = 0; i < children.size(); ++i) {
        if (children.get(i).getReferOneof() != null) {
            if (dumpedOneof == null) {
                throw new ConvException(String.format(
                        "Try to convert field %s of %s failed, found oneof descriptor but oneof set is not initialized.",
                        children.get(i).getName(), field.getTypeDescriptor().getFullName()));
            }
            if (dumpedOneof.contains(children.get(i).getReferOneof().getFullName())) {
                continue;
            }

            if (usedInputIdx >= inputs.length) {
                throw new ConvException(String.format(
                        "Try to convert %s of %s failed, field count not matched(expect %d, real %d).",
                        children.get(i).getReferOneof().getName(), field.getTypeDescriptor().getFullName(),
                        usedInputIdx + 1, inputs.length));
            }

            if (dumpPlainField(ret, null, children.get(i).getReferOneof(), false, inputs[usedInputIdx])) {
                hasData = true;
                dumpedOneof.add(children.get(i).getReferOneof().getFullName());
            }

            ++usedInputIdx;
        } else {
            if (usedInputIdx >= inputs.length) {
                throw new ConvException(String.format(
                        "Try to convert %s of %s failed, field count not matched(expect %d, real %d).",
                        children.get(i).getName(), field.getTypeDescriptor().getFullName(), usedInputIdx + 1,
                        inputs.length));
            }

            if (dumpPlainField(ret, null, children.get(i), false, inputs[usedInputIdx])) {
                hasData = true;
            }

            ++usedInputIdx;
        }
    }

    if (usedInputIdx != inputs.length) {
        DataSrcImpl current_source = DataSrcImpl.getOurInstance();
        if (null == current_source) {
            ProgramOptions.getLoger().warn("Try to convert %s need %d fields, but provide %d fields.",
                    field.getTypeDescriptor().getFullName(), usedInputIdx, inputs.length);
        } else {
            ProgramOptions.getLoger().warn(
                    "Try to convert %s need %d fields, but provide %d fields.%s  > File: %s, Table: %s, Row: %d, Column: %d",
                    field.getTypeDescriptor().getFullName(), usedInputIdx, inputs.length, ProgramOptions.getEndl(),
                    current_source.getCurrentFileName(), current_source.getCurrentTableName(),
                    current_source.getCurrentRowNum() + 1, current_source.getLastColomnNum() + 1);
        }
    }

    if (!hasData) {
        return null;
    }

    return ret.build();
}
 

public static DynamicMessage.Builder generateDynamicBuilder(Descriptor desc) {
    return DynamicMessage.newBuilder(desc);
}
 

private void dumpDefault(DynamicMessage.Builder builder, Descriptors.FieldDescriptor fd) {
    switch (fd.getType()) {
        case DOUBLE:
            dumpValue(builder, fd, Double.valueOf(0.0));
            break;
        case FLOAT:
            dumpValue(builder, fd, Float.valueOf(0));
            break;
        case INT64:
        case UINT64:
        case INT32:
        case FIXED64:
        case FIXED32:
        case UINT32:
        case SFIXED32:
        case SFIXED64:
        case SINT32:
        case SINT64:
            dumpValue(builder, fd, 0);
            break;
        case ENUM:
            dumpValue(builder, fd, fd.getEnumType().findValueByNumber(0));
            break;
        case BOOL:
            dumpValue(builder, fd, false);
            break;
        case STRING:
            dumpValue(builder, fd, "");
            break;
        case GROUP:
            dumpValue(builder, fd, new byte[0]);
            break;
        case MESSAGE: {
            DynamicMessage.Builder subnode = DynamicMessage.newBuilder(fd.getMessageType());

            // fill required
            for (Descriptors.FieldDescriptor sub_fd : fd.getMessageType().getFields()) {
                if (checkFieldIsRequired(sub_fd) || ProgramOptions.getInstance().enbleEmptyList) {
                    dumpDefault(subnode, sub_fd);
                }
            }

            dumpValue(builder, fd, subnode.build());
            break;
        }
        case BYTES:
            dumpValue(builder, fd, new byte[0]);
            break;
    }
}
 

/**
 * Serializes a field path in a record to a protobuf message using the specified descriptor.
 *
 * @param record                    Record with the field to serialize
 * @param field                     The field to serialize
 * @param fieldPath                 The field path of the specified field
 * @param desc                      Protobuf descriptor
 * @param messageTypeToExtensionMap Protobuf extension map
 * @param defaultValueMap           Protobuf default field values
 * @return serialized message
 * @throws DataGeneratorException
 */
private static DynamicMessage sdcFieldToProtobufMsg(
    Record record,
    Field field,
    String fieldPath,
    Descriptors.Descriptor desc,
    Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
    Map<String, Object> defaultValueMap
) throws DataGeneratorException {
  if (field == null) {
    return null;
  }

  // compute all fields to look for including extensions
  DynamicMessage.Builder builder = DynamicMessage.newBuilder(desc);
  List<Descriptors.FieldDescriptor> fields = new ArrayList<>();
  fields.addAll(desc.getFields());
  if (messageTypeToExtensionMap.containsKey(desc.getFullName())) {
    fields.addAll(messageTypeToExtensionMap.get(desc.getFullName()));
  }

  // root field is always a Map in a record representing protobuf data
  Map<String, Field> valueAsMap = field.getValueAsMap();

  for (Descriptors.FieldDescriptor f : fields) {
    Field mapField = valueAsMap.get(f.getName());
    // Repeated field
    if (f.isMapField()) {
      handleMapField(record, mapField, fieldPath, messageTypeToExtensionMap, defaultValueMap, f, builder);
    } else if (f.isRepeated()) {
      if (mapField != null) {
        handleRepeatedField(
            record,
            mapField,
            fieldPath,
            messageTypeToExtensionMap,
            defaultValueMap,
            f,
            builder
        );
      }
    } else {
      // non repeated field
      handleNonRepeatedField(
          record,
          valueAsMap,
          fieldPath,
          messageTypeToExtensionMap,
          defaultValueMap,
          desc,
          f,
          builder
      );
    }
  }

  // if record has unknown fields for this field path, handle it
  try {
    handleUnknownFields(record, fieldPath, builder);
  } catch (IOException e) {
    throw new DataGeneratorException(Errors.PROTOBUF_05, e.toString(), e);
  }

  return builder.build();
}
 

/**
 * Creates a new dynamic message builder for the given message type
 * 
 * @param msgTypeName the message type name
 * @return the message builder (null if not found)
 */
public DynamicMessage.Builder newMessageBuilder(String msgTypeName) {
	Descriptor msgType = getMessageDescriptor(msgTypeName);
	if (msgType == null) return null;
	return DynamicMessage.newBuilder(msgType);
}