下面列出了怎么用org.apache.hadoop.io.ByteWritable的API类实例代码及写法，或者点击链接到github查看源代码。
public void delete(String crawldb, String solrUrl, boolean noCommit) throws IOException {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
long start = System.currentTimeMillis();
LOG.info("SolrClean: starting at " + sdf.format(start));
JobConf job = new NutchJob(getConf());
FileInputFormat.addInputPath(job, new Path(crawldb, CrawlDb.CURRENT_NAME));
job.setBoolean("noCommit", noCommit);
job.set(SolrConstants.SERVER_URL, solrUrl);
job.setInputFormat(SequenceFileInputFormat.class);
job.setOutputFormat(NullOutputFormat.class);
job.setMapOutputKeyClass(ByteWritable.class);
job.setMapOutputValueClass(Text.class);
job.setMapperClass(DBFilter.class);
job.setReducerClass(SolrDeleter.class);
JobClient.runJob(job);
long end = System.currentTimeMillis();
LOG.info("SolrClean: finished at " + sdf.format(end) + ", elapsed: " + TimingUtil.elapsedTime(start, end));
}
private static final WritableComparable<?> createWritable(DataType type)
{
switch (type)
{
case BOOLEAN:
return new BooleanWritable();
case BYTE:
return new ByteWritable();
case INT:
return new IntWritable();
case LONG:
return new LongWritable();
case FLOAT:
return new FloatWritable();
case DOUBLE:
return new DoubleWritable();
case STRING:
return new Text();
default:
return null;
}
}
protected Object translateWritableToPigDataType(Writable w, byte dataType) {
switch(dataType) {
case DataType.CHARARRAY: return ((Text) w).toString();
case DataType.BYTEARRAY:
BytesWritable bw = (BytesWritable) w;
// Make a copy
return new DataByteArray(bw.getBytes(), 0, bw.getLength());
case DataType.BOOLEAN: return ((BooleanWritable) w).get();
case DataType.INTEGER: return ((IntWritable) w).get();
case DataType.LONG: return ((LongWritable) w).get();
case DataType.FLOAT: return ((FloatWritable) w).get();
case DataType.DOUBLE: return ((DoubleWritable) w).get();
case DataType.BYTE: return ((ByteWritable) w).get();
case DataType.DATETIME: return ((DateTimeWritable) w).get();
}
return null;
}
@Override
public void reduce(ByteWritable key, Iterator<Text> values,
OutputCollector<Text, ByteWritable> output, Reporter reporter)
throws IOException {
while (values.hasNext()) {
Text document = values.next();
writers.delete(document.toString());
totalDeleted++;
reporter.incrCounter("CleaningJobStatus", "Deleted documents",
1);
// if (numDeletes >= NUM_MAX_DELETE_REQUEST) {
// LOG.info("CleaningJob: deleting " + numDeletes
// + " documents");
// // TODO updateRequest.process(solr);
// // TODO updateRequest = new UpdateRequest();
// writers.delete(key.toString());
// totalDeleted += numDeletes;
// numDeletes = 0;
// }
}
}
@SuppressWarnings("unchecked")
private<T> T convert(Record stratosphereType, int pos, Class<T> hadoopType) {
if(hadoopType == LongWritable.class ) {
return (T) new LongWritable((stratosphereType.getField(pos, LongValue.class)).getValue());
}
if(hadoopType == org.apache.hadoop.io.Text.class) {
return (T) new Text((stratosphereType.getField(pos, StringValue.class)).getValue());
}
if(hadoopType == org.apache.hadoop.io.IntWritable.class) {
return (T) new IntWritable((stratosphereType.getField(pos, IntValue.class)).getValue());
}
if(hadoopType == org.apache.hadoop.io.FloatWritable.class) {
return (T) new FloatWritable((stratosphereType.getField(pos, FloatValue.class)).getValue());
}
if(hadoopType == org.apache.hadoop.io.DoubleWritable.class) {
return (T) new DoubleWritable((stratosphereType.getField(pos, DoubleValue.class)).getValue());
}
if(hadoopType == org.apache.hadoop.io.BooleanWritable.class) {
return (T) new BooleanWritable((stratosphereType.getField(pos, BooleanValue.class)).getValue());
}
if(hadoopType == org.apache.hadoop.io.ByteWritable.class) {
return (T) new ByteWritable((stratosphereType.getField(pos, ByteValue.class)).getValue());
}
throw new RuntimeException("Unable to convert Stratosphere type ("+stratosphereType.getClass().getCanonicalName()+") to Hadoop.");
}
/**
* Test convertKeyValue for Byte type
*/
@Test
public void convertKeyValueByte() {
byte key = 1;
initRecordkeyAdapter();
runConvertKeyValue(key, new ByteWritable(key));
}
public void write(Writable w) throws IOException {
if (w instanceof TypedBytesWritable) {
writeTypedBytes((TypedBytesWritable) w);
} else if (w instanceof BytesWritable) {
writeBytes((BytesWritable) w);
} else if (w instanceof ByteWritable) {
writeByte((ByteWritable) w);
} else if (w instanceof BooleanWritable) {
writeBoolean((BooleanWritable) w);
} else if (w instanceof IntWritable) {
writeInt((IntWritable) w);
} else if (w instanceof VIntWritable) {
writeVInt((VIntWritable) w);
} else if (w instanceof LongWritable) {
writeLong((LongWritable) w);
} else if (w instanceof VLongWritable) {
writeVLong((VLongWritable) w);
} else if (w instanceof FloatWritable) {
writeFloat((FloatWritable) w);
} else if (w instanceof DoubleWritable) {
writeDouble((DoubleWritable) w);
} else if (w instanceof Text) {
writeText((Text) w);
} else if (w instanceof ArrayWritable) {
writeArray((ArrayWritable) w);
} else if (w instanceof MapWritable) {
writeMap((MapWritable) w);
} else if (w instanceof SortedMapWritable) {
writeSortedMap((SortedMapWritable) w);
} else if (w instanceof Record) {
writeRecord((Record) w);
} else {
writeWritable(w); // last resort
}
}
public Class<? extends Writable> readType() throws IOException {
Type type = in.readType();
if (type == null) {
return null;
}
switch (type) {
case BYTES:
return BytesWritable.class;
case BYTE:
return ByteWritable.class;
case BOOL:
return BooleanWritable.class;
case INT:
return VIntWritable.class;
case LONG:
return VLongWritable.class;
case FLOAT:
return FloatWritable.class;
case DOUBLE:
return DoubleWritable.class;
case STRING:
return Text.class;
case VECTOR:
return ArrayWritable.class;
case MAP:
return MapWritable.class;
case WRITABLE:
return Writable.class;
default:
throw new RuntimeException("unknown type");
}
}
public void write(Writable w) throws IOException {
if (w instanceof TypedBytesWritable) {
writeTypedBytes((TypedBytesWritable) w);
} else if (w instanceof BytesWritable) {
writeBytes((BytesWritable) w);
} else if (w instanceof ByteWritable) {
writeByte((ByteWritable) w);
} else if (w instanceof BooleanWritable) {
writeBoolean((BooleanWritable) w);
} else if (w instanceof IntWritable) {
writeInt((IntWritable) w);
} else if (w instanceof VIntWritable) {
writeVInt((VIntWritable) w);
} else if (w instanceof LongWritable) {
writeLong((LongWritable) w);
} else if (w instanceof VLongWritable) {
writeVLong((VLongWritable) w);
} else if (w instanceof FloatWritable) {
writeFloat((FloatWritable) w);
} else if (w instanceof DoubleWritable) {
writeDouble((DoubleWritable) w);
} else if (w instanceof Text) {
writeText((Text) w);
} else if (w instanceof ArrayWritable) {
writeArray((ArrayWritable) w);
} else if (w instanceof MapWritable) {
writeMap((MapWritable) w);
} else if (w instanceof SortedMapWritable) {
writeSortedMap((SortedMapWritable) w);
} else if (w instanceof Record) {
writeRecord((Record) w);
} else {
writeWritable(w); // last resort
}
}
public Class<? extends Writable> readType() throws IOException {
Type type = in.readType();
if (type == null) {
return null;
}
switch (type) {
case BYTES:
return BytesWritable.class;
case BYTE:
return ByteWritable.class;
case BOOL:
return BooleanWritable.class;
case INT:
return VIntWritable.class;
case LONG:
return VLongWritable.class;
case FLOAT:
return FloatWritable.class;
case DOUBLE:
return DoubleWritable.class;
case STRING:
return Text.class;
case VECTOR:
return ArrayWritable.class;
case MAP:
return MapWritable.class;
case WRITABLE:
return Writable.class;
default:
throw new RuntimeException("unknown type");
}
}
public static Writable toWritable(Object object) {
if (object == null) {
return null; //return NullWritable.get();
}
if (object instanceof Writable) {
return (Writable) object;
}
if (object instanceof String) {
return new Text((String) object);
}
if (object instanceof Long) {
return new VLongWritable((Long) object);
}
if (object instanceof Integer) {
return new VIntWritable((Integer) object);
}
if (object instanceof Byte) {
return new ByteWritable((Byte) object);
}
if (object instanceof Double) {
return new DoubleWritable((Double) object);
}
if (object instanceof Float) {
return new FloatWritable((Float) object);
}
if (object instanceof Boolean) {
return new BooleanWritable((Boolean) object);
}
if (object instanceof byte[]) {
return new BytesWritable((byte[]) object);
}
return new BytesWritable(object.toString().getBytes());
}
@Override
public void map(Text key, CrawlDatum value,
OutputCollector<ByteWritable, Text> output, Reporter reporter)
throws IOException {
if (value.getStatus() == CrawlDatum.STATUS_DB_GONE) {
output.collect(OUT, key);
}
}
/**
* Returns the object inside Writable
*
* @param writable
* @return
* @throws IllegalAccessException
* @throws InstantiationException
* @throws InvocationTargetException
* @throws NoSuchMethodException
*/
private static Object getObjectFromWritable(Writable writable)
throws IllegalAccessException, InstantiationException, InvocationTargetException {
Object object = null;
if (writable instanceof NullWritable) {
object = NullWritable.get();
} else if (writable instanceof BooleanWritable) {
object = ((BooleanWritable) writable).get();
} else if (writable instanceof Text) {
object = writable.toString();
} else if (writable instanceof ByteWritable) {
object = ((ByteWritable) writable).get();
} else if (writable instanceof IntWritable) {
object = ((IntWritable) writable).get();
} else if (writable instanceof LongWritable) {
object = ((LongWritable) writable).get();
} else if (writable instanceof BytesWritable) {
object = ((BytesWritable) writable).getBytes();
} else if (writable instanceof DoubleWritable) {
object = ((DoubleWritable) writable).get();
} else if (writable instanceof FloatWritable) {
object = ((FloatWritable) writable).get();
} else {
// TODO : do nothing
}
return object;
}
protected byte inferPigDataType(Type t) {
if (t == BytesWritable.class) return DataType.BYTEARRAY;
else if (t == Text.class) return DataType.CHARARRAY;
else if (t == IntWritable.class) return DataType.INTEGER;
else if (t == LongWritable.class) return DataType.LONG;
else if (t == FloatWritable.class) return DataType.FLOAT;
else if (t == DoubleWritable.class) return DataType.DOUBLE;
else if (t == BooleanWritable.class) return DataType.BOOLEAN;
else if (t == ByteWritable.class) return DataType.BYTE;
else if (t == DateTimeWritable.class) return DataType.DATETIME;
// not doing maps or other complex types for now
else return DataType.ERROR;
}
@Override
public void map(Text key, CrawlDatum value,
OutputCollector<ByteWritable, Text> output, Reporter reporter)
throws IOException {
if (value.getStatus() == CrawlDatum.STATUS_DB_GONE || value.getStatus() == CrawlDatum.STATUS_DB_DUPLICATE) {
output.collect(OUT, key);
}
}
public void delete(String crawldb, boolean noCommit) throws IOException {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
long start = System.currentTimeMillis();
LOG.info("CleaningJob: starting at " + sdf.format(start));
JobConf job = new NutchJob(getConf());
FileInputFormat.addInputPath(job, new Path(crawldb,
CrawlDb.CURRENT_NAME));
job.setBoolean("noCommit", noCommit);
job.setInputFormat(SequenceFileInputFormat.class);
job.setOutputFormat(NullOutputFormat.class);
job.setMapOutputKeyClass(ByteWritable.class);
job.setMapOutputValueClass(Text.class);
job.setMapperClass(DBFilter.class);
job.setReducerClass(DeleterReducer.class);
job.setJobName("CleaningJob");
// need to expicitely allow deletions
job.setBoolean(IndexerMapReduce.INDEXER_DELETE, true);
JobClient.runJob(job);
long end = System.currentTimeMillis();
LOG.info("CleaningJob: finished at " + sdf.format(end) + ", elapsed: "
+ TimingUtil.elapsedTime(start, end));
}
protected Value convert(Object hadoopType) {
if(hadoopType instanceof org.apache.hadoop.io.LongWritable ) {
return new LongValue(((LongWritable)hadoopType).get());
}
if(hadoopType instanceof org.apache.hadoop.io.Text) {
return new StringValue(((Text)hadoopType).toString());
}
if(hadoopType instanceof org.apache.hadoop.io.IntWritable) {
return new IntValue(((IntWritable)hadoopType).get());
}
if(hadoopType instanceof org.apache.hadoop.io.FloatWritable) {
return new FloatValue(((FloatWritable)hadoopType).get());
}
if(hadoopType instanceof org.apache.hadoop.io.DoubleWritable) {
return new DoubleValue(((DoubleWritable)hadoopType).get());
}
if(hadoopType instanceof org.apache.hadoop.io.BooleanWritable) {
return new BooleanValue(((BooleanWritable)hadoopType).get());
}
if(hadoopType instanceof org.apache.hadoop.io.ByteWritable) {
return new ByteValue(((ByteWritable)hadoopType).get());
}
if (hadoopType instanceof NullWritable) {
return NullValue.getInstance();
}
throw new RuntimeException("Unable to convert Hadoop type ("+hadoopType.getClass().getCanonicalName()+") to Stratosphere.");
}
private static Object decodeRecordReaderValue(Type type, Object actualValue)
{
if (actualValue instanceof BooleanWritable) {
actualValue = ((BooleanWritable) actualValue).get();
}
else if (actualValue instanceof ByteWritable) {
actualValue = ((ByteWritable) actualValue).get();
}
else if (actualValue instanceof BytesWritable) {
actualValue = new SqlVarbinary(((BytesWritable) actualValue).copyBytes());
}
else if (actualValue instanceof DateWritable) {
actualValue = new SqlDate(((DateWritable) actualValue).getDays());
}
else if (actualValue instanceof DoubleWritable) {
actualValue = ((DoubleWritable) actualValue).get();
}
else if (actualValue instanceof FloatWritable) {
actualValue = ((FloatWritable) actualValue).get();
}
else if (actualValue instanceof IntWritable) {
actualValue = ((IntWritable) actualValue).get();
}
else if (actualValue instanceof HiveCharWritable) {
actualValue = ((HiveCharWritable) actualValue).getPaddedValue().toString();
}
else if (actualValue instanceof LongWritable) {
actualValue = ((LongWritable) actualValue).get();
}
else if (actualValue instanceof ShortWritable) {
actualValue = ((ShortWritable) actualValue).get();
}
else if (actualValue instanceof HiveDecimalWritable) {
DecimalType decimalType = (DecimalType) type;
HiveDecimalWritable writable = (HiveDecimalWritable) actualValue;
// writable messes with the scale so rescale the values to the Presto type
BigInteger rescaledValue = rescale(writable.getHiveDecimal().unscaledValue(), writable.getScale(), decimalType.getScale());
actualValue = new SqlDecimal(rescaledValue, decimalType.getPrecision(), decimalType.getScale());
}
else if (actualValue instanceof Text) {
actualValue = actualValue.toString();
}
else if (actualValue instanceof TimestampWritable) {
TimestampWritable timestamp = (TimestampWritable) actualValue;
actualValue = sqlTimestampOf((timestamp.getSeconds() * 1000) + (timestamp.getNanos() / 1000000L), SESSION);
}
else if (actualValue instanceof OrcStruct) {
List<Object> fields = new ArrayList<>();
OrcStruct structObject = (OrcStruct) actualValue;
for (int fieldId = 0; fieldId < structObject.getNumFields(); fieldId++) {
fields.add(OrcUtil.getFieldValue(structObject, fieldId));
}
actualValue = decodeRecordReaderStruct(type, fields);
}
else if (actualValue instanceof List) {
actualValue = decodeRecordReaderList(type, ((List<?>) actualValue));
}
else if (actualValue instanceof Map) {
actualValue = decodeRecordReaderMap(type, (Map<?, ?>) actualValue);
}
return actualValue;
}
private static Object decodeRecordReaderValue(Type type, Object actualValue)
{
if (actualValue instanceof LazyPrimitive) {
actualValue = ((LazyPrimitive<?, ?>) actualValue).getWritableObject();
}
if (actualValue instanceof BooleanWritable) {
actualValue = ((BooleanWritable) actualValue).get();
}
else if (actualValue instanceof ByteWritable) {
actualValue = ((ByteWritable) actualValue).get();
}
else if (actualValue instanceof BytesWritable) {
actualValue = new SqlVarbinary(((BytesWritable) actualValue).copyBytes());
}
else if (actualValue instanceof DateWritable) {
actualValue = new SqlDate(((DateWritable) actualValue).getDays());
}
else if (actualValue instanceof DoubleWritable) {
actualValue = ((DoubleWritable) actualValue).get();
}
else if (actualValue instanceof FloatWritable) {
actualValue = ((FloatWritable) actualValue).get();
}
else if (actualValue instanceof IntWritable) {
actualValue = ((IntWritable) actualValue).get();
}
else if (actualValue instanceof LongWritable) {
actualValue = ((LongWritable) actualValue).get();
}
else if (actualValue instanceof ShortWritable) {
actualValue = ((ShortWritable) actualValue).get();
}
else if (actualValue instanceof HiveDecimalWritable) {
DecimalType decimalType = (DecimalType) type;
HiveDecimalWritable writable = (HiveDecimalWritable) actualValue;
// writable messes with the scale so rescale the values to the Presto type
BigInteger rescaledValue = rescale(writable.getHiveDecimal().unscaledValue(), writable.getScale(), decimalType.getScale());
actualValue = new SqlDecimal(rescaledValue, decimalType.getPrecision(), decimalType.getScale());
}
else if (actualValue instanceof Text) {
actualValue = actualValue.toString();
}
else if (actualValue instanceof TimestampWritable) {
TimestampWritable timestamp = (TimestampWritable) actualValue;
if (SESSION.isLegacyTimestamp()) {
actualValue = SqlTimestamp.legacyFromMillis(3, (timestamp.getSeconds() * 1000) + (timestamp.getNanos() / 1000000L), UTC_KEY);
}
else {
actualValue = SqlTimestamp.fromMillis(3, (timestamp.getSeconds() * 1000) + (timestamp.getNanos() / 1000000L));
}
}
else if (actualValue instanceof StructObject) {
StructObject structObject = (StructObject) actualValue;
actualValue = decodeRecordReaderStruct(type, structObject.getFieldsAsList());
}
else if (actualValue instanceof LazyBinaryArray) {
actualValue = decodeRecordReaderList(type, ((LazyBinaryArray) actualValue).getList());
}
else if (actualValue instanceof LazyBinaryMap) {
actualValue = decodeRecordReaderMap(type, ((LazyBinaryMap) actualValue).getMap());
}
else if (actualValue instanceof LazyArray) {
actualValue = decodeRecordReaderList(type, ((LazyArray) actualValue).getList());
}
else if (actualValue instanceof LazyMap) {
actualValue = decodeRecordReaderMap(type, ((LazyMap) actualValue).getMap());
}
else if (actualValue instanceof List) {
actualValue = decodeRecordReaderList(type, ((List<?>) actualValue));
}
return actualValue;
}
public void writeByte(ByteWritable bw) throws IOException {
out.writeByte(bw.get());
}
public ByteWritable readByte() throws IOException {
return readByte(null);
}
public void writeByte(ByteWritable bw) throws IOException {
out.writeByte(bw.get());
}
public ByteWritable readByte() throws IOException {
return readByte(null);
}
public static Writable toWritable(Object o) throws IOException {
if (o instanceof Long) {
return new LongWritable(((Long) o).longValue());
} else if (o instanceof String) {
return new Text(o.toString());
} else if (o instanceof byte[]) {
return new BytesWritable((byte[]) o);
} else if (o instanceof Integer) {
return new IntWritable(((Integer) o).intValue());
} else if (o instanceof Short) {
return new ShortWritable(((Short) o).shortValue());
} else if (o instanceof Byte) {
return new ByteWritable(((Byte) o).byteValue());
} else if (o instanceof Double) {
return new DoubleWritable(((Double) o).doubleValue());
} else if (o instanceof Float) {
return new FloatWritable(((Float) o).floatValue());
} else if (o instanceof Boolean) {
return new BooleanWritable(((Boolean) o).booleanValue());
} else if (o instanceof List) {
Writable[] a = new Writable[((List) o).size()];
for (int i = 0; i < a.length; i++) {
a[i] = new ObjectWritable(toWritable(((List) o).get(i)));
}
return new ArrayWritable(ObjectWritable.class, a);
} else if (o instanceof Map) {
MapWritable map = new MapWritable();
for (Entry<Object,Object> entry: ((Map<Object,Object>) o).entrySet()) {
map.put(toWritable(entry.getKey()), toWritable(entry.getValue()));
}
return map;
} else if (null == o) {
return NullWritable.get();
} else {
ObjectWritable ow = new ObjectWritable();
ow.set(o);
return ow;
}// else {
// throw new IOException("Unsupported type " + o.getClass());
//}
}
/**
* Copy the value of {@param from} object into {@param to} with supporting of type-widening that ORC allowed.
*/
public static void handlePrimitiveWritableComparable(WritableComparable from, WritableComparable to) {
if (from instanceof ByteWritable) {
if (to instanceof ByteWritable) {
((ByteWritable) to).set(((ByteWritable) from).get());
return;
} else if (to instanceof ShortWritable) {
((ShortWritable) to).set(((ByteWritable) from).get());
return;
} else if (to instanceof IntWritable) {
((IntWritable) to).set(((ByteWritable) from).get());
return;
} else if (to instanceof LongWritable) {
((LongWritable) to).set(((ByteWritable) from).get());
return;
} else if (to instanceof DoubleWritable) {
((DoubleWritable) to).set(((ByteWritable) from).get());
return;
}
} else if (from instanceof ShortWritable) {
if (to instanceof ShortWritable) {
((ShortWritable) to).set(((ShortWritable) from).get());
return;
} else if (to instanceof IntWritable) {
((IntWritable) to).set(((ShortWritable) from).get());
return;
} else if (to instanceof LongWritable) {
((LongWritable) to).set(((ShortWritable) from).get());
return;
} else if (to instanceof DoubleWritable) {
((DoubleWritable) to).set(((ShortWritable) from).get());
return;
}
} else if (from instanceof IntWritable) {
if (to instanceof IntWritable) {
((IntWritable) to).set(((IntWritable) from).get());
return;
} else if (to instanceof LongWritable) {
((LongWritable) to).set(((IntWritable) from).get());
return;
} else if (to instanceof DoubleWritable) {
((DoubleWritable) to).set(((IntWritable) from).get());
return;
}
} else if (from instanceof LongWritable) {
if (to instanceof LongWritable) {
((LongWritable) to).set(((LongWritable) from).get());
return;
} else if (to instanceof DoubleWritable) {
((DoubleWritable) to).set(((LongWritable) from).get());
return;
}
// Following from this branch, type-widening is not allowed and only value-copy will happen.
} else if (from instanceof DoubleWritable) {
if (to instanceof DoubleWritable) {
((DoubleWritable) to).set(((DoubleWritable) from).get());
return;
}
} else if (from instanceof BytesWritable) {
if (to instanceof BytesWritable) {
((BytesWritable) to).set((BytesWritable) from);
return;
}
} else if (from instanceof FloatWritable) {
if (to instanceof FloatWritable) {
((FloatWritable) to).set(((FloatWritable) from).get());
return;
}
} else if (from instanceof Text) {
if (to instanceof Text) {
((Text) to).set((Text) from);
return;
}
} else if (from instanceof DateWritable) {
if (to instanceof DateWritable) {
((DateWritable) to).set(((DateWritable) from).get());
return;
}
} else if (from instanceof OrcTimestamp && to instanceof OrcTimestamp) {
((OrcTimestamp) to).set(((OrcTimestamp) from).toString());
return;
} else if (from instanceof HiveDecimalWritable && to instanceof HiveDecimalWritable) {
((HiveDecimalWritable) to).set(((HiveDecimalWritable) from).getHiveDecimal());
return;
} else if (from instanceof BooleanWritable && to instanceof BooleanWritable) {
((BooleanWritable) to).set(((BooleanWritable) from).get());
return;
}
throw new UnsupportedOperationException(String
.format("The conversion of primitive-type WritableComparable object from %s to %s is not supported",
from.getClass(), to.getClass()));
}
/**
* Fill in value in OrcStruct with given schema, assuming {@param w} contains the same schema as {@param schema}.
* {@param schema} is still necessary to given given {@param w} do contains schema information itself, because the
* actual value type is only available in {@link TypeDescription} but not {@link org.apache.orc.mapred.OrcValue}.
*
* For simplicity here are some assumptions:
* - We only give 3 primitive values and use them to construct compound values. To make it work for different types that
* can be widened or shrunk to each other, please use value within small range.
* - For List, Map or Union, make sure there's at least one entry within the record-container.
* you may want to try createValueRecursively(TypeDescription) instead of {@link OrcStruct#createValue(TypeDescription)}
*/
public static void fillOrcStructWithFixedValue(WritableComparable w, TypeDescription schema, int unionTag,
int intValue, String stringValue, boolean booleanValue) {
switch (schema.getCategory()) {
case BOOLEAN:
((BooleanWritable) w).set(booleanValue);
break;
case BYTE:
((ByteWritable) w).set((byte) intValue);
break;
case SHORT:
((ShortWritable) w).set((short) intValue);
break;
case INT:
((IntWritable) w).set(intValue);
break;
case LONG:
((LongWritable) w).set(intValue);
break;
case FLOAT:
((FloatWritable) w).set(intValue * 1.0f);
break;
case DOUBLE:
((DoubleWritable) w).set(intValue * 1.0);
break;
case STRING:
case CHAR:
case VARCHAR:
((Text) w).set(stringValue);
break;
case BINARY:
throw new UnsupportedOperationException("Binary type is not supported in random orc data filler");
case DECIMAL:
throw new UnsupportedOperationException("Decimal type is not supported in random orc data filler");
case DATE:
case TIMESTAMP:
case TIMESTAMP_INSTANT:
throw new UnsupportedOperationException(
"Timestamp and its derived types is not supported in random orc data filler");
case LIST:
OrcList castedList = (OrcList) w;
// Here it is not trivial to create typed-object in element-type. So this method expect the value container
// to at least contain one element, or the traversing within the list will be skipped.
for (Object i : castedList) {
fillOrcStructWithFixedValue((WritableComparable) i, schema.getChildren().get(0), unionTag, intValue,
stringValue, booleanValue);
}
break;
case MAP:
OrcMap castedMap = (OrcMap) w;
for (Object entry : castedMap.entrySet()) {
Map.Entry<WritableComparable, WritableComparable> castedEntry =
(Map.Entry<WritableComparable, WritableComparable>) entry;
fillOrcStructWithFixedValue(castedEntry.getKey(), schema.getChildren().get(0), unionTag, intValue,
stringValue, booleanValue);
fillOrcStructWithFixedValue(castedEntry.getValue(), schema.getChildren().get(1), unionTag, intValue,
stringValue, booleanValue);
}
break;
case STRUCT:
OrcStruct castedStruct = (OrcStruct) w;
int fieldIdx = 0;
for (TypeDescription child : schema.getChildren()) {
fillOrcStructWithFixedValue(castedStruct.getFieldValue(fieldIdx), child, unionTag, intValue, stringValue,
booleanValue);
fieldIdx += 1;
}
break;
case UNION:
OrcUnion castedUnion = (OrcUnion) w;
TypeDescription targetMemberSchema = schema.getChildren().get(unionTag);
castedUnion.set(unionTag, OrcUtils.createValueRecursively(targetMemberSchema));
fillOrcStructWithFixedValue((WritableComparable) castedUnion.getObject(), targetMemberSchema, unionTag,
intValue, stringValue, booleanValue);
break;
default:
throw new IllegalArgumentException("Unknown type " + schema.toString());
}
}
@Test
public void testByte() {
writableTypeToJson(new ByteWritable(Byte.MAX_VALUE));
}