下面列出了怎么用org.apache.hadoop.io.BytesWritable的API类实例代码及写法,或者点击链接到github查看源代码。
@Test
public void parseBitcoinRawBlock() throws HadoopCryptoLedgerConfigurationException, IOException {
ClassLoader classLoader = getClass().getClassLoader();
String fileName="genesis.blk";
String fileNameBlock=classLoader.getResource("testdata/"+fileName).getFile();
Path file = new Path(fileNameBlock);
FileInputSplit blockInputSplit = new FileInputSplit(0,file,0, -1, null);
BitcoinRawBlockFlinkInputFormat inputFormat = new BitcoinRawBlockFlinkInputFormat(1024*1024,"F9BEB4D9",false);
inputFormat.open(blockInputSplit);
assertFalse(inputFormat.reachedEnd(),"End not reached");
BytesWritable reuse = new BytesWritable();
BytesWritable nextBlock = inputFormat.nextRecord(reuse);
assertNotNull(nextBlock,"First Block returned");
assertEquals( 293, nextBlock.getLength(),"First Block must have size of 293");
nextBlock=inputFormat.nextRecord(reuse);
assertNull(nextBlock,"No further block");
assertTrue(inputFormat.reachedEnd(),"End reached");
}
public void testNestedIterable() throws Exception {
Random r = new Random();
Writable[] writs = {
new BooleanWritable(r.nextBoolean()),
new FloatWritable(r.nextFloat()),
new FloatWritable(r.nextFloat()),
new IntWritable(r.nextInt()),
new LongWritable(r.nextLong()),
new BytesWritable("dingo".getBytes()),
new LongWritable(r.nextLong()),
new IntWritable(r.nextInt()),
new BytesWritable("yak".getBytes()),
new IntWritable(r.nextInt())
};
TupleWritable sTuple = makeTuple(writs);
assertTrue("Bad count", writs.length == verifIter(writs, sTuple, 0));
}
public DoubleWritable evaluate(BytesWritable geomref) {
if (geomref == null || geomref.getLength() == 0) {
LogUtils.Log_ArgumentsNull(LOG);
return null;
}
OGCGeometry ogcGeometry = GeometryUtils.geometryFromEsriShape(geomref);
if (ogcGeometry == null) {
LogUtils.Log_ArgumentsNull(LOG);
return null;
}
Envelope envBound = new Envelope();
ogcGeometry.getEsriGeometry().queryEnvelope(envBound);
resultDouble.set(envBound.getYMin());
return resultDouble;
}
@Override
public Class<? extends Writable> getHadoopWritableCompatibleClass( ValueMetaInterface kettleType ) {
if ( kettleType == null ) {
return NullWritable.class;
}
switch ( kettleType.getType() ) {
case ValueMetaInterface.TYPE_STRING:
case ValueMetaInterface.TYPE_BIGNUMBER:
case ValueMetaInterface.TYPE_DATE:
return Text.class;
case ValueMetaInterface.TYPE_INTEGER:
return LongWritable.class;
case ValueMetaInterface.TYPE_NUMBER:
return DoubleWritable.class;
case ValueMetaInterface.TYPE_BOOLEAN:
return BooleanWritable.class;
case ValueMetaInterface.TYPE_BINARY:
return BytesWritable.class;
default:
return Text.class;
}
}
public BytesWritable evaluate(Text wkwrap, int wkid) throws UDFArgumentException {
String wkt = wkwrap.toString();
try {
SpatialReference spatialReference = null;
if (wkid != GeometryUtils.WKID_UNKNOWN) {
spatialReference = SpatialReference.create(wkid);
}
OGCGeometry ogcObj = OGCGeometry.fromText(wkt);
ogcObj.setSpatialReference(spatialReference);
return GeometryUtils.geometryToEsriShapeBytesWritable(ogcObj);
} catch (Exception e) { // IllegalArgumentException, GeometryException
LogUtils.Log_InvalidText(LOG, wkt);
return null;
}
}
@Test
public void partial1ModeIntKeysDefaultParams() throws Exception {
ObjectInspector[] inspectors = new ObjectInspector[] { intInspector, doubleInspector };
GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false, false);
try (GenericUDAFEvaluator eval = new DataToDoubleSummarySketchUDAF().getEvaluator(info)) {
ObjectInspector resultInspector = eval.init(Mode.PARTIAL1, inspectors);
checkIntermediateResultInspector(resultInspector);
@SuppressWarnings("unchecked")
State<DoubleSummary> state = (State<DoubleSummary>) eval.getNewAggregationBuffer();
eval.iterate(state, new Object[] {new IntWritable(1), new DoubleWritable(1)});
eval.iterate(state, new Object[] {new IntWritable(2), new DoubleWritable(1)});
Object result = eval.terminatePartial(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof List);
List<?> r = (List<?>) result;
Assert.assertEquals(r.size(), 2);
Assert.assertEquals(((IntWritable) r.get(0)).get(), DEFAULT_NOMINAL_ENTRIES);
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(1)), new DoubleSummaryDeserializer());
Assert.assertFalse(resultSketch.isEstimationMode());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
}
}
@Override
public void map(Text key, CrawlDatum value,
OutputCollector<BytesWritable, CrawlDatum> output,
Reporter reporter) throws IOException {
if (value.getStatus() == CrawlDatum.STATUS_DB_FETCHED
|| value.getStatus() == CrawlDatum.STATUS_DB_NOTMODIFIED) {
// || value.getStatus() ==CrawlDatum.STATUS_DB_GONE){
byte[] signature = value.getSignature();
if (signature == null) return;
BytesWritable sig = new BytesWritable(signature);
// add the URL as a temporary MD
value.getMetaData().put(urlKey, key);
// reduce on the signature
output.collect(sig, value);
}
}
/**
* Configure a waiting job
*/
static void configureWaitingJobConf(JobConf jobConf, Path inDir,
Path outputPath, int numMaps, int numRed,
String jobName, String mapSignalFilename,
String redSignalFilename)
throws IOException {
jobConf.setJobName(jobName);
jobConf.setInputFormat(NonSplitableSequenceFileInputFormat.class);
jobConf.setOutputFormat(SequenceFileOutputFormat.class);
FileInputFormat.setInputPaths(jobConf, inDir);
FileOutputFormat.setOutputPath(jobConf, outputPath);
jobConf.setMapperClass(UtilsForTests.HalfWaitingMapper.class);
jobConf.setReducerClass(IdentityReducer.class);
jobConf.setOutputKeyClass(BytesWritable.class);
jobConf.setOutputValueClass(BytesWritable.class);
jobConf.setInputFormat(RandomInputFormat.class);
jobConf.setNumMapTasks(numMaps);
jobConf.setNumReduceTasks(numRed);
jobConf.setJar("build/test/mapred/testjar/testjob.jar");
jobConf.set(getTaskSignalParameter(true), mapSignalFilename);
jobConf.set(getTaskSignalParameter(false), redSignalFilename);
}
/**
* Configure a waiting job
*/
static void configureWaitingJobConf(JobConf jobConf, Path inDir,
Path outputPath, int numMaps, int numRed,
String jobName, String mapSignalFilename,
String redSignalFilename)
throws IOException {
jobConf.setJobName(jobName);
jobConf.setInputFormat(NonSplitableSequenceFileInputFormat.class);
jobConf.setOutputFormat(SequenceFileOutputFormat.class);
FileInputFormat.setInputPaths(jobConf, inDir);
FileOutputFormat.setOutputPath(jobConf, outputPath);
jobConf.setMapperClass(UtilsForTests.HalfWaitingMapper.class);
jobConf.setReducerClass(IdentityReducer.class);
jobConf.setOutputKeyClass(BytesWritable.class);
jobConf.setOutputValueClass(BytesWritable.class);
jobConf.setInputFormat(RandomInputFormat.class);
jobConf.setNumMapTasks(numMaps);
jobConf.setNumReduceTasks(numRed);
jobConf.setJar("build/test/testjar/testjob.jar");
jobConf.set(getTaskSignalParameter(true), mapSignalFilename);
jobConf.set(getTaskSignalParameter(false), redSignalFilename);
}
public BytesWritable evaluate(Text wkwrap) throws UDFArgumentException {
String wkt = wkwrap.toString();
try {
OGCGeometry ogcObj = OGCGeometry.fromText(wkt);
ogcObj.setSpatialReference(null);
if (ogcObj.geometryType().equals("Point")) {
return GeometryUtils.geometryToEsriShapeBytesWritable(ogcObj);
} else {
LogUtils.Log_InvalidType(LOG, GeometryUtils.OGCType.ST_POINT, GeometryUtils.OGCType.UNKNOWN);
return null;
}
} catch (Exception e) { // IllegalArgumentException, GeometryException
LogUtils.Log_InvalidText(LOG, wkt);
return null;
}
}
public static long readAndAppendCommit(Configuration conf, FileSystem fileSystem, MapFile.Writer writer, String fileName, long lastAstKey, long lastCommitKey) throws IOException {
long newLastKey = lastCommitKey;
SequenceFile.Reader r = new SequenceFile.Reader(fileSystem, new Path(fileName), conf);
LongWritable longKey = new LongWritable();
BytesWritable value = new BytesWritable();
try {
while (r.next(longKey, value)) {
newLastKey = longKey.get() + lastCommitKey;
Revision rev = Revision.parseFrom(CodedInputStream.newInstance(value.getBytes(), 0, value.getLength()));
Revision.Builder rb = Revision.newBuilder(rev);
for (ChangedFile.Builder cfb : rb.getFilesBuilderList()) {
long key = cfb.getKey();
if (key > 0)
cfb.setKey(lastAstKey + key);
}
writer.append(new LongWritable(newLastKey), new BytesWritable(rb.build().toByteArray()));
}
} catch (Exception e) {
System.err.println(fileName);
e.printStackTrace();
} finally {
r.close();
}
return newLastKey;
}
private static void createFiles(int length, int numFiles, Random random,
Job job) throws IOException {
Range[] ranges = createRanges(length, numFiles, random);
for (int i = 0; i < numFiles; i++) {
Path file = new Path(workDir, "test_" + i + ".seq");
// create a file with length entries
@SuppressWarnings("deprecation")
SequenceFile.Writer writer =
SequenceFile.createWriter(localFs, job.getConfiguration(), file,
IntWritable.class, BytesWritable.class);
Range range = ranges[i];
try {
for (int j = range.start; j < range.end; j++) {
IntWritable key = new IntWritable(j);
byte[] data = new byte[random.nextInt(10)];
random.nextBytes(data);
BytesWritable value = new BytesWritable(data);
writer.append(key, value);
}
} finally {
writer.close();
}
}
}
@Test
public void extractNamecoinFieldFirstUpdate() throws HiveException {
String firstUpdateScript ="520A642F666C6173687570641460C7B068EDEA60281DAF424C38D8DAB87C96CF993D7B226970223A223134352E3234392E3130362E323238222C226D6170223A7B222A223A7B226970223A223134352E3234392E3130362E323238227D7D7D6D6D76A91451B4FC93AAB8CBDBD0AC9BC8EAF824643FC1E29B88AC";
byte[] firstUpdateScriptBytes = BitcoinUtil.convertHexStringToByteArray(firstUpdateScript);
NamecoinExtractFieldUDF nefu = new NamecoinExtractFieldUDF();
ObjectInspector[] arguments = new ObjectInspector[1];
arguments[0] = PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;;
nefu.initialize(arguments);
GenericUDF.DeferredObject[] doa = new GenericUDF.DeferredObject[1];
doa[0]=new GenericUDF.DeferredJavaObject(new BytesWritable(firstUpdateScriptBytes));
List<Text> resultList = (List<Text>) nefu.evaluate(doa);
Text[] result=resultList.toArray(new Text[resultList.size()]);
assertNotNull( result,"Valid result obtained");
// test for domain name
assertEquals("d/flashupd",result[0].toString(),"Domain name of first update detected correctly");
// test for domain value
assertEquals("{\"ip\":\"145.249.106.228\",\"map\":{\"*\":{\"ip\":\"145.249.106.228\"}}}",result[1].toString(),"Domain value of first update detected correctly");
}
@Override
public void run() {
for (int i = 0; i < count; i++) {
try {
int byteSize = RANDOM.nextInt(BYTE_COUNT);
byte[] bytes = new byte[byteSize];
System.arraycopy(BYTES, 0, bytes, 0, byteSize);
Writable param = new BytesWritable(bytes);
Writable value = client.call(param, address);
Thread.sleep(RANDOM.nextInt(20));
} catch (Exception e) {
LOG.fatal("Caught: " + e);
failed = true;
}
}
}
/** Compute the (offset+1)th to (offset+length)th digits. */
protected void map(LongWritable offset, IntWritable length,
final Context context) throws IOException, InterruptedException {
LOG.info("offset=" + offset + ", length=" + length);
// compute digits
final byte[] bytes = new byte[length.get() >> 1];
long d = offset.get();
for (int i = 0; i < bytes.length; d += 4) {
final long digits = hexDigits(d);
bytes[i++] = (byte) (digits >> 8);
bytes[i++] = (byte) digits;
}
// output map results
context.write(offset, new BytesWritable(bytes));
}
protected void readObject(Writable obj, DataInputStream inStream) throws IOException {
int numBytes = WritableUtils.readVInt(inStream);
byte[] buffer;
// For BytesWritable and Text, use the specified length to set the length
// this causes the "obvious" translations to work. So that if you emit
// a string "abc" from C++, it shows up as "abc".
if (obj instanceof BytesWritable) {
buffer = new byte[numBytes];
inStream.readFully(buffer);
((BytesWritable) obj).set(buffer, 0, numBytes);
} else if (obj instanceof Text) {
buffer = new byte[numBytes];
inStream.readFully(buffer);
((Text) obj).set(buffer);
} else {
obj.readFields(inStream);
}
}
@Override
public int compare(GroupbyKey key1, GroupbyKey key2){
List<BytesWritable> list1 = key1.getValue();
List<BytesWritable> list2 = key2.getValue();
if(list1 == null || list2 == null || list1.size() != list2.size())
throw new IllegalArgumentException("2 list of groupby fields must be non-null and have the same size");
ListIterator<BytesWritable> e1 = list1.listIterator();
ListIterator<BytesWritable> e2 = list2.listIterator();
while(e1.hasNext() && e2.hasNext()){
int r = Bytes.compareTo(e1.next().copyBytes(), e2.next().copyBytes());
if(r != 0)
return r;
}
return 0;
}
@Override
public int compare(GroupbyKey key1, GroupbyKey key2) {
List<BytesWritable> list1 = key1.getValue();
List<BytesWritable> list2 = key2.getValue();
if (list1 == null || list2 == null || list1.size() != list2.size()) {
throw new IllegalArgumentException("2 list of groupby fields must be non-null and have the same size");
}
ListIterator<BytesWritable> e1 = list1.listIterator();
ListIterator<BytesWritable> e2 = list2.listIterator();
while (e1.hasNext() && e2.hasNext()) {
int r = Bytes.compareTo(e1.next().copyBytes(), e2.next().copyBytes());
if (r != 0) {
return r;
}
}
return 0;
}
public void testWritable() throws Exception {
Random r = new Random();
Writable[] writs = {
new BooleanWritable(r.nextBoolean()),
new FloatWritable(r.nextFloat()),
new FloatWritable(r.nextFloat()),
new IntWritable(r.nextInt()),
new LongWritable(r.nextLong()),
new BytesWritable("dingo".getBytes()),
new LongWritable(r.nextLong()),
new IntWritable(r.nextInt()),
new BytesWritable("yak".getBytes()),
new IntWritable(r.nextInt())
};
TupleWritable sTuple = makeTuple(writs);
ByteArrayOutputStream out = new ByteArrayOutputStream();
sTuple.write(new DataOutputStream(out));
ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
TupleWritable dTuple = new TupleWritable();
dTuple.readFields(new DataInputStream(in));
assertTrue("Failed to write/read tuple", sTuple.equals(dTuple));
}
/**
* Copy the value into BytesWritable. The input BytesWritable will be
* automatically resized to the actual value size. The implementation
* directly uses the buffer inside BytesWritable for storing the value.
* The call does not require the value length to be known.
*
* @param value
* @throws IOException
*/
public long getValue(BytesWritable value) throws IOException {
DataInputStream dis = getValueStream();
int size = 0;
try {
int remain;
while ((remain = valueBufferInputStream.getRemain()) > 0) {
value.setSize(size + remain);
dis.readFully(value.getBytes(), size, remain);
size += remain;
}
return value.getLength();
} finally {
dis.close();
}
}
@SuppressWarnings("rawtypes")
public static Class cloverType2Hadoop(DataFieldMetadata field) throws IOException{
switch (field.getDataType()){
case BOOLEAN:
return BooleanWritable.class;
case BYTE:
case CBYTE:
return BytesWritable.class;
case DATE:
return LongWritable.class;
case INTEGER:
return IntWritable.class;
case LONG:
return LongWritable.class;
case NUMBER:
return DoubleWritable.class;
case STRING:
return Text.class;
default:
throw new IOException(String.format("Unsupported CloverDX data type \"%s\" of field \"%s\" in conversion to Hadoop.",field.getDataType().getName(),field.getName()));
}
}
protected byte inferPigDataType(Type t) {
if (t == BytesWritable.class) return DataType.BYTEARRAY;
else if (t == Text.class) return DataType.CHARARRAY;
else if (t == IntWritable.class) return DataType.INTEGER;
else if (t == LongWritable.class) return DataType.LONG;
else if (t == FloatWritable.class) return DataType.FLOAT;
else if (t == DoubleWritable.class) return DataType.DOUBLE;
else if (t == BooleanWritable.class) return DataType.BOOLEAN;
else if (t == ByteWritable.class) return DataType.BYTE;
else if (t == DateTimeWritable.class) return DataType.DATETIME;
// not doing maps or other complex types for now
else return DataType.ERROR;
}
private void logGroupbyKeyValue(List<GroupbyKeyValue> keyValues) {
for (GroupbyKeyValue keyValue : keyValues) {
GroupbyKey key = keyValue.getKey();
List<String> keys = new ArrayList<String>();
for (BytesWritable bytes : key.getValue()) {
keys.add(new String(bytes.copyBytes()));
}
List<Double> vals = new ArrayList<Double>();
GroupbyValue val = keyValue.getValue();
for (DoubleWritable dw : val.getValue()) {
vals.add(dw.get());
}
if (LOG.isDebugEnabled()) LOG.debug("KEY: " + keys + ", VALUE: " + vals);
}
}
public static void filter(String alignpath,
String outpath,
int nummappers,
int numreducers) throws IOException, Exception
{
System.out.println("NUM_FMAP_TASKS: " + nummappers);
System.out.println("NUM_FREDUCE_TASKS: " + numreducers);
JobConf conf = new JobConf(FilterAlignments.class);
conf.setJobName("FilterAlignments");
conf.setNumMapTasks(nummappers);
conf.setNumReduceTasks(numreducers);
FileInputFormat.addInputPath(conf, new Path(alignpath));
conf.setMapperClass(FilterMapClass.class);
conf.setInputFormat(SequenceFileInputFormat.class);
conf.setMapOutputKeyClass(IntWritable.class);
conf.setMapOutputValueClass(BytesWritable.class);
conf.setCombinerClass(FilterCombinerClass.class);
conf.setReducerClass(FilterReduceClass.class);
conf.setOutputKeyClass(IntWritable.class);
conf.setOutputValueClass(BytesWritable.class);
conf.setOutputFormat(SequenceFileOutputFormat.class);
Path oPath = new Path(outpath);
FileOutputFormat.setOutputPath(conf, oPath);
System.err.println(" Removing old results");
FileSystem.get(conf).delete(oPath);
JobClient.runJob(conf);
System.err.println("FilterAlignments Finished");
}
@Test
public void emptyListOfFractions() {
UpdateDoublesSketch sketch = DoublesSketch.builder().build();
sketch.update(1);
sketch.update(2);
sketch.update(3);
List<Double> result = new GetQuantilesFromDoublesSketchUDF().evaluate(new BytesWritable(sketch.toByteArray()));
Assert.assertNotNull(result);
Assert.assertEquals(result.size(), 0);
}
@Override
public void merge(final @SuppressWarnings("deprecation") AggregationBuffer buf, final Object data)
throws HiveException {
if (data == null) { return; }
final ArrayOfDoublesUnionState state = (ArrayOfDoublesUnionState) buf;
if (!state.isInitialized()) {
initializeState(state, data);
}
final Memory serializedSketch = BytesWritableHelper.wrapAsMemory(
(BytesWritable) intermediateInspector_.getStructFieldData(
data, intermediateInspector_.getStructFieldRef(SKETCH_FIELD)));
state.update(ArrayOfDoublesSketches.wrapSketch(serializedSketch));
}
static void writeFile(NameNode namenode, Configuration conf, Path name,
short replication)
throws IOException, TimeoutException, InterruptedException {
FileSystem fileSys = FileSystem.get(conf);
SequenceFile.Writer writer =
SequenceFile.createWriter(fileSys, conf, name,
BytesWritable.class, BytesWritable.class,
CompressionType.NONE);
writer.append(new BytesWritable(), new BytesWritable());
writer.close();
fileSys.setReplication(name, replication);
DFSTestUtil.waitReplication(fileSys, name, replication);
}
@Test
public void complete1ModeDefaultK() throws Exception {
ObjectInspector[] inspectors = new ObjectInspector[] { binaryInspector };
GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false, false);
try (GenericUDAFEvaluator eval = new UnionStringsSketchUDAF().getEvaluator(info)) {
ObjectInspector resultInspector = eval.init(Mode.COMPLETE, inspectors);
DataToDoublesSketchUDAFTest.checkResultInspector(resultInspector);
@SuppressWarnings("unchecked")
ItemsUnionState<String> state = (ItemsUnionState<String>) eval.getNewAggregationBuffer();
ItemsSketch<String> sketch1 = ItemsSketch.getInstance(comparator);
sketch1.update("a");
eval.iterate(state, new Object[] { new BytesWritable(sketch1.toByteArray(serDe)) });
ItemsSketch<String> sketch2 = ItemsSketch.getInstance(comparator);
sketch2.update("b");
eval.iterate(state, new Object[] { new BytesWritable(sketch2.toByteArray(serDe)) });
BytesWritable bytes = (BytesWritable) eval.terminate(state);
ItemsSketch<String> resultSketch = ItemsSketch.getInstance(BytesWritableHelper.wrapAsMemory(bytes), comparator, serDe);
Assert.assertEquals(resultSketch.getK(), 128);
Assert.assertEquals(resultSketch.getRetainedItems(), 2);
Assert.assertEquals(resultSketch.getMinValue(), "a");
Assert.assertEquals(resultSketch.getMaxValue(), "b");
eval.reset(state);
Assert.assertNull(eval.terminate(state));
}
}
public synchronized void reduce(BytesWritable mer, Iterator<IntWritable> values,
OutputCollector<Text, Text> output, Reporter reporter)
throws IOException
{
int cnt = 0;
builder.setLength(0);
while (values.hasNext())
{
cnt++;
if (SHOW_POS)
{
builder.append('\t');
builder.append(values.next().get());
}
}
String val = DNAString.bytesToString(DNAString.bytesWritableDNAToArr(mer));
mertext.set(val);
if (SHOW_POS)
{
builder.insert(0, cnt);
String locs = builder.toString();
locations.set(locs);
}
else
{
locations.set(Integer.toString(cnt));
}
output.collect(mertext, locations);
}
public MapTask(String jobFile, TaskAttemptID taskId,
int partition, String splitClass, BytesWritable split
) {
super(jobFile, taskId, partition);
this.splitClass = splitClass;
this.split = split;
}