下面列出了org.apache.hadoop.io.BytesWritable#getBytes ( ) 实例代码，或者点击链接到github查看源代码，也可以在右侧发表评论。
public BytesWritable evaluate(BytesWritable wkb, int wkid) throws UDFArgumentException {
try {
SpatialReference spatialReference = null;
if (wkid != GeometryUtils.WKID_UNKNOWN) {
spatialReference = SpatialReference.create(wkid);
}
byte [] byteArr = wkb.getBytes();
ByteBuffer byteBuf = ByteBuffer.allocate(byteArr.length);
byteBuf.put(byteArr);
OGCGeometry ogcObj = OGCGeometry.fromBinary(byteBuf);
ogcObj.setSpatialReference(spatialReference);
if (ogcObj.geometryType().equals("Polygon")) {
return GeometryUtils.geometryToEsriShapeBytesWritable(ogcObj);
} else {
LogUtils.Log_InvalidType(LOG, GeometryUtils.OGCType.ST_POLYGON, GeometryUtils.OGCType.UNKNOWN);
return null;
}
} catch (Exception e) { // IllegalArgumentException, GeometryException
LOG.error(e.getMessage());
return null;
}
}
public BytesWritable evaluate(BytesWritable wkb, int wkid) throws UDFArgumentException {
try {
SpatialReference spatialReference = null;
if (wkid != GeometryUtils.WKID_UNKNOWN) {
spatialReference = SpatialReference.create(wkid);
}
byte [] byteArr = wkb.getBytes();
ByteBuffer byteBuf = ByteBuffer.allocate(byteArr.length);
byteBuf.put(byteArr);
OGCGeometry ogcObj = OGCGeometry.fromBinary(byteBuf);
ogcObj.setSpatialReference(spatialReference);
if (ogcObj.geometryType().equals("Point")) {
return GeometryUtils.geometryToEsriShapeBytesWritable(ogcObj);
} else {
LogUtils.Log_InvalidType(LOG, GeometryUtils.OGCType.ST_POINT, GeometryUtils.OGCType.UNKNOWN);
return null;
}
} catch (Exception e) { // IllegalArgumentException, GeometryException
LOG.error(e.getMessage());
return null;
}
}
/**
* @param compressionAlgo
* The compression algorithm to be used to for compression.
* @throws IOException
*/
public WBlockState(Algorithm compressionAlgo, FSDataOutputStream fsOut,
BytesWritable fsOutputBuffer, Configuration conf) throws IOException {
this.compressAlgo = compressionAlgo;
this.fsOut = fsOut;
this.posStart = fsOut.getPos();
fsOutputBuffer.setCapacity(TFile.getFSOutputBufferSize(conf));
this.fsBufferedOutput =
new SimpleBufferedOutputStream(this.fsOut, fsOutputBuffer.getBytes());
this.compressor = compressAlgo.getCompressor();
try {
this.out =
compressionAlgo.createCompressionStream(fsBufferedOutput,
compressor, 0);
} catch (IOException e) {
compressAlgo.returnCompressor(compressor);
throw e;
}
}
public BytesWritable evaluate(BytesWritable wkb, int wkid) throws UDFArgumentException {
try {
SpatialReference spatialReference = null;
if (wkid != GeometryUtils.WKID_UNKNOWN) {
spatialReference = SpatialReference.create(wkid);
}
byte [] byteArr = wkb.getBytes();
ByteBuffer byteBuf = ByteBuffer.allocate(byteArr.length);
byteBuf.put(byteArr);
OGCGeometry ogcObj = OGCGeometry.fromBinary(byteBuf);
ogcObj.setSpatialReference(spatialReference);
String gType = ogcObj.geometryType();
if (gType.equals("MultiPoint") || gType.equals("Point")) {
return GeometryUtils.geometryToEsriShapeBytesWritable(ogcObj);
} else {
LogUtils.Log_InvalidType(LOG, GeometryUtils.OGCType.ST_LINESTRING, GeometryUtils.OGCType.UNKNOWN);
return null;
}
} catch (Exception e) { // IllegalArgumentException, GeometryException
LOG.error(e.getMessage());
return null;
}
}
@Override
public void map(LongWritable key, BytesWritable value, Context context)
throws IOException, InterruptedException {
String fileName = new String(value.getBytes(), 0,
value.getLength(), charsetUTF8);
Path path = new Path(fileName);
FSDataOutputStream dos =
FileSystem.create(fs, path, new FsPermission(GRIDMIX_DISTCACHE_FILE_PERM));
int size = 0;
for (long bytes = key.get(); bytes > 0; bytes -= size) {
r.nextBytes(val.getBytes());
size = (int)Math.min(val.getLength(), bytes);
dos.write(val.getBytes(), 0, size);// Write to distCache file
}
dos.close();
}
protected static HashMap<Integer, HashMap<Integer, Declaration>> collectDeclarations(final SequenceFile.Reader ar, List<ChangedFile> snapshot) throws IOException {
HashMap<Integer, HashMap<Integer, Declaration>> fileNodeDeclaration = new HashMap<Integer, HashMap<Integer, Declaration>>();
for (int fileIndex = 0; fileIndex < snapshot.size(); fileIndex++) {
ChangedFile cf = snapshot.get(fileIndex);
long astpos = cf.getKey();
if (!cf.getAst())
continue;
if (astpos > -1) {
ar.seek(astpos);
Writable astkey = new LongWritable();
BytesWritable val = new BytesWritable();
ar.next(astkey, val);
byte[] bytes = val.getBytes();
ASTRoot root = ASTRoot.parseFrom(CodedInputStream.newInstance(bytes, 0, val.getLength()));
HashMap<Integer, Declaration> nodeDeclaration = collectDeclarations(root);
fileNodeDeclaration.put(fileIndex, nodeDeclaration);
}
}
return fileNodeDeclaration;
}
@Override
public void write(BytesWritable key, BytesWritable value)
throws IOException, InterruptedException {
// Mapreduce reuses the same Text objects and hence sometimes they have some
// additional left over garbage at the end from previous keys. So we need to
// retrieve the String objects which do not have the additional garbage. getBytes()
// call does not work well.
byte[] row = new byte[key.getLength()];
byte[] val = new byte[value.getLength()];
for (int i = 0; i < row.length; i++) {
row[i] = key.getBytes()[i];
}
for (int i = 0; i < val.length; i++) {
val[i] = value.getBytes()[i];
}
writer.append(new KeyValue(row,
Bytes.toBytes("cf"),
Bytes.toBytes(""),
val));
}
public static BytesWritable bytes(int... items) {
BytesWritable result = new BytesWritable();
result.setSize(items.length);
for(int i=0; i < items.length; ++i) {
result.getBytes()[i] = (byte) items[i];
}
return result;
}
protected static Message getMessage(final SequenceFile.Reader ar, final ChangedFile cf, final int nodeId) {
long astpos = cf.getKey();
if (cf.getAst() && astpos > -1) {
try {
ar.seek(astpos);
Writable astkey = new LongWritable();
BytesWritable val = new BytesWritable();
ar.next(astkey, val);
byte[] bytes = val.getBytes();
ASTRoot root = ASTRoot.parseFrom(CodedInputStream.newInstance(bytes, 0, val.getLength()));
return getMessage(root, nodeId);
} catch (IOException e) {}
}
return null;
}
public String getString(Object k) {
BytesWritable bytes = get(k);
if (null == bytes) {
return null;
} else {
try {
return new String(bytes.getBytes(), 0, bytes.getLength(), "UTF-8");
} catch (UnsupportedEncodingException uee) {
// Shouldn't happen; UTF-8 is always supported.
throw new RuntimeException(uee);
}
}
}
private static Rename[] readRenameOptions(DataInputStream in) throws IOException {
BytesWritable writable = new BytesWritable();
writable.readFields(in);
byte[] bytes = writable.getBytes();
Rename[] options = new Rename[bytes.length];
for (int i = 0; i < bytes.length; i++) {
options[i] = Rename.valueOf(bytes[i]);
}
return options;
}
private void printBytes(BytesWritable key) {
byte[] bytes = key.getBytes();
for (int i = 0; i < key.getLength(); i++) {
System.out.printf("%02x", bytes[i]);
}
System.out.println();
}
/**
* Validate setupGenerateDistCacheData by validating <li>permissions of the
* distributed cache directory and <li>content of the generated sequence file.
* This includes validation of dist cache file paths and their file sizes.
*/
private void doValidateSetupGenDC(
RecordReader<LongWritable, BytesWritable> reader, FileSystem fs,
long[] sortedFileSizes) throws IOException, InterruptedException {
// Validate permissions of dist cache directory
Path distCacheDir = dce.getDistributedCacheDir();
assertEquals(
"Wrong permissions for distributed cache dir " + distCacheDir,
fs.getFileStatus(distCacheDir).getPermission().getOtherAction()
.and(FsAction.EXECUTE), FsAction.EXECUTE);
// Validate the content of the sequence file generated by
// dce.setupGenerateDistCacheData().
LongWritable key = new LongWritable();
BytesWritable val = new BytesWritable();
for (int i = 0; i < sortedFileSizes.length; i++) {
assertTrue("Number of files written to the sequence file by "
+ "setupGenerateDistCacheData is less than the expected.",
reader.nextKeyValue());
key = reader.getCurrentKey();
val = reader.getCurrentValue();
long fileSize = key.get();
String file = new String(val.getBytes(), 0, val.getLength());
// Dist Cache files should be sorted based on file size.
assertEquals("Dist cache file size is wrong.", sortedFileSizes[i],
fileSize);
// Validate dist cache file path.
// parent dir of dist cache file
Path parent = new Path(file).getParent().makeQualified(fs.getUri(),fs.getWorkingDirectory());
// should exist in dist cache dir
assertTrue("Public dist cache file path is wrong.",
distCacheDir.equals(parent));
}
}
public void map(LongWritable key, BytesWritable value, Context context )
throws IOException, InterruptedException {
String val = new String(value.getBytes(), 0, value.getLength()-1);
String[] strs = val.split(" ");
context.write(new Text(strs[0]), new IntWritable(Integer.parseInt(strs[1])));
}
@Override
public void readFields(DataInput in) throws IOException {
// 0. Timestamp
Text tms_text = new Text(tms);
tms_text.readFields(in);
tms = tms_text.toString();
// 1. integers
IntWritable intw = new IntWritable();
for (int i = 0; i < num.length; i++) {
intw.readFields(in);
num[i] = intw.get();
}
intw.readFields(in);
int1 = intw.get();
intw.readFields(in);
int2 = intw.get();
// 2. strings
Text txt = new Text();
for (int i = 0; i < strings.length; i++) {
txt.readFields(in);
strings[i] = txt.toString();
}
txt.readFields(in);
st1 = txt.toString();
// 3. doubles
DoubleWritable dw = new DoubleWritable();
for (int i = 0; i < dubs.length; i++) {
dw.readFields(in);
dubs[i] = dw.get();
}
dw.readFields(in);
db = dw.get();
// 4. floats
FloatWritable fw = new FloatWritable();
for (int i = 0; i < fts.length; i++) {
fw.readFields(in);
fts[i] = fw.get();
}
fw.readFields(in);
ft = fw.get();
// 5. longs
LongWritable lw = new LongWritable();
for (int i = 0; i < lngs.length; i++) {
lw.readFields(in);
lngs[i] = lw.get();
}
lw.readFields(in);
lng = lw.get();
// 6. booleans
BooleanWritable bw = new BooleanWritable();
for (int i = 0; i < bools.length; ++i) {
bw.readFields(in);
bools[i] = bw.get();
}
bw.readFields(in);
bool = bw.get();
// 7. shorts
ShortWritable sw = new ShortWritable();
for (int i = 0; i < shrts.length; ++i) {
sw.readFields(in);
shrts[i] = sw.get();
}
sw.readFields(in);
shrt = sw.get();
// 8. bytes
BytesWritable btsw = new BytesWritable();
btsw.readFields(in);
byte[] buffer = btsw.getBytes();
bts = new byte[btsw.getLength()];
for (int i = 0; i < btsw.getLength(); i++) {
bts[i] = buffer[i];
}
}
private Object toHCat(Object val, HCatFieldSchema hfs) {
HCatFieldSchema.Type hfsType = hfs.getType();
if (val == null) {
return null;
}
Object retVal = null;
if (val instanceof Number) {
retVal = convertNumberTypes(val, hfs);
} else if (val instanceof Boolean) {
retVal = convertBooleanTypes(val, hfs);
} else if (val instanceof String) {
retVal = convertStringTypes(val, hfs);
} else if (val instanceof java.util.Date) {
retVal = converDateTypes(val, hfs);
} else if (val instanceof BytesWritable) {
if (hfsType == HCatFieldSchema.Type.BINARY) {
BytesWritable bw = (BytesWritable) val;
retVal = bw.getBytes();
}
} else if (val instanceof BlobRef) {
if (hfsType == HCatFieldSchema.Type.BINARY) {
BlobRef br = (BlobRef) val;
byte[] bytes = br.isExternal() ? br.toString().getBytes() : br
.getData();
retVal = bytes;
}
} else if (val instanceof ClobRef) {
retVal = convertClobType(val, hfs);
} else {
throw new UnsupportedOperationException("Objects of type "
+ val.getClass().getName() + " are not suported");
}
if (retVal == null) {
LOG.error("Unable to convert [" + val
+ "] of type " + val.getClass().getName()
+ " to HCatalog type " + hfs.getTypeString());
}
return retVal;
}
protected InputStream getInputStreamFromKeyValue(NullWritable key, BytesWritable value) {
return new ByteArrayInputStream(value.getBytes(), 0, value.getLength());
}
private void runRandomTests(CompressionCodec codec) throws IOException {
StringBuilder fileName = new StringBuilder("testFormat.txt");
if (codec != null) {
fileName.append(".gz");
}
localFs.delete(workDir, true);
Path file = new Path(workDir, fileName.toString());
int seed = new Random().nextInt();
LOG.info("Seed = " + seed);
Random random = new Random(seed);
int MAX_TESTS = 20;
LongWritable key = new LongWritable();
BytesWritable value = new BytesWritable();
for (int i = 0; i < MAX_TESTS; i++) {
LOG.info("----------------------------------------------------------");
// Maximum total records of 999
int totalRecords = random.nextInt(999)+1;
// Test an empty file
if (i == 8) {
totalRecords = 0;
}
// Maximum bytes in a record of 100K
int recordLength = random.nextInt(1024*100)+1;
// For the 11th test, force a record length of 1
if (i == 10) {
recordLength = 1;
}
// The total bytes in the test file
int fileSize = (totalRecords * recordLength);
LOG.info("totalRecords=" + totalRecords + " recordLength="
+ recordLength);
// Create the job
JobConf job = new JobConf(defaultConf);
if (codec != null) {
ReflectionUtils.setConf(codec, job);
}
// Create the test file
ArrayList<String> recordList
= createFile(file, codec, recordLength, totalRecords);
assertTrue(localFs.exists(file));
//set the fixed length record length config property for the job
FixedLengthInputFormat.setRecordLength(job, recordLength);
int numSplits = 1;
// Arbitrarily set number of splits.
if (i > 0) {
if (i == (MAX_TESTS-1)) {
// Test a split size that is less than record len
numSplits = (int)(fileSize/Math.floor(recordLength/2));
} else {
if (MAX_TESTS % i == 0) {
// Let us create a split size that is forced to be
// smaller than the end file itself, (ensures 1+ splits)
numSplits = fileSize/(fileSize - random.nextInt(fileSize));
} else {
// Just pick a random split size with no upper bound
numSplits = Math.max(1, fileSize/random.nextInt(Integer.MAX_VALUE));
}
}
LOG.info("Number of splits set to: " + numSplits);
}
// Setup the input path
FileInputFormat.setInputPaths(job, workDir);
// Try splitting the file in a variety of sizes
FixedLengthInputFormat format = new FixedLengthInputFormat();
format.configure(job);
InputSplit splits[] = format.getSplits(job, numSplits);
LOG.info("Actual number of splits = " + splits.length);
// Test combined split lengths = total file size
long recordOffset = 0;
int recordNumber = 0;
for (InputSplit split : splits) {
RecordReader<LongWritable, BytesWritable> reader =
format.getRecordReader(split, job, voidReporter);
Class<?> clazz = reader.getClass();
assertEquals("RecordReader class should be FixedLengthRecordReader:",
FixedLengthRecordReader.class, clazz);
// Plow through the records in this split
while (reader.next(key, value)) {
assertEquals("Checking key", (long)(recordNumber*recordLength),
key.get());
String valueString =
new String(value.getBytes(), 0, value.getLength());
assertEquals("Checking record length:", recordLength,
value.getLength());
assertTrue("Checking for more records than expected:",
recordNumber < totalRecords);
String origRecord = recordList.get(recordNumber);
assertEquals("Checking record content:", origRecord, valueString);
recordNumber++;
}
reader.close();
}
assertEquals("Total original records should be total read records:",
recordList.size(), recordNumber);
}
}
private void runRandomTests(CompressionCodec codec) throws IOException {
StringBuilder fileName = new StringBuilder("testFormat.txt");
if (codec != null) {
fileName.append(".gz");
}
localFs.delete(workDir, true);
Path file = new Path(workDir, fileName.toString());
int seed = new Random().nextInt();
LOG.info("Seed = " + seed);
Random random = new Random(seed);
int MAX_TESTS = 20;
LongWritable key = new LongWritable();
BytesWritable value = new BytesWritable();
for (int i = 0; i < MAX_TESTS; i++) {
LOG.info("----------------------------------------------------------");
// Maximum total records of 999
int totalRecords = random.nextInt(999)+1;
// Test an empty file
if (i == 8) {
totalRecords = 0;
}
// Maximum bytes in a record of 100K
int recordLength = random.nextInt(1024*100)+1;
// For the 11th test, force a record length of 1
if (i == 10) {
recordLength = 1;
}
// The total bytes in the test file
int fileSize = (totalRecords * recordLength);
LOG.info("totalRecords=" + totalRecords + " recordLength="
+ recordLength);
// Create the job
JobConf job = new JobConf(defaultConf);
if (codec != null) {
ReflectionUtils.setConf(codec, job);
}
// Create the test file
ArrayList<String> recordList
= createFile(file, codec, recordLength, totalRecords);
assertTrue(localFs.exists(file));
//set the fixed length record length config property for the job
FixedLengthInputFormat.setRecordLength(job, recordLength);
int numSplits = 1;
// Arbitrarily set number of splits.
if (i > 0) {
if (i == (MAX_TESTS-1)) {
// Test a split size that is less than record len
numSplits = (int)(fileSize/Math.floor(recordLength/2));
} else {
if (MAX_TESTS % i == 0) {
// Let us create a split size that is forced to be
// smaller than the end file itself, (ensures 1+ splits)
numSplits = fileSize/(fileSize - random.nextInt(fileSize));
} else {
// Just pick a random split size with no upper bound
numSplits = Math.max(1, fileSize/random.nextInt(Integer.MAX_VALUE));
}
}
LOG.info("Number of splits set to: " + numSplits);
}
// Setup the input path
FileInputFormat.setInputPaths(job, workDir);
// Try splitting the file in a variety of sizes
FixedLengthInputFormat format = new FixedLengthInputFormat();
format.configure(job);
InputSplit splits[] = format.getSplits(job, numSplits);
LOG.info("Actual number of splits = " + splits.length);
// Test combined split lengths = total file size
long recordOffset = 0;
int recordNumber = 0;
for (InputSplit split : splits) {
RecordReader<LongWritable, BytesWritable> reader =
format.getRecordReader(split, job, voidReporter);
Class<?> clazz = reader.getClass();
assertEquals("RecordReader class should be FixedLengthRecordReader:",
FixedLengthRecordReader.class, clazz);
// Plow through the records in this split
while (reader.next(key, value)) {
assertEquals("Checking key", (long)(recordNumber*recordLength),
key.get());
String valueString =
new String(value.getBytes(), 0, value.getLength());
assertEquals("Checking record length:", recordLength,
value.getLength());
assertTrue("Checking for more records than expected:",
recordNumber < totalRecords);
String origRecord = recordList.get(recordNumber);
assertEquals("Checking record content:", origRecord, valueString);
recordNumber++;
}
reader.close();
}
assertEquals("Total original records should be total read records:",
recordList.size(), recordNumber);
}
}
private static ByteBuffer getShapeByteBuffer(BytesWritable geomref){
byte [] geomBytes = geomref.getBytes();
int offset = SIZE_WKID + SIZE_TYPE;
return ByteBuffer.wrap(geomBytes, offset, geomBytes.length - offset).slice().order(ByteOrder.LITTLE_ENDIAN);
}