下面列出了org.apache.hadoop.io.BytesWritable#getLength ( ) 实例代码，或者点击链接到github查看源代码，也可以在右侧发表评论。
public DoubleWritable evaluate(BytesWritable geomref) {
if (geomref == null || geomref.getLength() == 0) {
LogUtils.Log_ArgumentsNull(LOG);
return null;
}
OGCGeometry ogcGeometry = GeometryUtils.geometryFromEsriShape(geomref);
if (ogcGeometry == null){
return null;
}
if (!ogcGeometry.isMeasured()) {
LogUtils.Log_NotMeasured(LOG);
return null;
}
switch(GeometryUtils.getType(geomref)) {
case ST_POINT:
OGCPoint pt = (OGCPoint)ogcGeometry;
resultDouble.set(pt.M());
return resultDouble;
default:
LogUtils.Log_InvalidType(LOG, GeometryUtils.OGCType.ST_POINT, GeometryUtils.getType(geomref));
return null;
}
}
@Override
public LWDocument[] toDocuments(Writable key, Writable value, Reporter reporter,
Configuration conf) throws IOException {
LWDocument doc = createDocument(key.toString(), null);
if (value instanceof Text) {
doc.setContent(((Text) value).getBytes());
reporter.getCounter(Counters.TEXT).increment(1);
} else if (value instanceof BytesWritable) {
// Copy the bytes for this one
BytesWritable value_ = (BytesWritable) value;
byte[] data = new byte[value_.getLength()];
doc.setContent(data);
System.arraycopy(value_.getBytes(), 0, data, 0, value_.getLength());
reporter.getCounter(Counters.BYTES_WRITABLE).increment(1);
} else {
doc.setContent(WritableUtils.toByteArray(value));
reporter.getCounter(Counters.RAW_WRITABLE).increment(1);
}
return new LWDocument[] {doc};
}
public BytesWritable evaluate(BytesWritable geometryref)
{
if (geometryref == null || geometryref.getLength() == 0) {
LogUtils.Log_ArgumentsNull(LOG);
return null;
}
OGCGeometry ogcGeometry = GeometryUtils.geometryFromEsriShape(geometryref);
if (ogcGeometry == null){
LogUtils.Log_ArgumentsNull(LOG);
return null;
}
int wkid = GeometryUtils.getWKID(geometryref);
SpatialReference spatialReference = null;
if (wkid != GeometryUtils.WKID_UNKNOWN) {
spatialReference = SpatialReference.create(wkid);
}
Envelope envBound = new Envelope();
ogcGeometry.getEsriGeometry().queryEnvelope(envBound);
return GeometryUtils.geometryToEsriShapeBytesWritable(OGCGeometry.createFromEsriGeometry(envBound,
spatialReference));
}
public Text evaluate(BytesWritable geomref){
if (geomref == null || geomref.getLength() == 0){
LogUtils.Log_ArgumentsNull(LOG);
return null;
}
OGCGeometry ogcGeometry = GeometryUtils.geometryFromEsriShape(geomref);
if (ogcGeometry == null){
LogUtils.Log_ArgumentsNull(LOG);
return null;
}
Geometry esriGeom = ogcGeometry.getEsriGeometry();
int wkid = GeometryUtils.getWKID(geomref);
return new Text(GeometryEngine.geometryToJson(wkid, esriGeom));
}
public DoubleWritable evaluate(BytesWritable geomref) {
if (geomref == null || geomref.getLength() == 0) {
LogUtils.Log_ArgumentsNull(LOG);
return null;
}
OGCGeometry ogcGeometry = GeometryUtils.geometryFromEsriShape(geomref);
if (ogcGeometry == null) {
LogUtils.Log_ArgumentsNull(LOG);
return null;
}
if (!ogcGeometry.is3D()) {
LogUtils.Log_Not3D(LOG);
return null;
}
resultDouble.set(ogcGeometry.MinZ());
return resultDouble;
}
public BytesWritable evaluate(BytesWritable geomref) {
if (geomref == null || geomref.getLength() == 0){
LogUtils.Log_ArgumentsNull(LOG);
return null;
}
OGCGeometry ogcGeometry = GeometryUtils.geometryFromEsriShape(geomref);
if (ogcGeometry == null){
LogUtils.Log_ArgumentsNull(LOG);
return null;
}
try {
// Get Esri shape representation
Geometry esriGeometry = ogcGeometry.getEsriGeometry();
byte[] esriShape = GeometryEngine.geometryToEsriShape(esriGeometry);
return new BytesWritable(esriShape);
} catch (Exception e){
LOG.error(e.getMessage());
return null;
}
}
public void printFile(String path) throws Exception {
FileSystem fileSystem = FileUtil.getFileSystem(path);
Path fsPath = new Path(path);
SequenceFile.Reader reader = new SequenceFile.Reader(fileSystem, fsPath,
new Configuration());
LongWritable key = (LongWritable) reader.getKeyClass().newInstance();
BytesWritable value = (BytesWritable) reader.getValueClass().newInstance();
System.out.println("reading file " + path);
while (reader.next(key, value)) {
if (mPrintOffsetsOnly) {
System.out.println(Long.toString(key.get()));
} else {
byte[] nonPaddedBytes = new byte[value.getLength()];
System.arraycopy(value.getBytes(), 0, nonPaddedBytes, 0, value.getLength());
System.out.println(Long.toString(key.get()) + ": " + new String(nonPaddedBytes));
}
}
}
public DoubleWritable evaluate(BytesWritable geomref) {
if (geomref == null || geomref.getLength() == 0) {
LogUtils.Log_ArgumentsNull(LOG);
return null;
}
OGCGeometry ogcGeometry = GeometryUtils.geometryFromEsriShape(geomref);
if (ogcGeometry == null){
return null;
}
if (!ogcGeometry.is3D()) {
LogUtils.Log_Not3D(LOG);
return null;
}
switch(GeometryUtils.getType(geomref)) {
case ST_POINT:
OGCPoint pt = (OGCPoint)ogcGeometry;
resultDouble.set(pt.Z());
return resultDouble;
default:
LogUtils.Log_InvalidType(LOG, GeometryUtils.OGCType.ST_POINT, GeometryUtils.getType(geomref));
return null;
}
}
/**
* Copy the value into BytesWritable. The input BytesWritable will be
* automatically resized to the actual value size. The implementation
* directly uses the buffer inside BytesWritable for storing the value.
* The call does not require the value length to be known.
*
* @param value
* @throws IOException
*/
public long getValue(BytesWritable value) throws IOException {
DataInputStream dis = getValueStream();
int size = 0;
try {
int remain;
while ((remain = valueBufferInputStream.getRemain()) > 0) {
value.setSize(size + remain);
dis.readFully(value.getBytes(), size, remain);
size += remain;
}
return value.getLength();
} finally {
dis.close();
}
}
public BytesWritable evaluate(BytesWritable geometryref1, BytesWritable geometryref2)
{
if (geometryref1 == null || geometryref2 == null ||
geometryref1.getLength() == 0 || geometryref2.getLength() == 0) {
LogUtils.Log_ArgumentsNull(LOG);
return null;
}
if (!GeometryUtils.compareSpatialReferences(geometryref1, geometryref2)) {
LogUtils.Log_SRIDMismatch(LOG, geometryref1, geometryref2);
return null;
}
OGCGeometry ogcGeom1 = GeometryUtils.geometryFromEsriShape(geometryref1);
OGCGeometry ogcGeom2 = GeometryUtils.geometryFromEsriShape(geometryref2);
if (ogcGeom1 == null || ogcGeom2 == null){
LogUtils.Log_ArgumentsNull(LOG);
return null;
}
try {
OGCGeometry diffGeometry = ogcGeom1.symDifference(ogcGeom2);
return GeometryUtils.geometryToEsriShapeBytesWritable(diffGeometry);
} catch (Exception e) {
LogUtils.Log_InternalError(LOG, "ST_SymmetricDiff: " + e);
return null;
}
}
public static void writeBytesWritable(BytesWritable val, int paramIdx,
int sqlType, PreparedStatement s) throws SQLException {
if (null == val) {
s.setNull(paramIdx, sqlType);
} else {
// val.getBytes() is only valid in [0, len)
byte [] rawBytes = val.getBytes();
int len = val.getLength();
byte [] outBytes = new byte[len];
System.arraycopy(rawBytes, 0, outBytes, 0, len);
s.setBytes(paramIdx, outBytes);
}
}
@Override
protected void map(BytesWritable key, NullWritable value, Context output) throws IOException {
current[i] = new byte[key.getLength()];
System.arraycopy(key.getBytes(), 0, current[i], 0, key.getLength());
if (++i == current.length) {
LOG.debug("Persisting current.length={}, count={}, id={}, current={}, i=",
current.length, count, Bytes.toStringBinary(id), Bytes.toStringBinary(current[0]), i);
persist(output, count, prev, current, id);
i = 0;
if (first == null) {
first = current;
}
prev = current;
current = new byte[this.width][];
count += current.length;
output.setStatus("Count " + count);
if (count % wrap == 0) {
// this block of code turns the 1 million linked list of length 25 into one giant
//circular linked list of 25 million
circularLeftShift(first);
persist(output, -1, prev, first, null);
// At this point the entire loop has been flushed so we can add one of its nodes to the
// concurrent walker
if (numWalkers > 0) {
addFlushed(key.getBytes());
if (walkers.isEmpty()) {
startWalkers(numWalkers, conf, output);
}
}
first = null;
prev = null;
}
}
}
/**
* Validate setupGenerateDistCacheData by validating <li>permissions of the
* distributed cache directory and <li>content of the generated sequence file.
* This includes validation of dist cache file paths and their file sizes.
*/
private void doValidateSetupGenDC(
RecordReader<LongWritable, BytesWritable> reader, FileSystem fs,
long[] sortedFileSizes) throws IOException, InterruptedException {
// Validate permissions of dist cache directory
Path distCacheDir = dce.getDistributedCacheDir();
assertEquals(
"Wrong permissions for distributed cache dir " + distCacheDir,
fs.getFileStatus(distCacheDir).getPermission().getOtherAction()
.and(FsAction.EXECUTE), FsAction.EXECUTE);
// Validate the content of the sequence file generated by
// dce.setupGenerateDistCacheData().
LongWritable key = new LongWritable();
BytesWritable val = new BytesWritable();
for (int i = 0; i < sortedFileSizes.length; i++) {
assertTrue("Number of files written to the sequence file by "
+ "setupGenerateDistCacheData is less than the expected.",
reader.nextKeyValue());
key = reader.getCurrentKey();
val = reader.getCurrentValue();
long fileSize = key.get();
String file = new String(val.getBytes(), 0, val.getLength());
// Dist Cache files should be sorted based on file size.
assertEquals("Dist cache file size is wrong.", sortedFileSizes[i],
fileSize);
// Validate dist cache file path.
// parent dir of dist cache file
Path parent = new Path(file).getParent().makeQualified(fs.getUri(),fs.getWorkingDirectory());
// should exist in dist cache dir
assertTrue("Public dist cache file path is wrong.",
distCacheDir.equals(parent));
}
}
/** Concatenate map outputs. */
@Override
protected void reduce(LongWritable offset, Iterable<BytesWritable> values,
Context context) throws IOException, InterruptedException {
// read map outputs
for (BytesWritable bytes : values) {
for (int i = 0; i < bytes.getLength(); i++)
hex.add(bytes.getBytes()[i]);
}
LOG.info("hex.size() = " + hex.size());
}
/**
* Return the first point of the ST_Linestring.
* @param geomref hive geometry bytes
* @return byte-reference of the first ST_Point
*/
public BytesWritable evaluate(BytesWritable geomref) {
if (geomref == null || geomref.getLength() == 0){
LogUtils.Log_ArgumentsNull(LOG);
return null;
}
OGCGeometry ogcGeometry = GeometryUtils.geometryFromEsriShape(geomref);
if (ogcGeometry == null){
LogUtils.Log_ArgumentsNull(LOG);
return null;
}
if (GeometryUtils.getType(geomref) == GeometryUtils.OGCType.ST_LINESTRING) {
MultiPath lines = (MultiPath)(ogcGeometry.getEsriGeometry());
int wkid = GeometryUtils.getWKID(geomref);
SpatialReference spatialReference = null;
if (wkid != GeometryUtils.WKID_UNKNOWN) {
spatialReference = SpatialReference.create(wkid);
}
return GeometryUtils.geometryToEsriShapeBytesWritable(OGCGeometry.createFromEsriGeometry(lines.getPoint(0),
spatialReference));
} else {
LogUtils.Log_InvalidType(LOG, GeometryUtils.OGCType.ST_LINESTRING, GeometryUtils.getType(geomref));
return null;
}
}
static private byte[] pair(BytesWritable a, BytesWritable b) {
byte[] pairData = new byte[a.getLength()+ b.getLength()];
System.arraycopy(a.getBytes(), 0, pairData, 0, a.getLength());
System.arraycopy(b.getBytes(), 0, pairData, a.getLength(), b.getLength());
return pairData;
}
private void runRandomTests(CompressionCodec codec) throws IOException {
StringBuilder fileName = new StringBuilder("testFormat.txt");
if (codec != null) {
fileName.append(".gz");
}
localFs.delete(workDir, true);
Path file = new Path(workDir, fileName.toString());
int seed = new Random().nextInt();
LOG.info("Seed = " + seed);
Random random = new Random(seed);
int MAX_TESTS = 20;
LongWritable key = new LongWritable();
BytesWritable value = new BytesWritable();
for (int i = 0; i < MAX_TESTS; i++) {
LOG.info("----------------------------------------------------------");
// Maximum total records of 999
int totalRecords = random.nextInt(999)+1;
// Test an empty file
if (i == 8) {
totalRecords = 0;
}
// Maximum bytes in a record of 100K
int recordLength = random.nextInt(1024*100)+1;
// For the 11th test, force a record length of 1
if (i == 10) {
recordLength = 1;
}
// The total bytes in the test file
int fileSize = (totalRecords * recordLength);
LOG.info("totalRecords=" + totalRecords + " recordLength="
+ recordLength);
// Create the job
JobConf job = new JobConf(defaultConf);
if (codec != null) {
ReflectionUtils.setConf(codec, job);
}
// Create the test file
ArrayList<String> recordList
= createFile(file, codec, recordLength, totalRecords);
assertTrue(localFs.exists(file));
//set the fixed length record length config property for the job
FixedLengthInputFormat.setRecordLength(job, recordLength);
int numSplits = 1;
// Arbitrarily set number of splits.
if (i > 0) {
if (i == (MAX_TESTS-1)) {
// Test a split size that is less than record len
numSplits = (int)(fileSize/Math.floor(recordLength/2));
} else {
if (MAX_TESTS % i == 0) {
// Let us create a split size that is forced to be
// smaller than the end file itself, (ensures 1+ splits)
numSplits = fileSize/(fileSize - random.nextInt(fileSize));
} else {
// Just pick a random split size with no upper bound
numSplits = Math.max(1, fileSize/random.nextInt(Integer.MAX_VALUE));
}
}
LOG.info("Number of splits set to: " + numSplits);
}
// Setup the input path
FileInputFormat.setInputPaths(job, workDir);
// Try splitting the file in a variety of sizes
FixedLengthInputFormat format = new FixedLengthInputFormat();
format.configure(job);
InputSplit splits[] = format.getSplits(job, numSplits);
LOG.info("Actual number of splits = " + splits.length);
// Test combined split lengths = total file size
long recordOffset = 0;
int recordNumber = 0;
for (InputSplit split : splits) {
RecordReader<LongWritable, BytesWritable> reader =
format.getRecordReader(split, job, voidReporter);
Class<?> clazz = reader.getClass();
assertEquals("RecordReader class should be FixedLengthRecordReader:",
FixedLengthRecordReader.class, clazz);
// Plow through the records in this split
while (reader.next(key, value)) {
assertEquals("Checking key", (long)(recordNumber*recordLength),
key.get());
String valueString =
new String(value.getBytes(), 0, value.getLength());
assertEquals("Checking record length:", recordLength,
value.getLength());
assertTrue("Checking for more records than expected:",
recordNumber < totalRecords);
String origRecord = recordList.get(recordNumber);
assertEquals("Checking record content:", origRecord, valueString);
recordNumber++;
}
reader.close();
}
assertEquals("Total original records should be total read records:",
recordList.size(), recordNumber);
}
}
public BytesWritable evaluate(BytesWritable geomref, IntWritable index) {
if (geomref == null || geomref.getLength() == 0 || index == null) {
LogUtils.Log_ArgumentsNull(LOG);
return null;
}
OGCGeometry ogcGeometry = GeometryUtils.geometryFromEsriShape(geomref);
if (ogcGeometry == null){
LogUtils.Log_ArgumentsNull(LOG);
return null;
}
int idx = index.get() - 1; // 1-based UI, 0-based engine
try {
GeometryUtils.OGCType ogcType = GeometryUtils.getType(geomref);
OGCGeometry ogcGeom = null;
switch(ogcType) {
case ST_POINT:
LogUtils.Log_InvalidType(LOG, GeometryUtils.OGCType.ST_MULTIPOINT, ogcType);
return null;
case ST_LINESTRING:
LogUtils.Log_InvalidType(LOG, GeometryUtils.OGCType.ST_MULTILINESTRING, ogcType);
return null;
case ST_POLYGON:
LogUtils.Log_InvalidType(LOG, GeometryUtils.OGCType.ST_MULTIPOLYGON, ogcType);
return null;
case ST_MULTIPOINT:
ogcGeom = ((OGCMultiPoint)ogcGeometry).geometryN(idx);
break;
case ST_MULTILINESTRING:
ogcGeom = ((OGCMultiLineString)ogcGeometry).geometryN(idx);
break;
case ST_MULTIPOLYGON:
ogcGeom = ((OGCMultiPolygon)ogcGeometry).geometryN(idx);
break;
}
return GeometryUtils.geometryToEsriShapeBytesWritable(ogcGeom);
} catch (Exception e) {
LogUtils.Log_InternalError(LOG, "ST_GeometryN: " + e);
return null;
}
}
private void runRandomTests(CompressionCodec codec) throws IOException {
StringBuilder fileName = new StringBuilder("testFormat.txt");
if (codec != null) {
fileName.append(".gz");
}
localFs.delete(workDir, true);
Path file = new Path(workDir, fileName.toString());
int seed = new Random().nextInt();
LOG.info("Seed = " + seed);
Random random = new Random(seed);
int MAX_TESTS = 20;
LongWritable key = new LongWritable();
BytesWritable value = new BytesWritable();
for (int i = 0; i < MAX_TESTS; i++) {
LOG.info("----------------------------------------------------------");
// Maximum total records of 999
int totalRecords = random.nextInt(999)+1;
// Test an empty file
if (i == 8) {
totalRecords = 0;
}
// Maximum bytes in a record of 100K
int recordLength = random.nextInt(1024*100)+1;
// For the 11th test, force a record length of 1
if (i == 10) {
recordLength = 1;
}
// The total bytes in the test file
int fileSize = (totalRecords * recordLength);
LOG.info("totalRecords=" + totalRecords + " recordLength="
+ recordLength);
// Create the job
JobConf job = new JobConf(defaultConf);
if (codec != null) {
ReflectionUtils.setConf(codec, job);
}
// Create the test file
ArrayList<String> recordList
= createFile(file, codec, recordLength, totalRecords);
assertTrue(localFs.exists(file));
//set the fixed length record length config property for the job
FixedLengthInputFormat.setRecordLength(job, recordLength);
int numSplits = 1;
// Arbitrarily set number of splits.
if (i > 0) {
if (i == (MAX_TESTS-1)) {
// Test a split size that is less than record len
numSplits = (int)(fileSize/Math.floor(recordLength/2));
} else {
if (MAX_TESTS % i == 0) {
// Let us create a split size that is forced to be
// smaller than the end file itself, (ensures 1+ splits)
numSplits = fileSize/(fileSize - random.nextInt(fileSize));
} else {
// Just pick a random split size with no upper bound
numSplits = Math.max(1, fileSize/random.nextInt(Integer.MAX_VALUE));
}
}
LOG.info("Number of splits set to: " + numSplits);
}
// Setup the input path
FileInputFormat.setInputPaths(job, workDir);
// Try splitting the file in a variety of sizes
FixedLengthInputFormat format = new FixedLengthInputFormat();
format.configure(job);
InputSplit splits[] = format.getSplits(job, numSplits);
LOG.info("Actual number of splits = " + splits.length);
// Test combined split lengths = total file size
long recordOffset = 0;
int recordNumber = 0;
for (InputSplit split : splits) {
RecordReader<LongWritable, BytesWritable> reader =
format.getRecordReader(split, job, voidReporter);
Class<?> clazz = reader.getClass();
assertEquals("RecordReader class should be FixedLengthRecordReader:",
FixedLengthRecordReader.class, clazz);
// Plow through the records in this split
while (reader.next(key, value)) {
assertEquals("Checking key", (long)(recordNumber*recordLength),
key.get());
String valueString =
new String(value.getBytes(), 0, value.getLength());
assertEquals("Checking record length:", recordLength,
value.getLength());
assertTrue("Checking for more records than expected:",
recordNumber < totalRecords);
String origRecord = recordList.get(recordNumber);
assertEquals("Checking record content:", origRecord, valueString);
recordNumber++;
}
reader.close();
}
assertEquals("Total original records should be total read records:",
recordList.size(), recordNumber);
}
}
/**
* Copy the key into BytesWritable. The input BytesWritable will be
* automatically resized to the actual key size.
*
* @param key
* BytesWritable to hold the key.
* @throws IOException
*/
public int getKey(BytesWritable key) throws IOException {
key.setSize(getKeyLength());
getKey(key.getBytes());
return key.getLength();
}