下面列出了怎么用org.apache.hadoop.io.FloatWritable的API类实例代码及写法,或者点击链接到github查看源代码。
private static final WritableComparable<?> createWritable(DataType type)
{
switch (type)
{
case BOOLEAN:
return new BooleanWritable();
case BYTE:
return new ByteWritable();
case INT:
return new IntWritable();
case LONG:
return new LongWritable();
case FLOAT:
return new FloatWritable();
case DOUBLE:
return new DoubleWritable();
case STRING:
return new Text();
default:
return null;
}
}
@Override
public void backward(Iterable<Synapse<FloatWritable, FloatWritable>> messages)
throws IOException {
if (!this.isDropped()) {
float delta = 0;
for (Synapse<FloatWritable, FloatWritable> m : messages) {
// Calculates error gradient for each neuron
delta += (m.getDelta() * m.getWeight());
// Weight corrections
float weight = -this.getLearningRate() * m.getDelta()
* this.getOutput() + this.getMomentumWeight() * m.getPrevWeight();
this.push(weight);
}
this.backpropagate(delta * squashingFunction.applyDerivative(getOutput()));
} else {
this.backpropagate(0);
}
}
@Test
public void completeModeDoubleValuesExplicitParameters() throws Exception {
ObjectInspector[] inspectors = new ObjectInspector[] { doubleInspector, intConstantInspector, floatConstantInspector, longConstantInspector };
GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false, false);
try (GenericUDAFEvaluator eval = new DataToSketchUDAF().getEvaluator(info)) {
ObjectInspector resultInspector = eval.init(Mode.COMPLETE, inspectors);
checkFinalResultInspector(resultInspector);
final long seed = 2;
UnionState state = (UnionState) eval.getNewAggregationBuffer();
eval.iterate(state, new Object[] {new DoubleWritable(1), new IntWritable(16), new FloatWritable(0.99f), new LongWritable(seed)});
eval.iterate(state, new Object[] {new DoubleWritable(2), new IntWritable(16), new FloatWritable(0.99f), new LongWritable(seed)});
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
Sketch resultSketch = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) result), seed);
// because of sampling probability < 1
Assert.assertTrue(resultSketch.isEstimationMode());
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.05);
}
}
private Writable[] makeRandomWritables() {
Random r = new Random();
Writable[] writs = {
new BooleanWritable(r.nextBoolean()),
new FloatWritable(r.nextFloat()),
new FloatWritable(r.nextFloat()),
new IntWritable(r.nextInt()),
new LongWritable(r.nextLong()),
new BytesWritable("dingo".getBytes()),
new LongWritable(r.nextLong()),
new IntWritable(r.nextInt()),
new BytesWritable("yak".getBytes()),
new IntWritable(r.nextInt())
};
return writs;
}
public void testIterable() throws Exception {
Random r = new Random();
Writable[] writs = {
new BooleanWritable(r.nextBoolean()),
new FloatWritable(r.nextFloat()),
new FloatWritable(r.nextFloat()),
new IntWritable(r.nextInt()),
new LongWritable(r.nextLong()),
new BytesWritable("dingo".getBytes()),
new LongWritable(r.nextLong()),
new IntWritable(r.nextInt()),
new BytesWritable("yak".getBytes()),
new IntWritable(r.nextInt())
};
TupleWritable t = new TupleWritable(writs);
for (int i = 0; i < 6; ++i) {
t.setWritten(i);
}
verifIter(writs, t, 0);
}
public void testWritable() throws Exception {
Random r = new Random();
Writable[] writs = {
new BooleanWritable(r.nextBoolean()),
new FloatWritable(r.nextFloat()),
new FloatWritable(r.nextFloat()),
new IntWritable(r.nextInt()),
new LongWritable(r.nextLong()),
new BytesWritable("dingo".getBytes()),
new LongWritable(r.nextLong()),
new IntWritable(r.nextInt()),
new BytesWritable("yak".getBytes()),
new IntWritable(r.nextInt())
};
TupleWritable sTuple = makeTuple(writs);
ByteArrayOutputStream out = new ByteArrayOutputStream();
sTuple.write(new DataOutputStream(out));
ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
TupleWritable dTuple = new TupleWritable();
dTuple.readFields(new DataInputStream(in));
assertTrue("Failed to write/read tuple", sTuple.equals(dTuple));
}
private Writable[] makeRandomWritables() {
Random r = new Random();
Writable[] writs = {
new BooleanWritable(r.nextBoolean()),
new FloatWritable(r.nextFloat()),
new FloatWritable(r.nextFloat()),
new IntWritable(r.nextInt()),
new LongWritable(r.nextLong()),
new BytesWritable("dingo".getBytes()),
new LongWritable(r.nextLong()),
new IntWritable(r.nextInt()),
new BytesWritable("yak".getBytes()),
new IntWritable(r.nextInt())
};
return writs;
}
private static Writable createPrimitive(Object obj, PrimitiveObjectInspector inspector)
throws SerDeException {
if (obj == null) {
return null;
}
switch (inspector.getPrimitiveCategory()) {
case DOUBLE:
return new DoubleWritable(((DoubleObjectInspector) inspector).get(obj));
case FLOAT:
return new FloatWritable(((FloatObjectInspector) inspector).get(obj));
case INT:
return new IntWritable(((IntObjectInspector) inspector).get(obj));
case LONG:
return new LongWritable(((LongObjectInspector) inspector).get(obj));
case STRING:
return new Text(((StringObjectInspector) inspector).getPrimitiveJavaObject(obj));
case DATE:
return ((DateObjectInspector) inspector).getPrimitiveWritableObject(obj);
case TIMESTAMP:
return ((TimestampObjectInspector) inspector).getPrimitiveWritableObject(obj);
default:
throw new SerDeException("Can't serialize primitive : " + inspector.getPrimitiveCategory());
}
}
public void testWritable() throws Exception {
Random r = new Random();
Writable[] writs = {
new BooleanWritable(r.nextBoolean()),
new FloatWritable(r.nextFloat()),
new FloatWritable(r.nextFloat()),
new IntWritable(r.nextInt()),
new LongWritable(r.nextLong()),
new BytesWritable("dingo".getBytes()),
new LongWritable(r.nextLong()),
new IntWritable(r.nextInt()),
new BytesWritable("yak".getBytes()),
new IntWritable(r.nextInt())
};
TupleWritable sTuple = makeTuple(writs);
ByteArrayOutputStream out = new ByteArrayOutputStream();
sTuple.write(new DataOutputStream(out));
ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
TupleWritable dTuple = new TupleWritable();
dTuple.readFields(new DataInputStream(in));
assertTrue("Failed to write/read tuple", sTuple.equals(dTuple));
}
@Test
public void completeModeGivenK() throws Exception {
ObjectInspector[] inspectors = new ObjectInspector[] { floatInspector, intInspector };
GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false, false);
try (GenericUDAFEvaluator eval = new DataToSketchUDAF().getEvaluator(info)) {
ObjectInspector resultInspector = eval.init(Mode.COMPLETE, inspectors);
checkResultInspector(resultInspector);
SketchState state = (SketchState) eval.getNewAggregationBuffer();
eval.iterate(state, new Object[] { new FloatWritable(1), new IntWritable(400) });
eval.iterate(state, new Object[] { new FloatWritable(2), new IntWritable(400) });
BytesWritable bytes = (BytesWritable) eval.terminate(state);
KllFloatsSketch resultSketch = KllFloatsSketch.heapify(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getNormalizedRankError(false), KllFloatsSketch.getNormalizedRankError(400, false));
Assert.assertEquals(resultSketch.getNumRetained(), 2);
Assert.assertEquals(resultSketch.getMinValue(), 1f);
Assert.assertEquals(resultSketch.getMaxValue(), 2f);
}
}
@Override
public Vertex<LongWritable, DoubleWritable, FloatWritable>
getCurrentVertex() throws IOException {
Vertex<LongWritable, DoubleWritable, FloatWritable> vertex =
getConf().createVertex();
LongWritable vertexId = new LongWritable(
(inputSplit.getSplitIndex() * totalRecords) + recordsRead);
DoubleWritable vertexValue = new DoubleWritable(vertexId.get() * 10d);
long targetVertexId =
(vertexId.get() + 1) %
(inputSplit.getNumSplits() * totalRecords);
float edgeValue = vertexId.get() * 100f;
List<Edge<LongWritable, FloatWritable>> edges = Lists.newLinkedList();
edges.add(EdgeFactory.create(new LongWritable(targetVertexId),
new FloatWritable(edgeValue)));
vertex.initialize(vertexId, vertexValue, edges);
++recordsRead;
if (LOG.isInfoEnabled()) {
LOG.info("next: Return vertexId=" + vertex.getId().get() +
", vertexValue=" + vertex.getValue() +
", targetVertexId=" + targetVertexId + ", edgeValue=" + edgeValue);
}
return vertex;
}
@Test
public void completeModeDoubleKeysExplicitParams() throws Exception {
ObjectInspector[] inspectors = new ObjectInspector[] { doubleInspector, doubleInspector, doubleInspector, intInspector, floatInspector };
GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false, false);
try (GenericUDAFEvaluator eval = new DataToArrayOfDoublesSketchUDAF().getEvaluator(info)) {
ObjectInspector resultInspector = eval.init(Mode.COMPLETE, inspectors);
checkFinalResultInspector(resultInspector);
ArrayOfDoublesState state = (ArrayOfDoublesState) eval.getNewAggregationBuffer();
eval.iterate(state, new Object[] {new DoubleWritable(1), new DoubleWritable(1), new DoubleWritable(2), new IntWritable(32), new FloatWritable(0.99f)});
eval.iterate(state, new Object[] {new DoubleWritable(2), new DoubleWritable(1), new DoubleWritable(2), new IntWritable(32), new FloatWritable(0.99f)});
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
// because of sampling probability < 1
Assert.assertTrue(resultSketch.isEstimationMode());
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.05);
eval.reset(state);
result = eval.terminate(state);
Assert.assertNull(result);
}
}
@Test
public void partial1ModeStringKeysExplicitParams() throws Exception {
ObjectInspector[] inspectors = new ObjectInspector[] { stringInspector, doubleInspector, intInspector, floatInspector };
GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false, false);
try (GenericUDAFEvaluator eval = new DataToDoubleSummarySketchUDAF().getEvaluator(info)) {
ObjectInspector resultInspector = eval.init(Mode.PARTIAL1, inspectors);
checkIntermediateResultInspector(resultInspector);
@SuppressWarnings("unchecked")
State<DoubleSummary> state = (State<DoubleSummary>) eval.getNewAggregationBuffer();
eval.iterate(state, new Object[] {new Text("a"), new DoubleWritable(1), new IntWritable(32), new FloatWritable(0.99f)});
eval.iterate(state, new Object[] {new Text("b"), new DoubleWritable(1), new IntWritable(32), new FloatWritable(0.99f)});
Object result = eval.terminatePartial(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof List);
List<?> r = (List<?>) result;
Assert.assertEquals(r.size(), 2);
Assert.assertEquals(((IntWritable) r.get(0)).get(), 32);
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(1)), new DoubleSummaryDeserializer());
// because of sampling probability < 1
Assert.assertTrue(resultSketch.isEstimationMode());
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.05);
}
}
/**
* Set the progress of the current task.
* *Note: Works only when using a Virtual Input Format
*
* @param value value of the progress must lie within [0.0, 1.0]
*/
public static void setProgress(float value)
{
if (PhaseContext.isIntialized())
{
final MapContext mapContext = PhaseContext.getMapContext();
try
{
final FloatWritable progress = (FloatWritable) mapContext.getCurrentKey();
progress.set(value);
mapContext.nextKeyValue();
}
catch (Exception e)
{
System.err.println("Unable to report progress in Load Cyclic. Exception: " + e);
e.printStackTrace();
}
}
}
public void testWritable() throws Exception {
Random r = new Random();
Writable[] writs = {
new BooleanWritable(r.nextBoolean()),
new FloatWritable(r.nextFloat()),
new FloatWritable(r.nextFloat()),
new IntWritable(r.nextInt()),
new LongWritable(r.nextLong()),
new BytesWritable("dingo".getBytes()),
new LongWritable(r.nextLong()),
new IntWritable(r.nextInt()),
new BytesWritable("yak".getBytes()),
new IntWritable(r.nextInt())
};
TupleWritable sTuple = makeTuple(writs);
ByteArrayOutputStream out = new ByteArrayOutputStream();
sTuple.write(new DataOutputStream(out));
ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
TupleWritable dTuple = new TupleWritable();
dTuple.readFields(new DataInputStream(in));
assertTrue("Failed to write/read tuple", sTuple.equals(dTuple));
}
public void testIterable() throws Exception {
Random r = new Random();
Writable[] writs = {
new BooleanWritable(r.nextBoolean()),
new FloatWritable(r.nextFloat()),
new FloatWritable(r.nextFloat()),
new IntWritable(r.nextInt()),
new LongWritable(r.nextLong()),
new BytesWritable("dingo".getBytes()),
new LongWritable(r.nextLong()),
new IntWritable(r.nextInt()),
new BytesWritable("yak".getBytes()),
new IntWritable(r.nextInt())
};
TupleWritable t = new TupleWritable(writs);
for (int i = 0; i < 6; ++i) {
t.setWritten(i);
}
verifIter(writs, t, 0);
}
/**
* Flips and collects the url and numeric sort value.
*/
public void reduce(FloatWritable key, Iterator<Text> values,
OutputCollector<Text, FloatWritable> output, Reporter reporter)
throws IOException {
// take the negative of the negative to get original value, sometimes 0
// value are a little weird
float val = key.get();
FloatWritable number = new FloatWritable(val == 0 ? 0 : -val);
long numCollected = 0;
// collect all values, this time with the url as key
while (values.hasNext() && (numCollected < topn)) {
Text url = WritableUtils.clone(values.next(), conf);
output.collect(url, number);
numCollected++;
}
}
@Override
public Object terminatePartial(@SuppressWarnings("deprecation") AggregationBuffer agg)
throws HiveException {
PLSAPredictAggregationBuffer myAggr = (PLSAPredictAggregationBuffer) agg;
if (myAggr.wcList.size() == 0) {
return null;
}
Object[] partialResult = new Object[5];
partialResult[0] = myAggr.wcList;
partialResult[1] = myAggr.probMap;
partialResult[2] = new IntWritable(myAggr.topics);
partialResult[3] = new FloatWritable(myAggr.alpha);
partialResult[4] = new DoubleWritable(myAggr.delta);
return partialResult;
}
@Override
public void backward(
Iterable<Synapse<FloatWritable, FloatWritable>> messages)
throws IOException {
float delta = 0;
if (!this.isDropped()) {
for (Synapse<FloatWritable, FloatWritable> m : messages) {
// Calculates error gradient for each neuron
delta += (m.getDelta() * m.getWeight());
// Weight corrections
float weight = -this.getLearningRate() * m.getDelta()
* this.getOutput() + this.getMomentumWeight() * m.getPrevWeight();
this.push(weight);
}
}
this.backpropagate(delta * squashingFunction.applyDerivative(getOutput()));
}
@Override
public void forward(Iterable<Synapse<FloatWritable, FloatWritable>> messages)
throws IOException {
m2 = (isTraining()) ? MathUtils.getBinomial(1, 0.5) : 0.5f;
if (m2 > 0) {
float sum = 0;
for (Synapse<FloatWritable, FloatWritable> m : messages) {
sum += m.getInput() * m.getWeight();
}
this.setDrop(false);
this.feedforward(squashingFunction.apply(sum) * m2);
} else {
this.setDrop(true);
this.feedforward(0);
}
}
/**
* Outputs the host or domain as key for this record and numInlinks, numOutlinks
* or score as the value.
*/
public void map(Text key, Node node,
OutputCollector<Text, FloatWritable> output, Reporter reporter)
throws IOException {
float number = 0;
if (inlinks) {
number = node.getNumInlinks();
}
else if (outlinks) {
number = node.getNumOutlinks();
}
else {
number = node.getInlinkScore();
}
if (host) {
key.set(URLUtil.getHost(key.toString()));
} else {
key.set(URLUtil.getDomainName(key.toString()));
}
output.collect(key, new FloatWritable(number));
}
public FloatWritable evaluate(float value, float mean, float stddev) {
if (stddev == 0.f) {
return new FloatWritable(0.f);
}
float v = (value - mean) / stddev;
return new FloatWritable(v);
}
@Override
public Synapse<FloatWritable, FloatWritable> next() {
prevNeuronID++;
i.set(layer[prevNeuronID].getOutput());
w.set(weightMat.get(currNeuronID, prevNeuronID));
msg.set(prevNeuronID, i, w);
return new Synapse<FloatWritable, FloatWritable>(prevNeuronID, i, w);
}
@Override
public void forward(Iterable<Synapse<FloatWritable, FloatWritable>> messages)
throws IOException {
float sum = 0;
for (Synapse<FloatWritable, FloatWritable> m : messages) {
sum += m.getInput() * m.getWeight();
}
this.feedforward(squashingFunction.apply(sum));
}
@Override
public void forward(Iterable<Synapse<FloatWritable, FloatWritable>> messages)
throws IOException {
float sum = 0;
for (Synapse<FloatWritable, FloatWritable> m : messages) {
sum += m.getInput() * m.getWeight();
}
this.feedforward(squashingFunction.apply(sum));
}
public DoubleWritable evaluate(Map<IntWritable, FloatWritable> map, List<IntWritable> keys) {
double sum = 0d;
for (IntWritable k : keys) {
FloatWritable v = map.get(k);
if (v != null) {
sum += (double) v.get();
}
}
return val(sum);
}
protected void forwardModel() throws HiveException {
final IntWritable topicIdx = new IntWritable();
final Text word = new Text();
final FloatWritable score = new FloatWritable();
final Object[] forwardObjs = new Object[3];
forwardObjs[0] = topicIdx;
forwardObjs[1] = word;
forwardObjs[2] = score;
for (int k = 0; k < topics; k++) {
topicIdx.set(k);
final SortedMap<Float, List<String>> topicWords = model.getTopicWords(k);
if (topicWords == null) {
continue;
}
for (Map.Entry<Float, List<String>> e : topicWords.entrySet()) {
score.set(e.getKey().floatValue());
for (String v : e.getValue()) {
word.set(v);
forward(forwardObjs);
}
}
}
logger.info("Forwarded topic words each of " + topics + " topics");
}
protected byte inferPigDataType(Type t) {
if (t == BytesWritable.class) return DataType.BYTEARRAY;
else if (t == Text.class) return DataType.CHARARRAY;
else if (t == IntWritable.class) return DataType.INTEGER;
else if (t == LongWritable.class) return DataType.LONG;
else if (t == FloatWritable.class) return DataType.FLOAT;
else if (t == DoubleWritable.class) return DataType.DOUBLE;
else if (t == BooleanWritable.class) return DataType.BOOLEAN;
else if (t == ByteWritable.class) return DataType.BYTE;
else if (t == DateTimeWritable.class) return DataType.DATETIME;
// not doing maps or other complex types for now
else return DataType.ERROR;
}
public void write(Writable w) throws IOException {
if (w instanceof TypedBytesWritable) {
writeTypedBytes((TypedBytesWritable) w);
} else if (w instanceof BytesWritable) {
writeBytes((BytesWritable) w);
} else if (w instanceof ByteWritable) {
writeByte((ByteWritable) w);
} else if (w instanceof BooleanWritable) {
writeBoolean((BooleanWritable) w);
} else if (w instanceof IntWritable) {
writeInt((IntWritable) w);
} else if (w instanceof VIntWritable) {
writeVInt((VIntWritable) w);
} else if (w instanceof LongWritable) {
writeLong((LongWritable) w);
} else if (w instanceof VLongWritable) {
writeVLong((VLongWritable) w);
} else if (w instanceof FloatWritable) {
writeFloat((FloatWritable) w);
} else if (w instanceof DoubleWritable) {
writeDouble((DoubleWritable) w);
} else if (w instanceof Text) {
writeText((Text) w);
} else if (w instanceof ArrayWritable) {
writeArray((ArrayWritable) w);
} else if (w instanceof MapWritable) {
writeMap((MapWritable) w);
} else if (w instanceof SortedMapWritable) {
writeSortedMap((SortedMapWritable) w);
} else if (w instanceof Record) {
writeRecord((Record) w);
} else {
writeWritable(w); // last resort
}
}
public FloatWritable evaluate(List<Float> Pu, List<Float> Qi, double Bi) throws HiveException {
if (Pu == null && Qi == null) {
return new FloatWritable(0.f);
}
if (Pu == null) {
return new FloatWritable((float) Bi);
} else if (Qi == null) {
return new FloatWritable(0.f);
}
final int PuSize = Pu.size();
final int QiSize = Qi.size();
// workaround for TD
if (PuSize == 0) {
if (QiSize == 0) {
return new FloatWritable(0.f);
} else {
return new FloatWritable((float) Bi);
}
} else if (QiSize == 0) {
return new FloatWritable(0.f);
}
if (QiSize != PuSize) {
throw new HiveException("|Pu| " + PuSize + " was not equal to |Qi| " + QiSize);
}
float ret = (float) Bi;
for (int k = 0; k < PuSize; k++) {
ret += Pu.get(k) * Qi.get(k);
}
return new FloatWritable(ret);
}