类org.apache.hadoop.util.bloom.Filter源码实例Demo

下面列出了怎么用org.apache.hadoop.util.bloom.Filter的API类实例代码及写法,或者点击链接到github查看源代码。

源代码1 项目: incubator-hivemall   文件: BloomContainsUDF.java
@Nullable
public Boolean evaluate(@Nullable Text bloomStr, @Nullable List<Text> keys)
        throws HiveException {
    if (bloomStr == null) {
        return null;
    }
    if (keys == null) {
        return Boolean.FALSE;
    }

    final Filter bloom = getFilter(bloomStr);

    for (Text keyStr : keys) {
        if (keyStr == null) {
            continue;
        }
        key.set(keyStr.copyBytes(), 1.0d);
        if (bloom.membershipTest(key) == false) {
            return Boolean.FALSE;
        }
    }

    return Boolean.TRUE;
}
 
源代码2 项目: incubator-hivemall   文件: BloomContainsUDF.java
@Nonnull
private Filter getFilter(@Nonnull final Text bloomStr) throws HiveException {
    final Filter bloom;
    if (prevBf != null && prevBfStr.equals(bloomStr)) {
        bloom = prevBf;
    } else {
        try {
            bloom = BloomFilterUtils.deserialize(bloomStr, new DynamicBloomFilter());
        } catch (IOException e) {
            throw new HiveException(e);
        }
        this.prevBfStr = new Text(bloomStr);
        this.prevBf = bloom;
    }
    return bloom;
}
 
源代码3 项目: incubator-hivemall   文件: BloomContainsAnyUDF.java
@Nullable
public Boolean evaluate(@Nullable Text bloomStr, @Nullable List<Text> keys)
        throws HiveException {
    if (bloomStr == null) {
        return null;
    }
    if (keys == null) {
        return Boolean.FALSE;
    }

    final Filter bloom = getFilter(bloomStr);

    for (Text keyStr : keys) {
        if (keyStr == null) {
            continue;
        }
        key.set(keyStr.copyBytes(), 1.0d);
        if (bloom.membershipTest(key)) {
            return Boolean.TRUE;
        }
    }

    return Boolean.FALSE;
}
 
源代码4 项目: incubator-hivemall   文件: BloomContainsAnyUDF.java
@Nonnull
private Filter getFilter(@Nonnull final Text bloomStr) throws HiveException {
    final Filter bloom;
    if (prevBf != null && prevBfStr.equals(bloomStr)) {
        bloom = prevBf;
    } else {
        try {
            bloom = BloomFilterUtils.deserialize(bloomStr, new DynamicBloomFilter());
        } catch (IOException e) {
            throw new HiveException(e);
        }
        this.prevBfStr = new Text(bloomStr);
        this.prevBf = bloom;
    }
    return bloom;
}
 
源代码5 项目: incubator-hivemall   文件: BloomFilterUtils.java
@Nonnull
public static byte[] serialize(@Nonnull final Filter filter) throws IOException {
    FastByteArrayOutputStream bos = new FastByteArrayOutputStream();
    Base91OutputStream base91 = new Base91OutputStream(bos);
    DataOutputStream out = new DataOutputStream(base91);
    filter.write(out);
    out.flush();
    base91.finish();
    return bos.toByteArray();
}
 
源代码6 项目: incubator-hivemall   文件: BloomFilterUtils.java
@Nonnull
public static Text serialize(@Nonnull final Filter filter, @Nonnull final Text dst)
        throws IOException {
    FastByteArrayOutputStream bos = new FastByteArrayOutputStream();
    Base91OutputStream base91 = new Base91OutputStream(bos);
    DataOutputStream out = new DataOutputStream(base91);
    filter.write(out);
    out.flush();
    base91.finish();
    dst.set(bos.getInternalArray(), 0, bos.size());
    return dst;
}
 
源代码7 项目: incubator-hivemall   文件: BloomFilterUtils.java
@Nonnull
public static <F extends Filter> F deserialize(@Nonnull final byte[] buf,
        @Nonnegative final int offset, @Nonnegative final int len, @Nonnull final F dst)
        throws IOException {
    FastByteArrayInputStream fis = new FastByteArrayInputStream(buf, offset, len);
    DataInput in = new DataInputStream(new Base91InputStream(fis));
    dst.readFields(in);
    return dst;
}
 
源代码8 项目: incubator-hivemall   文件: BloomContainsUDF.java
@Nullable
public Boolean evaluate(@Nullable Text bloomStr, @Nullable Text keyStr) throws HiveException {
    if (bloomStr == null) {
        return null;
    }
    if (keyStr == null) {
        return Boolean.FALSE;
    }

    Filter bloom = getFilter(bloomStr);
    key.set(keyStr.copyBytes(), 1.0d);
    return Boolean.valueOf(bloom.membershipTest(key));
}
 
源代码9 项目: incubator-hivemall   文件: BloomOrUDFTest.java
private static void assertEquals(@Nonnull Filter expected, @Nonnull Filter actual, long seed,
        int size) {
    final Key key = new Key();

    final Random rnd1 = new Random(seed);
    for (int i = 0; i < size; i++) {
        double d = rnd1.nextGaussian();
        String s = Double.toHexString(d);
        key.set(s.getBytes(), 1.0);
        Assert.assertEquals(expected.membershipTest(key), actual.membershipTest(key));
    }
}
 
源代码10 项目: incubator-hivemall   文件: BloomAndUDFTest.java
private static void assertNotContains(@Nonnull Filter expected, @Nonnull Filter actual,
        long seed, int size) {
    final Key key = new Key();

    final Random rnd1 = new Random(seed);
    for (int i = 0; i < size; i++) {
        double d = rnd1.nextGaussian();
        String s = Double.toHexString(d);
        key.set(s.getBytes(), 1.0);
        Assert.assertEquals(expected.membershipTest(key), actual.membershipTest(key));
    }
}
 
源代码11 项目: incubator-hivemall   文件: BloomFilterUtils.java
@Nonnull
public static <F extends Filter> F deserialize(@Nonnull final Text in, @Nonnull final F dst)
        throws IOException {
    return deserialize(in.getBytes(), 0, in.getLength(), dst);
}
 
源代码12 项目: incubator-hivemall   文件: BloomFilterUtils.java
@Nonnull
public static <F extends Filter> F deserialize(@Nonnull final byte[] buf, @Nonnull final F dst)
        throws IOException {
    return deserialize(buf, 0, buf.length, dst);
}
 
源代码13 项目: hadoop   文件: BloomMapFile.java
/**
 * Retrieve the Bloom filter used by this instance of the Reader.
 * @return a Bloom filter (see {@link Filter})
 */
public Filter getBloomFilter() {
  return bloomFilter;
}
 
源代码14 项目: big-c   文件: BloomMapFile.java
/**
 * Retrieve the Bloom filter used by this instance of the Reader.
 * @return a Bloom filter (see {@link Filter})
 */
public Filter getBloomFilter() {
  return bloomFilter;
}
 
源代码15 项目: RDFS   文件: BloomMapFile.java
/**
 * Retrieve the Bloom filter used by this instance of the Reader.
 * @return a Bloom filter (see {@link Filter})
 */
public Filter getBloomFilter() {
  return bloomFilter;
}
 
源代码16 项目: hadoop-gpu   文件: BloomMapFile.java
/**
 * Retrieve the Bloom filter used by this instance of the Reader.
 * @return a Bloom filter (see {@link Filter})
 */
public Filter getBloomFilter() {
  return bloomFilter;
}
 
 类所在包
 类方法
 同包方法