下面列出了org.apache.hadoop.hbase.filter.PageFilter#org.apache.hadoop.hbase.filter.KeyOnlyFilter 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
public static boolean isReallyEmptyRegion(HConnection connection,
String tableName, HRegionInfo regionInfo) throws IOException {
boolean emptyRegion = false;
// verify really empty region by scanning records
try (HTableInterface table = connection.getTable(tableName)) {
Scan scan = new Scan(regionInfo.getStartKey(), regionInfo.getEndKey());
FilterList filterList = new FilterList();
filterList.addFilter(new KeyOnlyFilter());
filterList.addFilter(new FirstKeyOnlyFilter());
scan.setFilter(filterList);
scan.setCacheBlocks(false);
scan.setSmall(true);
scan.setCaching(1);
try (ResultScanner scanner = table.getScanner(scan)) {
if (scanner.next() == null) emptyRegion = true;
}
}
return emptyRegion;
}
@Override
public void run() {
try (HTableInterface table = connection.getTable(tableName.getBytes())) {
// Do not use Get not to increase read request count metric.
// Use Scan.
Scan scan = new Scan("".getBytes(), "".getBytes());
FilterList filterList = new FilterList();
filterList.addFilter(new KeyOnlyFilter());
filterList.addFilter(new FirstKeyOnlyFilter());
scan.setFilter(filterList);
//noinspection EmptyTryBlock
try(ResultScanner ignored = table.getScanner(scan)) {
}
return;
} catch (IOException ignore) {
}
clean(tableName);
}
public static boolean isReallyEmptyRegion(HConnection connection,
String tableName, HRegionInfo regionInfo) throws IOException {
boolean emptyRegion = false;
// verify really empty region by scanning records
try (HTableInterface table = connection.getTable(tableName)) {
Scan scan = new Scan(regionInfo.getStartKey(), regionInfo.getEndKey());
FilterList filterList = new FilterList();
filterList.addFilter(new KeyOnlyFilter());
filterList.addFilter(new FirstKeyOnlyFilter());
scan.setFilter(filterList);
scan.setCacheBlocks(false);
scan.setSmall(true);
scan.setCaching(1);
try (ResultScanner scanner = table.getScanner(scan)) {
if (scanner.next() == null) emptyRegion = true;
}
}
return emptyRegion;
}
@Override
public void run() {
try (HTableInterface table = connection.getTable(tableName.getBytes())) {
// Do not use Get not to increase read request count metric.
// Use Scan.
Scan scan = new Scan("".getBytes(), "".getBytes());
FilterList filterList = new FilterList();
filterList.addFilter(new KeyOnlyFilter());
filterList.addFilter(new FirstKeyOnlyFilter());
scan.setFilter(filterList);
//noinspection EmptyTryBlock
try(ResultScanner ignored = table.getScanner(scan)) {
}
return;
} catch (IOException ignore) {
}
clean(tableName);
}
public static boolean isReallyEmptyRegion(HConnection connection,
String tableName, HRegionInfo regionInfo) throws IOException {
boolean emptyRegion = false;
// verify really empty region by scanning records
try (HTableInterface table = connection.getTable(tableName)) {
Scan scan = new Scan(regionInfo.getStartKey(), regionInfo.getEndKey());
FilterList filterList = new FilterList();
filterList.addFilter(new KeyOnlyFilter());
filterList.addFilter(new FirstKeyOnlyFilter());
scan.setFilter(filterList);
scan.setCacheBlocks(false);
scan.setSmall(true);
scan.setCaching(1);
try (ResultScanner scanner = table.getScanner(scan)) {
if (scanner.next() == null) emptyRegion = true;
}
}
return emptyRegion;
}
@Override
public void run() {
try (HTableInterface table = connection.getTable(tableName.getBytes())) {
// Do not use Get not to increase read request count metric.
// Use Scan.
Scan scan = new Scan("".getBytes(), "".getBytes());
FilterList filterList = new FilterList();
filterList.addFilter(new KeyOnlyFilter());
filterList.addFilter(new FirstKeyOnlyFilter());
scan.setFilter(filterList);
//noinspection EmptyTryBlock
try(ResultScanner ignored = table.getScanner(scan)) {
}
return;
} catch (IOException ignore) {
}
clean(tableName);
}
public static boolean isReallyEmptyRegion(HConnection connection,
String tableName, HRegionInfo regionInfo) throws IOException {
boolean emptyRegion = false;
// verify really empty region by scanning records
try (HTableInterface table = connection.getTable(tableName)) {
Scan scan = new Scan(regionInfo.getStartKey(), regionInfo.getEndKey());
FilterList filterList = new FilterList();
filterList.addFilter(new KeyOnlyFilter());
filterList.addFilter(new FirstKeyOnlyFilter());
scan.setFilter(filterList);
scan.setCacheBlocks(false);
scan.setSmall(true);
scan.setCaching(1);
try (ResultScanner scanner = table.getScanner(scan)) {
if (scanner.next() == null) emptyRegion = true;
}
}
return emptyRegion;
}
@Override
public void run() {
try (HTableInterface table = connection.getTable(tableName.getBytes())) {
// Do not use Get not to increase read request count metric.
// Use Scan.
Scan scan = new Scan("".getBytes(), "".getBytes());
FilterList filterList = new FilterList();
filterList.addFilter(new KeyOnlyFilter());
filterList.addFilter(new FirstKeyOnlyFilter());
scan.setFilter(filterList);
//noinspection EmptyTryBlock
try(ResultScanner ignored = table.getScanner(scan)) {
}
return;
} catch (IOException ignore) {
}
clean(tableName);
}
@Override
protected boolean doAction() throws Exception {
ResultScanner rs = null;
try {
Scan s = new Scan();
s.setBatch(2);
s.addFamily(FAMILY);
s.setFilter(new KeyOnlyFilter());
s.readVersions(1);
rs = table.getScanner(s);
Result result = rs.next();
return result != null && result.size() > 0;
} finally {
if (rs != null) {
rs.close();
}
}
}
private Result getByScan(String path, byte[] family, byte[] column) throws IOException {
byte[] startRow = Bytes.toBytes(path);
byte[] endRow = plusZero(startRow);
Scan scan = new Scan(startRow, endRow);
if (family == null || column == null) {
scan.setFilter(new KeyOnlyFilter());
} else {
scan.addColumn(family, column);
}
HTableInterface table = getConnection().getTable(getAllInOneTableName());
try {
ResultScanner scanner = table.getScanner(scan);
Result result = null;
for (Result r : scanner) {
result = r;
}
return result == null || result.isEmpty() ? null : result;
} finally {
IOUtils.closeQuietly(table);
}
}
private static void prepareData(Connection conn) throws IOException {
Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
// check how many rows existing
int nRows = 0;
Scan scan = new Scan();
scan.setFilter(new KeyOnlyFilter());
ResultScanner scanner = table.getScanner(scan);
for (Result r : scanner) {
r.getRow(); // nothing to do
nRows++;
}
if (nRows > 0) {
logger.info("{} existing rows", nRows);
if (nRows != N_ROWS)
throw new IOException("Expect " + N_ROWS + " rows but it is not");
return;
}
// insert rows into empty table
logger.info("Writing {} rows to {}", N_ROWS, TEST_TABLE);
long nBytes = 0;
for (int i = 0; i < N_ROWS; i++) {
byte[] rowkey = Bytes.toBytes(i);
Put put = new Put(rowkey);
byte[] cell = randomBytes();
put.addColumn(CF, QN, cell);
table.put(put);
nBytes += cell.length;
dot(i, N_ROWS);
}
logger.info("Written {} rows, {} bytes", N_ROWS, nBytes);
} finally {
IOUtils.closeQuietly(table);
}
}
private static void prepareData(Connection conn) throws IOException {
Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
// check how many rows existing
int nRows = 0;
Scan scan = new Scan();
scan.setFilter(new KeyOnlyFilter());
ResultScanner scanner = table.getScanner(scan);
for (Result r : scanner) {
r.getRow(); // nothing to do
nRows++;
}
if (nRows > 0) {
logger.info("{} existing rows", nRows);
if (nRows != N_ROWS)
throw new IOException("Expect " + N_ROWS + " rows but it is not");
return;
}
// insert rows into empty table
logger.info("Writing {} rows to {}", N_ROWS, TEST_TABLE);
long nBytes = 0;
for (int i = 0; i < N_ROWS; i++) {
byte[] rowkey = Bytes.toBytes(i);
Put put = new Put(rowkey);
byte[] cell = randomBytes();
put.addColumn(CF, QN, cell);
table.put(put);
nBytes += cell.length;
dot(i, N_ROWS);
}
logger.info("Written {} rows, {} bytes", N_ROWS, nBytes);
} finally {
IOUtils.closeQuietly(table);
}
}
@Test
public void testKeyOnlyFilterWithReverseScan() throws Exception {
final TableName tableName = name.getTableName();
try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
byte[][] ROWS = makeN(ROW, 10);
byte[][] QUALIFIERS = {Bytes.toBytes("col0-<d2v1>-<d3v2>"),
Bytes.toBytes("col1-<d2v1>-<d3v2>"),
Bytes.toBytes("col2-<d2v1>-<d3v2>"),
Bytes.toBytes("col3-<d2v1>-<d3v2>"),
Bytes.toBytes("col4-<d2v1>-<d3v2>"),
Bytes.toBytes("col5-<d2v1>-<d3v2>"),
Bytes.toBytes("col6-<d2v1>-<d3v2>"),
Bytes.toBytes("col7-<d2v1>-<d3v2>"),
Bytes.toBytes("col8-<d2v1>-<d3v2>"),
Bytes.toBytes("col9-<d2v1>-<d3v2>")};
for (int i = 0; i < 10; i++) {
Put put = new Put(ROWS[i]);
put.addColumn(FAMILY, QUALIFIERS[i], VALUE);
ht.put(put);
}
Scan scan = new Scan();
scan.setReversed(true);
scan.addFamily(FAMILY);
Filter filter = new KeyOnlyFilter(true);
scan.setFilter(filter);
try (ResultScanner ignored = ht.getScanner(scan)) {
int count = 0;
for (Result result : ht.getScanner(scan)) {
assertEquals(1, result.size());
assertEquals(Bytes.SIZEOF_INT, result.rawCells()[0].getValueLength());
assertEquals(VALUE.length, Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0])));
count++;
}
assertEquals(10, count);
}
}
}
@Test public void testKeyOnlyFilter() throws Exception {
final TableName tableName = name.getTableName();
try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
byte[][] ROWS = makeN(ROW, 10);
byte[][] QUALIFIERS =
{ Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>") };
for (int i = 0; i < 10; i++) {
Put put = new Put(ROWS[i]);
put.setDurability(Durability.SKIP_WAL);
put.addColumn(FAMILY, QUALIFIERS[i], VALUE);
ht.put(put);
}
Scan scan = new Scan();
scan.addFamily(FAMILY);
Filter filter = new KeyOnlyFilter(true);
scan.setFilter(filter);
try (ResultScanner scanner = ht.getScanner(scan)) {
int count = 0;
for (Result result : scanner) {
assertEquals(1, result.size());
assertEquals(Bytes.SIZEOF_INT, result.rawCells()[0].getValueLength());
assertEquals(VALUE.length, Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0])));
count++;
}
assertEquals(10, count);
}
}
}
@ProcessElement
public void processElement(ProcessContext c) throws IOException {
Scan scan = new Scan()
.setRowPrefixFilter(c.element().getBytes())
.setFilter(new KeyOnlyFilter());
Table table = getConnection().getTable(TableName.valueOf(tableId));
for (Result result : table.getScanner(scan)) {
c.output(result.getRow());
}
}
@ProcessElement
public void processElement(PipelineOptions po) {
// Determine which column will be drawn based on runtime of job.
long timestampDiff = System.currentTimeMillis() - START_TIME;
long minutes = (timestampDiff / 1000) / 60;
int timeOffsetIndex = Math.toIntExact(minutes / KEY_VIZ_WINDOW_MINUTES);
ReadDataOptions options = po.as(ReadDataOptions.class);
long count = 0;
List<RowRange> ranges = getRangesForTimeIndex(timeOffsetIndex, getNumRows(options));
if (ranges.size() == 0) {
return;
}
try {
// Scan with a filter that will only return the first key from each row. This filter is used
// to more efficiently perform row count operations.
Filter rangeFilters = new MultiRowRangeFilter(ranges);
FilterList firstKeyFilterWithRanges = new FilterList(
rangeFilters,
new FirstKeyOnlyFilter(),
new KeyOnlyFilter());
Scan scan =
new Scan()
.addFamily(Bytes.toBytes(COLUMN_FAMILY))
.setFilter(firstKeyFilterWithRanges);
Table table = getConnection().getTable(TableName.valueOf(options.getBigtableTableId()));
ResultScanner imageData = table.getScanner(scan);
// Iterate over stream of rows to count them.
for (Result row : imageData) {
count++;
}
} catch (Exception e) {
System.out.println("Error reading.");
e.printStackTrace();
}
System.out.printf("got %d rows\n", count);
}
@Override
public Boolean call() throws Exception {
// total length in bytes of all read rows.
int result = 0;
// Number of rows the scan will read before being considered done.
int toRead = 100;
try (Table t = connection.getTable(tableName)) {
byte[] rk = Bytes.toBytes(ThreadLocalRandom.current().nextLong());
Scan s = new Scan().withStartRow(rk);
// This filter will keep the values from being sent accross the wire.
// This is good for counting or other scans that are checking for
// existence and don't rely on the value.
s.setFilter(new KeyOnlyFilter());
// Don't go back to the server for every single row.
// We know these rows are small. So ask for 20 at a time.
// This would be application specific.
//
// The goal is to reduce round trips but asking for too
// many rows can lead to GC problems on client and server sides.
s.setCaching(20);
// Don't use the cache. While this is a silly test program it's still good to be
// explicit that scans normally don't use the block cache.
s.setCacheBlocks(false);
// Open up the scanner and close it automatically when done.
try (ResultScanner rs = t.getScanner(s)) {
// Now go through rows.
for (Result r : rs) {
// Keep track of things size to simulate doing some real work.
result += r.getRow().length;
toRead -= 1;
// Most online applications won't be
// reading the entire table so this break
// simulates small to medium size scans,
// without needing to know an end row.
if (toRead <= 0) {
break;
}
}
}
}
return result > 0;
}
private static void prepareData(HConnection conn) throws IOException {
HTableInterface table = conn.getTable(TEST_TABLE);
try {
// check how many rows existing
int nRows = 0;
Scan scan = new Scan();
scan.setFilter(new KeyOnlyFilter());
ResultScanner scanner = table.getScanner(scan);
for (Result r : scanner) {
r.getRow(); // nothing to do
nRows++;
}
if (nRows > 0) {
System.out.println(nRows + " existing rows");
if (nRows != N_ROWS)
throw new IOException("Expect " + N_ROWS + " rows but it is not");
return;
}
// insert rows into empty table
System.out.println("Writing " + N_ROWS + " rows to " + TEST_TABLE);
long nBytes = 0;
for (int i = 0; i < N_ROWS; i++) {
byte[] rowkey = Bytes.toBytes(i);
Put put = new Put(rowkey);
byte[] cell = randomBytes();
put.add(CF, QN, cell);
table.put(put);
nBytes += cell.length;
dot(i, N_ROWS);
}
System.out.println();
System.out.println("Written " + N_ROWS + " rows, " + nBytes + " bytes");
} finally {
IOUtils.closeQuietly(table);
}
}