下面列出了怎么用org.apache.hadoop.hbase.filter.FilterList.Operator的API类实例代码及写法，或者点击链接到github查看源代码。
private FilterList getColumnValueFilters(Row row) {
FilterList filterList = new FilterList(Operator.MUST_PASS_ALL);
Set<String> filterColumnNames = Sets.newHashSet(row.schema().fieldNames());
for (Map.Entry<String, ColumnDef> column : columns.entrySet()) {
if (!column.getValue().cf.equals("rowkey")) {
if (filterColumnNames.contains(column.getKey())) {
byte[] value = getColumnValueAsBytes(column.getValue().name, column.getValue().type, row);
if (value != null) {
SingleColumnValueFilter columnValueFilter = new SingleColumnValueFilter(
Bytes.toBytes(column.getValue().cf),
Bytes.toBytes(column.getValue().name),
CompareFilter.CompareOp.EQUAL,
value
);
filterList.addFilter(columnValueFilter);
}
}
}
}
return filterList;
}
public static void filterLimitValueRange(String projectId, String instanceId, String tableId) {
// A filter that matches cells whose values are between the given values
ValueFilter valueGreaterFilter =
new ValueFilter(
CompareFilter.CompareOp.GREATER_OR_EQUAL,
new BinaryComparator(Bytes.toBytes("PQ2A.190405")));
ValueFilter valueLesserFilter =
new ValueFilter(
CompareFilter.CompareOp.LESS_OR_EQUAL,
new BinaryComparator(Bytes.toBytes("PQ2A.190406")));
FilterList filter = new FilterList(FilterList.Operator.MUST_PASS_ALL);
filter.addFilter(valueGreaterFilter);
filter.addFilter(valueLesserFilter);
Scan scan = new Scan().setFilter(filter);
readWithFilter(projectId, instanceId, tableId, scan);
}
/**
* When we do a "MUST_PASS_ONE" (a logical 'OR') of the two filters
* we expect to get the same result as the inclusive stop result.
* @throws Exception
*/
@Test
public void testFilterListWithInclusiveStopFilterMustPassOne() throws Exception {
byte[] r1 = Bytes.toBytes("Row1");
byte[] r11 = Bytes.toBytes("Row11");
byte[] r2 = Bytes.toBytes("Row2");
FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
flist.addFilter(new AlwaysNextColFilter());
flist.addFilter(new InclusiveStopFilter(r1));
flist.filterRowKey(KeyValueUtil.createFirstOnRow(r1));
assertEquals(ReturnCode.INCLUDE, flist.filterCell(new KeyValue(r1, r1, r1)));
assertEquals(ReturnCode.INCLUDE, flist.filterCell(new KeyValue(r11, r11, r11)));
flist.reset();
flist.filterRowKey(KeyValueUtil.createFirstOnRow(r2));
assertEquals(ReturnCode.NEXT_COL, flist.filterCell(new KeyValue(r2, r2, r2)));
}
/**
* Test serialization
* @throws Exception
*/
@Test
public void testSerialization() throws Exception {
List<Filter> filters = new ArrayList<>();
filters.add(new PageFilter(MAX_PAGES));
filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy"))));
Filter filterMPALL =
new FilterList(FilterList.Operator.MUST_PASS_ALL, filters);
// Decompose filterMPALL to bytes.
byte[] buffer = filterMPALL.toByteArray();
// Recompose filterMPALL.
FilterList newFilter = FilterList.parseFrom(buffer);
// Run tests
mpOneTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getFilterMPONE())));
mpAllTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getMPALLFilter())));
orderingTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getOrderingFilter())));
}
@Test
public void testWithMultiVersionsInSameRow() throws Exception {
FilterList filterList01 =
new FilterList(Operator.MUST_PASS_ONE, new ColumnPaginationFilter(1, 0));
KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual"),
1, Bytes.toBytes("value"));
KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual"),
2, Bytes.toBytes("value"));
KeyValue kv3 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual"),
3, Bytes.toBytes("value"));
assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList01.filterCell(kv1));
assertEquals(ReturnCode.NEXT_COL, filterList01.filterCell(kv2));
assertEquals(ReturnCode.NEXT_COL, filterList01.filterCell(kv3));
FilterList filterList11 =
new FilterList(Operator.MUST_PASS_ONE, new ColumnPaginationFilter(1, 1));
assertEquals(ReturnCode.NEXT_COL, filterList11.filterCell(kv1));
assertEquals(ReturnCode.NEXT_COL, filterList11.filterCell(kv2));
assertEquals(ReturnCode.NEXT_COL, filterList11.filterCell(kv3));
}
@Test
public void testMPONEWithSeekNextUsingHint() throws Exception {
byte[] col = Bytes.toBytes("c");
FilterList filterList =
new FilterList(Operator.MUST_PASS_ONE, new ColumnPaginationFilter(1, col));
KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
Bytes.toBytes("value"));
KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("b"), 2,
Bytes.toBytes("value"));
KeyValue kv3 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("c"), 3,
Bytes.toBytes("value"));
KeyValue kv4 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("c"), 4,
Bytes.toBytes("value"));
assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv1));
assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv2));
assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterCell(kv3));
assertEquals(ReturnCode.NEXT_COL, filterList.filterCell(kv4));
}
public static void filterComposingChain(String projectId, String instanceId, String tableId) {
// A filter that selects one cell per row AND within the column family cell_plan
Filter familyFilter =
new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("cell_plan")));
Filter columnCountGetFilter = new ColumnCountGetFilter(3);
FilterList filter = new FilterList(FilterList.Operator.MUST_PASS_ALL);
filter.addFilter(columnCountGetFilter);
filter.addFilter(familyFilter);
Scan scan = new Scan().setFilter(filter);
readWithFilter(projectId, instanceId, tableId, scan);
}
public static void filterComposingInterleave(
String projectId, String instanceId, String tableId) {
// A filter that matches cells with the value true OR with the column qualifier os_build
Filter qualifierFilter =
new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("os_build")));
Filter valueFilter =
new ValueFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("true")));
FilterList filter = new FilterList(Operator.MUST_PASS_ONE);
filter.addFilter(qualifierFilter);
filter.addFilter(valueFilter);
Scan scan = new Scan().setFilter(filter).setMaxVersions();
readWithFilter(projectId, instanceId, tableId, scan);
}
private void runScanner(Table hTable, int expectedSize, Filter filter1, Filter filter2)
throws IOException {
String cf = "f";
Scan scan = new Scan();
scan.addFamily(Bytes.toBytes(cf));
FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
scan.setFilter(filterList);
ResultScanner scanner = hTable.getScanner(scan);
List<Cell> results = new ArrayList<>();
Result result;
long timeBeforeScan = System.currentTimeMillis();
while ((result = scanner.next()) != null) {
for (Cell kv : result.listCells()) {
LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: "
+ Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
results.add(kv);
}
}
long scanTime = System.currentTimeMillis() - timeBeforeScan;
scanner.close();
LOG.info("scan time = " + scanTime + "ms");
LOG.info("found " + results.size() + " results");
assertEquals(expectedSize, results.size());
}
@Test
public void testFiltersWithOR() throws Exception {
TableName tn = TableName.valueOf(name.getMethodName());
Table table = TEST_UTIL.createTable(tn, new String[] { "cf1", "cf2" });
byte[] CF1 = Bytes.toBytes("cf1");
byte[] CF2 = Bytes.toBytes("cf2");
Put put1 = new Put(Bytes.toBytes("0"));
put1.addColumn(CF1, Bytes.toBytes("col_a"), Bytes.toBytes(0));
table.put(put1);
Put put2 = new Put(Bytes.toBytes("0"));
put2.addColumn(CF2, Bytes.toBytes("col_b"), Bytes.toBytes(0));
table.put(put2);
FamilyFilter filterCF1 =
new FamilyFilter(CompareOperator.EQUAL, new BinaryComparator(CF1));
FamilyFilter filterCF2 =
new FamilyFilter(CompareOperator.EQUAL, new BinaryComparator(CF2));
FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
filterList.addFilter(filterCF1);
filterList.addFilter(filterCF2);
Scan scan = new Scan();
scan.setFilter(filterList);
ResultScanner scanner = table.getScanner(scan);
LOG.info("Filter list: " + filterList);
for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
Assert.assertEquals(2, rr.size());
}
}
@Test
public void testAddFilter() throws Exception {
Filter filter1 = new FirstKeyOnlyFilter();
Filter filter2 = new FirstKeyOnlyFilter();
FilterList filterList = new FilterList(filter1, filter2);
filterList.addFilter(new FirstKeyOnlyFilter());
filterList = new FilterList(Arrays.asList(filter1, filter2));
filterList.addFilter(new FirstKeyOnlyFilter());
filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
filterList.addFilter(new FirstKeyOnlyFilter());
filterList = new FilterList(Operator.MUST_PASS_ALL, Arrays.asList(filter1, filter2));
filterList.addFilter(new FirstKeyOnlyFilter());
filterList.setReversed(false);
FirstKeyOnlyFilter f = new FirstKeyOnlyFilter();
f.setReversed(true);
try {
filterList.addFilter(f);
fail("The IllegalArgumentException should be thrown because the added filter is reversed");
} catch (IllegalArgumentException e) {
}
}
private Filter getFilterMPONE() {
List<Filter> filters = new ArrayList<>();
filters.add(new PageFilter(MAX_PAGES));
filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy"))));
Filter filterMPONE =
new FilterList(FilterList.Operator.MUST_PASS_ONE, filters);
return filterMPONE;
}
private Filter getMPALLFilter() {
List<Filter> filters = new ArrayList<>();
filters.add(new PageFilter(MAX_PAGES));
filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy"))));
Filter filterMPALL =
new FilterList(FilterList.Operator.MUST_PASS_ALL, filters);
return filterMPALL;
}
public Filter getOrderingFilter() {
List<Filter> filters = new ArrayList<>();
filters.add(new PrefixFilter(Bytes.toBytes("yyy")));
filters.add(new PageFilter(MAX_PAGES));
Filter filterMPONE =
new FilterList(FilterList.Operator.MUST_PASS_ONE, filters);
return filterMPONE;
}
/**
* When we do a "MUST_PASS_ONE" (a logical 'OR') of the above two filters
* we expect to get the same result as the 'prefix' only result.
* @throws Exception
*/
@Test
public void testFilterListTwoFiltersMustPassOne() throws Exception {
byte[] r1 = Bytes.toBytes("Row1");
byte[] r11 = Bytes.toBytes("Row11");
byte[] r2 = Bytes.toBytes("Row2");
FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
flist.addFilter(new PrefixFilter(r1));
flist.filterRowKey(KeyValueUtil.createFirstOnRow(r1));
assertEquals(ReturnCode.INCLUDE, flist.filterCell(new KeyValue(r1, r1, r1)));
assertEquals(ReturnCode.INCLUDE, flist.filterCell(new KeyValue(r11, r11, r11)));
flist.reset();
flist.filterRowKey(KeyValueUtil.createFirstOnRow(r2));
assertEquals(ReturnCode.SKIP, flist.filterCell(new KeyValue(r2, r2, r2)));
flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
flist.addFilter(new AlwaysNextColFilter());
flist.addFilter(new PrefixFilter(r1));
flist.filterRowKey(KeyValueUtil.createFirstOnRow(r1));
assertEquals(ReturnCode.INCLUDE, flist.filterCell(new KeyValue(r1, r1, r1)));
assertEquals(ReturnCode.INCLUDE, flist.filterCell(new KeyValue(r11, r11, r11)));
flist.reset();
flist.filterRowKey(KeyValueUtil.createFirstOnRow(r2));
assertEquals(ReturnCode.NEXT_COL, flist.filterCell(new KeyValue(r2, r2, r2)));
}
/**
* Tests the behavior of transform() in a hierarchical filter.
*
* transform() only applies after a filterCell() whose return-code includes the KeyValue.
* Lazy evaluation of AND
*/
@Test
public void testTransformMPO() throws Exception {
// Apply the following filter:
// (family=fam AND qualifier=qual1 AND KeyOnlyFilter)
// OR (family=fam AND qualifier=qual2)
final FilterList flist = new FilterList(Operator.MUST_PASS_ONE, Lists.<Filter>newArrayList(
new FilterList(Operator.MUST_PASS_ALL, Lists.<Filter>newArrayList(
new FamilyFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))),
new QualifierFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("qual1"))),
new KeyOnlyFilter())),
new FilterList(Operator.MUST_PASS_ALL, Lists.<Filter>newArrayList(
new FamilyFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))),
new QualifierFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("qual2")))))));
final KeyValue kvQual1 = new KeyValue(
Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual1"), Bytes.toBytes("value"));
final KeyValue kvQual2 = new KeyValue(
Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual2"), Bytes.toBytes("value"));
final KeyValue kvQual3 = new KeyValue(
Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual3"), Bytes.toBytes("value"));
// Value for fam:qual1 should be stripped:
assertEquals(Filter.ReturnCode.INCLUDE, flist.filterCell(kvQual1));
final KeyValue transformedQual1 = KeyValueUtil.ensureKeyValue(flist.transformCell(kvQual1));
assertEquals(0, transformedQual1.getValueLength());
// Value for fam:qual2 should not be stripped:
assertEquals(Filter.ReturnCode.INCLUDE, flist.filterCell(kvQual2));
final KeyValue transformedQual2 = KeyValueUtil.ensureKeyValue(flist.transformCell(kvQual2));
assertEquals("value", Bytes.toString(transformedQual2.getValueArray(),
transformedQual2.getValueOffset(), transformedQual2.getValueLength()));
// Other keys should be skipped:
assertEquals(Filter.ReturnCode.SKIP, flist.filterCell(kvQual3));
}
@Test
public void testTheMaximalRule() throws IOException {
KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
Bytes.toBytes("value"));
MockFilter filter1 = new MockFilter(ReturnCode.INCLUDE);
MockFilter filter2 = new MockFilter(ReturnCode.INCLUDE_AND_NEXT_COL);
MockFilter filter3 = new MockFilter(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW);
MockFilter filter4 = new MockFilter(ReturnCode.NEXT_COL);
MockFilter filter5 = new MockFilter(ReturnCode.SKIP);
MockFilter filter6 = new MockFilter(ReturnCode.SEEK_NEXT_USING_HINT);
MockFilter filter7 = new MockFilter(ReturnCode.NEXT_ROW);
FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ALL, filter2, filter3);
assertEquals(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ALL, filter4, filter5, filter6);
assertEquals(ReturnCode.NEXT_COL, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ALL, filter4, filter6);
assertEquals(ReturnCode.NEXT_COL, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ALL, filter3, filter1);
assertEquals(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ALL, filter3, filter2, filter1, filter5);
assertEquals(ReturnCode.NEXT_ROW, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ALL, filter2,
new FilterList(Operator.MUST_PASS_ALL, filter3, filter4));
assertEquals(ReturnCode.NEXT_ROW, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ALL, filter3, filter7);
assertEquals(ReturnCode.NEXT_ROW, filterList.filterCell(kv1));
}
@Test
public void testTheMinimalRule() throws IOException {
KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
Bytes.toBytes("value"));
MockFilter filter1 = new MockFilter(ReturnCode.INCLUDE);
MockFilter filter2 = new MockFilter(ReturnCode.INCLUDE_AND_NEXT_COL);
MockFilter filter3 = new MockFilter(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW);
MockFilter filter4 = new MockFilter(ReturnCode.NEXT_COL);
MockFilter filter5 = new MockFilter(ReturnCode.SKIP);
MockFilter filter6 = new MockFilter(ReturnCode.SEEK_NEXT_USING_HINT);
FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, filter1, filter2);
assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ONE, filter2, filter3);
assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ONE, filter4, filter5, filter6);
assertEquals(ReturnCode.SKIP, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ONE, filter4, filter6);
assertEquals(ReturnCode.SKIP, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ONE, filter3, filter1);
assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ONE, filter3, filter2, filter1, filter5);
assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ONE, filter2,
new FilterList(Operator.MUST_PASS_ONE, filter3, filter4));
assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ONE, filter2,
new FilterList(Operator.MUST_PASS_ONE, filter3, filter4));
assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ONE, filter6, filter6);
assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv1));
}
@Test
public void testReversedFilterListWithMockSeekHintFilter() throws IOException {
KeyValue kv1 = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
Bytes.toBytes("value"));
KeyValue kv2 = new KeyValue(Bytes.toBytes("row2"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
Bytes.toBytes("value"));
KeyValue kv3 = new KeyValue(Bytes.toBytes("row3"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
Bytes.toBytes("value"));
Filter filter1 = new MockSeekHintFilter(kv1);
filter1.setReversed(true);
Filter filter2 = new MockSeekHintFilter(kv2);
filter2.setReversed(true);
Filter filter3 = new MockSeekHintFilter(kv3);
filter3.setReversed(true);
FilterList filterList = new FilterList(Operator.MUST_PASS_ONE);
filterList.setReversed(true);
filterList.addFilter(filter1);
filterList.addFilter(filter2);
filterList.addFilter(filter3);
Assert.assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv1));
Assert.assertEquals(kv3, filterList.getNextCellHint(kv1));
filterList = new FilterList(Operator.MUST_PASS_ALL);
filterList.setReversed(true);
filterList.addFilter(filter1);
filterList.addFilter(filter2);
filterList.addFilter(filter3);
Assert.assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv1));
Assert.assertEquals(kv1, filterList.getNextCellHint(kv1));
}
@Test
public void testReversedFilterListWithOR() throws IOException {
byte[] r22 = Bytes.toBytes("Row22");
byte[] r2 = Bytes.toBytes("Row2");
byte[] r1 = Bytes.toBytes("Row1");
FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
filterList.setReversed(true);
PrefixFilter prefixFilter = new PrefixFilter(r2);
prefixFilter.setReversed(true);
filterList.addFilter(prefixFilter);
filterList.filterRowKey(KeyValueUtil.createFirstOnRow(r22));
assertEquals(ReturnCode.INCLUDE, filterList.filterCell(new KeyValue(r22, r22, r22)));
assertEquals(ReturnCode.INCLUDE, filterList.filterCell(new KeyValue(r2, r2, r2)));
filterList.reset();
filterList.filterRowKey(KeyValueUtil.createFirstOnRow(r1));
assertEquals(ReturnCode.SKIP, filterList.filterCell(new KeyValue(r1, r1, r1)));
filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
filterList.setReversed(true);
AlwaysNextColFilter alwaysNextColFilter = new AlwaysNextColFilter();
alwaysNextColFilter.setReversed(true);
prefixFilter = new PrefixFilter(r2);
prefixFilter.setReversed(true);
filterList.addFilter(alwaysNextColFilter);
filterList.addFilter(prefixFilter);
filterList.filterRowKey(KeyValueUtil.createFirstOnRow(r22));
assertEquals(ReturnCode.INCLUDE, filterList.filterCell(new KeyValue(r22, r22, r22)));
assertEquals(ReturnCode.INCLUDE, filterList.filterCell(new KeyValue(r2, r2, r2)));
filterList.reset();
filterList.filterRowKey(KeyValueUtil.createFirstOnRow(r1));
assertEquals(ReturnCode.NEXT_COL, filterList.filterCell(new KeyValue(r1, r1, r1)));
}
@Test
public void testKeyOnlyFilterTransformCell() throws IOException {
Cell c;
KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"),
1, Bytes.toBytes("value1"));
KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"),
2, Bytes.toBytes("value2"));
Filter filter1 = new SingleColumnValueFilter(Bytes.toBytes("cf"), Bytes.toBytes("column1"),
CompareOperator.EQUAL, Bytes.toBytes("value1"));
Filter filter2 = new SingleColumnValueFilter(Bytes.toBytes("cf"), Bytes.toBytes("column1"),
CompareOperator.EQUAL, Bytes.toBytes("value2"));
FilterList internalFilterList = new FilterList(Operator.MUST_PASS_ONE, filter1, filter2);
FilterList keyOnlyFilterFirst =
new FilterList(Operator.MUST_PASS_ALL, new KeyOnlyFilter(), internalFilterList);
assertEquals(ReturnCode.INCLUDE, keyOnlyFilterFirst.filterCell(kv1));
c = keyOnlyFilterFirst.transformCell(kv1);
assertEquals(0, c.getValueLength());
assertEquals(ReturnCode.INCLUDE, keyOnlyFilterFirst.filterCell(kv2));
c = keyOnlyFilterFirst.transformCell(kv2);
assertEquals(0, c.getValueLength());
internalFilterList.reset();
FilterList keyOnlyFilterLast =
new FilterList(Operator.MUST_PASS_ALL, new KeyOnlyFilter(), internalFilterList);
assertEquals(ReturnCode.INCLUDE, keyOnlyFilterLast.filterCell(kv1));
c = keyOnlyFilterLast.transformCell(kv1);
assertEquals(0, c.getValueLength());
assertEquals(ReturnCode.INCLUDE, keyOnlyFilterLast.filterCell(kv2));
c = keyOnlyFilterLast.transformCell(kv2);
assertEquals(0, c.getValueLength());
}
@Test
public void testEmptyFilterListTransformCell() throws IOException {
KeyValue kv = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"),
1, Bytes.toBytes("value"));
FilterList filterList = new FilterList(Operator.MUST_PASS_ALL);
assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv));
assertEquals(kv, filterList.transformCell(kv));
filterList = new FilterList(Operator.MUST_PASS_ONE);
assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv));
assertEquals(kv, filterList.transformCell(kv));
}
@Test
public void testRowCountFilter() throws IOException {
KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam1"), Bytes.toBytes("a"), 1,
Bytes.toBytes("value"));
KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam2"), Bytes.toBytes("a"), 2,
Bytes.toBytes("value"));
MockNextRowFilter mockNextRowFilter = new MockNextRowFilter();
FilterList filter = new FilterList(Operator.MUST_PASS_ONE, mockNextRowFilter);
filter.filterCell(kv1);
filter.filterCell(kv2);
assertEquals(2, mockNextRowFilter.getHitCount());
}
@Test
public void testTransformCell() throws IOException {
KeyValue kv =
new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"), 1,
Bytes.toBytes("value"));
// case MUST_PASS_ONE
TransformFilter filter1 = new TransformFilter(ReturnCode.INCLUDE);
TransformFilter filter2 = new TransformFilter(ReturnCode.NEXT_ROW);
TransformFilter filter3 = new TransformFilter(ReturnCode.SEEK_NEXT_USING_HINT);
FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, filter1, filter2, filter3);
Assert.assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv));
Assert.assertEquals(kv, filterList.transformCell(kv));
Assert.assertEquals(true, filter1.getTransformed());
Assert.assertEquals(false, filter2.getTransformed());
Assert.assertEquals(false, filter3.getTransformed());
// case MUST_PASS_ALL
filter1 = new TransformFilter(ReturnCode.INCLUDE);
filter2 = new TransformFilter(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW);
filter3 = new TransformFilter(ReturnCode.INCLUDE_AND_NEXT_COL);
filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2, filter3);
Assert.assertEquals(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, filterList.filterCell(kv));
Assert.assertEquals(kv, filterList.transformCell(kv));
Assert.assertEquals(true, filter1.getTransformed());
Assert.assertEquals(true, filter2.getTransformed());
Assert.assertEquals(true, filter3.getTransformed());
}
@Test
public void testForceSkipScanOnSaltedTable() throws SQLException {
Connection conn = DriverManager.getConnection(getUrl());
conn.createStatement().execute("CREATE TABLE IF NOT EXISTS user_messages (\n" +
" SENDER_ID UNSIGNED_LONG NOT NULL,\n" +
" RECIPIENT_ID UNSIGNED_LONG NOT NULL,\n" +
" SENDER_IP VARCHAR,\n" +
" IS_READ VARCHAR,\n" +
" IS_DELETED VARCHAR,\n" +
" M_TEXT VARCHAR,\n" +
" M_TIMESTAMP timestamp NOT NULL,\n" +
" ROW_ID UNSIGNED_LONG NOT NULL\n" +
" constraint rowkey primary key (SENDER_ID,RECIPIENT_ID,M_TIMESTAMP DESC,ROW_ID))\n" +
"SALT_BUCKETS=12\n");
String query = "select /*+ SKIP_SCAN */ count(*) from user_messages where is_read='N' and recipient_id=5399179882";
StatementContext context = compileStatement(query);
Scan scan = context.getScan();
Filter filter = scan.getFilter();
assertNotNull(filter);
assertTrue(filter instanceof FilterList);
FilterList filterList = (FilterList)filter;
assertEquals(FilterList.Operator.MUST_PASS_ALL, filterList.getOperator());
assertEquals(2, filterList.getFilters().size());
assertTrue(filterList.getFilters().get(0) instanceof SkipScanFilter);
assertTrue(filterList.getFilters().get(1) instanceof SingleKeyValueComparisonFilter);
ScanRanges scanRanges = context.getScanRanges();
assertNotNull(scanRanges);
assertEquals(3,scanRanges.getRanges().size());
assertEquals(1,scanRanges.getRanges().get(1).size());
assertEquals(KeyRange.EVERYTHING_RANGE,scanRanges.getRanges().get(1).get(0));
assertEquals(1,scanRanges.getRanges().get(2).size());
assertTrue(scanRanges.getRanges().get(2).get(0).isSingleKey());
assertEquals(Long.valueOf(5399179882L), PUnsignedLong.INSTANCE.toObject(scanRanges.getRanges().get(2).get(0).getLowerRange()));
}
/**
* get StringBuilder with print according to given {@link FilterList}
*
* @param result as {@link StringBuilder}
* @param filterList {@link FilterList} with HBase filters
*/
public static void getFilterListPrint(StringBuilder result, FilterList filterList) {
if (filterList == null || filterList.getFilters().size() == 0) {
return;
}
if (result == null) {
result = new StringBuilder();
}
List<Filter> list = filterList.getFilters();
Operator operator = filterList.getOperator();
String operatorString = (operator.equals(Operator.MUST_PASS_ALL)) ? " AND " : " OR ";
// open current filter list
result.append("[");
int listIndex = 0;
// go over each filter in the list
for (Filter filter : list) {
// if current filter is a FilterList, call the method again with it
if (filter instanceof FilterList) {
getFilterListPrint(result, (FilterList) filter);
} else {
// append filter to result
result.append("(" + filter + ")");
// if end of list and not FilterList, close the current list
if (listIndex == list.size() - 1) {
result.append("]");
}
}
// if still not the end of list put the operator
if (listIndex != list.size() - 1) {
result.append(operatorString);
}
listIndex++;
}
}
/**
* @see org.apache.eagle.query.parser.TokenType
*
* @return
*/
public FilterList buildFilters(){
// TODO: Optimize to select between row filter or column filter for better performance
// Use row key filter priority by default
boolean rowFilterPriority = true;
FilterList fltList = new FilterList(Operator.MUST_PASS_ONE);
for(ANDExpression andExpr : _orExpr.getANDExprList()){
FilterList list = new FilterList(Operator.MUST_PASS_ALL);
Map<String, List<String>> tagFilters = new HashMap<String, List<String>>();
List<QualifierFilterEntity> qualifierFilters = new ArrayList<QualifierFilterEntity>();
// List<QualifierFilterEntry> tagLikeQualifierFilters = new ArrayList<QualifierFilterEntry>();
// TODO refactor not to use too much if/else
for(AtomicExpression ae : andExpr.getAtomicExprList()){
// TODO temporarily ignore those fields which are not for attributes
String fieldName = ae.getKey();
if(ae.getKeyType() == TokenType.ID){
fieldName = parseEntityAttribute(fieldName);
if(fieldName == null){
LOG.warn(fieldName + " field does not have format @<FieldName>, ignored");
continue;
}
}
String value = ae.getValue();
ComparisonOperator op = ae.getOp();
TokenType keyType = ae.getKeyType();
TokenType valueType = ae.getValueType();
QualifierFilterEntity entry = new QualifierFilterEntity(fieldName,value,op,keyType,valueType);
// TODO Exact match, need to add escape for those special characters here, including:
// "-", "[", "]", "/", "{", "}", "(", ")", "*", "+", "?", ".", "\\", "^", "$", "|"
if(keyType == TokenType.ID && isTag(fieldName)){
if ((ComparisonOperator.EQUAL.equals(op) || ComparisonOperator.IS.equals(op))
&& !TokenType.NULL.equals(valueType))
{
// Use RowFilter for equal TAG
if(tagFilters.get(fieldName) == null) tagFilters.put(fieldName, new ArrayList<String>());
tagFilters.get(fieldName).add(value);
} else if (rowFilterPriority && ComparisonOperator.IN.equals(op))
{
// Use RowFilter here by default
if(tagFilters.get(fieldName) == null) tagFilters.put(fieldName, new ArrayList<String>());
tagFilters.get(fieldName).addAll(EntityQualifierUtils.parseList(value));
} else if (ComparisonOperator.LIKE.equals(op)
|| ComparisonOperator.NOT_LIKE.equals(op)
|| ComparisonOperator.CONTAINS.equals(op)
|| ComparisonOperator.NOT_CONTAINS.equals(op)
|| ComparisonOperator.IN.equals(op)
|| ComparisonOperator.IS.equals(op)
|| ComparisonOperator.IS_NOT.equals(op)
|| ComparisonOperator.NOT_EQUAL.equals(op)
|| ComparisonOperator.EQUAL.equals(op)
|| ComparisonOperator.NOT_IN.equals(op))
{
qualifierFilters.add(entry);
} else
{
LOG.warn("Don't support operation: \"" + op + "\" on tag field: " + fieldName + " yet, going to ignore");
throw new IllegalArgumentException("Don't support operation: "+op+" on tag field: "+fieldName+", avaliable options: =, =!, =~, !=~, in, not in, contains, not contains");
}
}else{
qualifierFilters.add(entry);
}
}
// Build RowFilter for equal tags
list.addFilter(buildTagFilter(tagFilters));
// Build SingleColumnValueFilter
FilterList qualifierFilterList = buildQualifierFilter(qualifierFilters);
if(qualifierFilterList != null && qualifierFilterList.getFilters().size()>0){
list.addFilter(qualifierFilterList);
}else {
if(LOG.isDebugEnabled()) LOG.debug("Ignore empty qualifier filter from "+qualifierFilters.toString());
}
fltList.addFilter(list);
}
LOG.info("Query: " + _orExpr.toString() + " => Filter: " + fltList.toString());
return fltList;
}
/**
* all qualifiers' condition must be satisfied.
*
* <H1>Use RegexStringComparator for:</H1>
* IN
* LIKE
* NOT_LIKE
*
* <H1>Use SubstringComparator for:</H1>
* CONTAINS
*
* <H1>Use EntityQualifierHelper for:</H1>
* EQUALS
* NOT_EUQALS
* LESS
* LESS_OR_EQUAL
* GREATER
* GREATER_OR_EQUAL
*
* <H2>
* TODO: Compare performance of RegexStringComparator ,SubstringComparator ,EntityQualifierHelper
* </H2>
*
* @param qualifierFilters
* @return
*/
protected FilterList buildQualifierFilter(List<QualifierFilterEntity> qualifierFilters){
FilterList list = new FilterList(Operator.MUST_PASS_ALL);
// iterate all the qualifiers
for(QualifierFilterEntity entry : qualifierFilters){
// if contains expression based filter
if(entry.getKeyType() == TokenType.EXP
|| entry.getValueType() == TokenType.EXP
|| entry.getKeyType() != TokenType.ID){
if(!EagleConfigFactory.load().isCoprocessorEnabled()) {
LOG.warn("Expression in filter may not support, because custom filter and coprocessor is disabled: " + entry.toString());
}
list.addFilter(buildExpressionBasedFilter(entry));
continue;
}
// else using SingleColumnValueFilter
String qualifierName = entry.getKey();
if(!isTag(entry.getKey())){
Qualifier qualifier = _ed.getDisplayNameMap().get(entry.getKey());
qualifierName = qualifier.getQualifierName();
}
// Comparator to be used for building HBase Filter
// WritableByteArrayComparable comparator;
ByteArrayComparable comparable;
if(ComparisonOperator.IN.equals(entry.getOp())
|| ComparisonOperator.NOT_IN.equals(entry.getOp())){
Filter setFilter = buildListQualifierFilter(entry);
if(setFilter!=null){
list.addFilter(setFilter);
}
}else{
// If [=,!=,is,is not] NULL, use NullComparator else throw exception
if(TokenType.NULL.equals(entry.getValueType())){
if(ComparisonOperator.EQUAL.equals(entry.getOp())
||ComparisonOperator.NOT_EQUAL.equals(entry.getOp())
||ComparisonOperator.IS.equals(entry.getOp())
||ComparisonOperator.IS_NOT.equals(entry.getOp()))
comparable = new NullComparator();
else
throw new IllegalArgumentException("Operation: "+entry.getOp()+" with NULL is not supported yet: "+entry.toString()+", avaliable options: [=, !=, is, is not] null|NULL");
}
// If [contains, not contains],use SubstringComparator
else if (ComparisonOperator.CONTAINS.equals(entry.getOp())
|| ComparisonOperator.NOT_CONTAINS.equals(entry.getOp())) {
comparable = new SubstringComparator(entry.getValue());
}
// If [like, not like], use RegexStringComparator
else if (ComparisonOperator.LIKE.equals(entry.getOp())
|| ComparisonOperator.NOT_LIKE.equals(entry.getOp())){
// Use RegexStringComparator for LIKE / NOT_LIKE
RegexStringComparator _comparator = new RegexStringComparator(buildQualifierRegex(entry.getValue()));
_comparator.setCharset(_charset);
comparable = _comparator;
} else{
Class type = EntityQualifierUtils.getType(_ed, entry.getKey());
// if type is null (is Tag or not found) or not defined for TypedByteArrayComparator
if(!EagleConfigFactory.load().isCoprocessorEnabled() || type == null || TypedByteArrayComparator.get(type) == null){
comparable = new BinaryComparator(EntityQualifierUtils.toBytes(_ed, entry.getKey(), entry.getValue()));
}else {
comparable = new TypedByteArrayComparator(EntityQualifierUtils.toBytes(_ed, entry.getKey(), entry.getValue()),type);
}
}
SingleColumnValueFilter filter =
new SingleColumnValueFilter(_ed.getColumnFamily().getBytes(), qualifierName.getBytes(), convertToHBaseCompareOp(entry.getOp()), comparable);
filter.setFilterIfMissing(_filterIfMissing);
list.addFilter(filter);
}
}
return list;
}
/**
* Currently use BinaryComparator only
* <h2>TODO: </h2>
* Possibility to tune performance by using: OR[BinaryComparator,...] instead of RegexStringComparator?
*
*<br/> <br/>
*
* ! Check op must be IN or NOTIN in caller
*
* @param entry
* @return
*/
private Filter buildListQualifierFilter(QualifierFilterEntity entry){
List<String> valueSet = EntityQualifierUtils.parseList(entry.getValue());
Iterator<String> it = valueSet.iterator();
String fieldName = entry.getKey();
String qualifierName = fieldName;
if(!_ed.isTag(entry.getKey())){
qualifierName = _ed.getDisplayNameMap().get(entry.getKey()).getQualifierName();
}
// TODO: Try to use RegExp just work if possible
// Because single SingleColumnValueFilter is much faster than multi SingleColumnValueFilters in OR list.
// Class qualifierType = EntityQualifierHelper.getType(_ed,fieldName);
// if( qualifierType == null || qualifierType == String.class){
// boolean first = true;
// StringBuilder filterRegex = new StringBuilder();
// filterRegex.append("^(");
// while(it.hasNext()) {
// String value = it.next();
// if(value == null) {
// logger.warn("ignore empty value in set qualifier filter: "+entry.toString());
// continue;
// }
// if(!first) filterRegex.append("|");
// filterRegex.append(value);
// first = false;
// }
// filterRegex.append(")$");
// RegexStringComparator regexStringComparator = new RegexStringComparator(filterRegex.toString());
// return new SingleColumnValueFilter(_ed.getColumnFamily().getBytes(), qualifierName.getBytes(),
// convertToHBaseCompareOp(entry.getOp()), regexStringComparator);
// }else{
FilterList setFilterList;
if(ComparisonOperator.IN.equals(entry.getOp())){
setFilterList = new FilterList(Operator.MUST_PASS_ONE);
}else if(ComparisonOperator.NOT_IN.equals(entry.getOp())) {
setFilterList = new FilterList(Operator.MUST_PASS_ALL);
}else{
throw new IllegalArgumentException(String.format("Don't support operation: %s on LIST type of value yet: %s, valid options: IN/NOT IN [LIST]",entry.getOp(),entry.toString()));
}
while(it.hasNext()) {
String value = it.next();
BinaryComparator comparator = new BinaryComparator(EntityQualifierUtils.toBytes(_ed, fieldName, value));
SingleColumnValueFilter filter =
new SingleColumnValueFilter(_ed.getColumnFamily().getBytes(), qualifierName.getBytes(), convertToHBaseCompareOp(entry.getOp()), comparator);
filter.setFilterIfMissing(_filterIfMissing);
setFilterList.addFilter(filter);
}
return setFilterList;
// }
}
/**
* @see org.apache.eagle.query.parser.TokenType
* @return
*/
public FilterList buildFilters() {
// TODO: Optimize to select between row filter or column filter for better performance
// Use row key filter priority by default
boolean rowFilterPriority = true;
FilterList fltList = new FilterList(Operator.MUST_PASS_ONE);
for (ANDExpression andExpr : orExpr.getANDExprList()) {
FilterList list = new FilterList(Operator.MUST_PASS_ALL);
Map<String, List<String>> tagFilters = new HashMap<String, List<String>>();
List<QualifierFilterEntity> qualifierFilters = new ArrayList<QualifierFilterEntity>();
// List<QualifierFilterEntry> tagLikeQualifierFilters = new ArrayList<QualifierFilterEntry>();
// TODO refactor not to use too much if/else
for (AtomicExpression ae : andExpr.getAtomicExprList()) {
// TODO temporarily ignore those fields which are not for attributes
String fieldName = ae.getKey();
if (ae.getKeyType() == TokenType.ID) {
fieldName = parseEntityAttribute(fieldName);
if (fieldName == null) {
LOG.warn(fieldName + " field does not have format @<FieldName>, ignored");
continue;
}
}
String value = ae.getValue();
ComparisonOperator op = ae.getOp();
TokenType keyType = ae.getKeyType();
TokenType valueType = ae.getValueType();
QualifierFilterEntity entry = new QualifierFilterEntity(fieldName, value, op, keyType,
valueType);
// TODO Exact match, need to add escape for those special characters here, including:
// "-", "[", "]", "/", "{", "}", "(", ")", "*", "+", "?", ".", "\\", "^", "$", "|"
if (keyType == TokenType.ID && isTag(fieldName)) {
if ((ComparisonOperator.EQUAL.equals(op) || ComparisonOperator.IS.equals(op))
&& !TokenType.NULL.equals(valueType)) {
// Use RowFilter for equal TAG
if (tagFilters.get(fieldName) == null) {
tagFilters.put(fieldName, new ArrayList<String>());
}
tagFilters.get(fieldName).add(value);
} else if (rowFilterPriority && ComparisonOperator.IN.equals(op)) {
// Use RowFilter here by default
if (tagFilters.get(fieldName) == null) {
tagFilters.put(fieldName, new ArrayList<String>());
}
tagFilters.get(fieldName).addAll(EntityQualifierUtils.parseList(value));
} else if (ComparisonOperator.LIKE.equals(op) || ComparisonOperator.NOT_LIKE.equals(op)
|| ComparisonOperator.CONTAINS.equals(op)
|| ComparisonOperator.NOT_CONTAINS.equals(op)
|| ComparisonOperator.IN.equals(op) || ComparisonOperator.IS.equals(op)
|| ComparisonOperator.IS_NOT.equals(op)
|| ComparisonOperator.NOT_EQUAL.equals(op)
|| ComparisonOperator.EQUAL.equals(op)
|| ComparisonOperator.NOT_IN.equals(op)) {
qualifierFilters.add(entry);
} else {
LOG.warn("Don't support operation: \"" + op + "\" on tag field: " + fieldName
+ " yet, going to ignore");
throw new IllegalArgumentException("Don't support operation: " + op
+ " on tag field: " + fieldName
+ ", avaliable options: =, =!, =~, !=~, in, not in, contains, not contains");
}
} else {
qualifierFilters.add(entry);
}
}
// Build RowFilter for equal tags
list.addFilter(buildTagFilter(tagFilters));
// Build SingleColumnValueFilter
FilterList qualifierFilterList = buildQualifierFilter(qualifierFilters);
if (qualifierFilterList != null && qualifierFilterList.getFilters().size() > 0) {
list.addFilter(qualifierFilterList);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("Ignore empty qualifier filter from " + qualifierFilters.toString());
}
}
fltList.addFilter(list);
}
LOG.info("Query: " + orExpr.toString() + " => Filter: " + fltList.toString());
return fltList;
}