下面列出了怎么用org.apache.hadoop.hbase.KeepDeletedCells的API类实例代码及写法,或者点击链接到github查看源代码。
protected final MatchCode tryDropDelete(Cell cell) {
long timestamp = cell.getTimestamp();
// If it is not the time to drop the delete marker, just return
if (timeToPurgeDeletes > 0 && now - timestamp <= timeToPurgeDeletes) {
return MatchCode.INCLUDE;
}
if (keepDeletedCells == KeepDeletedCells.TRUE
|| (keepDeletedCells == KeepDeletedCells.TTL && timestamp >= oldestUnexpiredTS)) {
// If keepDeletedCell is true, or the delete marker is not expired yet, we should include it
// in version counting to see if we can drop it. The only exception is that, we can make
// sure that no put is older than this delete marker. And under this situation, all later
// cells of this column(must be delete markers) can be skipped.
if (timestamp < earliestPutTs) {
return columns.getNextRowOrNextColumn(cell);
} else {
return null;
}
} else {
return MatchCode.SKIP;
}
}
private ScanInfo(byte[] family, int minVersions, int maxVersions, long ttl,
KeepDeletedCells keepDeletedCells, long timeToPurgeDeletes, CellComparator comparator,
long tableMaxRowSize, boolean usePread, long cellsPerTimeoutCheck,
boolean parallelSeekEnabled, long preadMaxBytes, boolean newVersionBehavior) {
this.family = family;
this.minVersions = minVersions;
this.maxVersions = maxVersions;
this.ttl = ttl;
this.keepDeletedCells = keepDeletedCells;
this.timeToPurgeDeletes = timeToPurgeDeletes;
this.comparator = comparator;
this.tableMaxRowSize = tableMaxRowSize;
this.usePread = usePread;
this.cellsPerTimeoutCheck = cellsPerTimeoutCheck;
this.parallelSeekEnabled = parallelSeekEnabled;
this.preadMaxBytes = preadMaxBytes;
this.newVersionBehavior = newVersionBehavior;
}
/**
* The ExplicitColumnTracker does not support "raw" scanning.
*/
@Test
public void testRawScanWithColumns() throws Exception {
HTableDescriptor htd = hbu.createTableDescriptor(TableName.valueOf(name.getMethodName()), 0, 3,
HConstants.FOREVER, KeepDeletedCells.TRUE);
Region region = hbu.createLocalHRegion(htd, null, null);
Scan s = new Scan();
s.setRaw(true);
s.readAllVersions();
s.addColumn(c0, c0);
try {
region.getScanner(s);
fail("raw scanner with columns should have failed");
} catch (org.apache.hadoop.hbase.DoNotRetryIOException dnre) {
// ok!
}
HBaseTestingUtility.closeRegionAndWAL(region);
}
private void testDropDeletes(byte[] from, byte[] to, byte[][] rows, MatchCode... expected)
throws IOException {
long now = EnvironmentEdgeManager.currentTime();
// Set time to purge deletes to negative value to avoid it ever happening.
ScanInfo scanInfo = new ScanInfo(this.conf, fam2, 0, 1, ttl, KeepDeletedCells.FALSE,
HConstants.DEFAULT_BLOCKSIZE, -1L, rowComparator, false);
CompactionScanQueryMatcher qm = CompactionScanQueryMatcher.create(scanInfo,
ScanType.COMPACT_RETAIN_DELETES, Long.MAX_VALUE, HConstants.OLDEST_TIMESTAMP,
HConstants.OLDEST_TIMESTAMP, now, from, to, null);
List<ScanQueryMatcher.MatchCode> actual = new ArrayList<>(rows.length);
byte[] prevRow = null;
for (byte[] row : rows) {
if (prevRow == null || !Bytes.equals(prevRow, row)) {
qm.setToNewRow(KeyValueUtil.createFirstOnRow(row));
prevRow = row;
}
actual.add(qm.match(new KeyValue(row, fam2, null, now, Type.Delete)));
}
assertEquals(expected.length, actual.size());
for (int i = 0; i < expected.length; i++) {
LOG.debug("expected " + expected[i] + ", actual " + actual.get(i));
assertEquals(expected[i], actual.get(i));
}
}
@Test
public void testMinVersionsWithKeepDeletedCellsTTL() throws Exception {
int ttl = 4;
ColumnFamilyDescriptor cfd =
ColumnFamilyDescriptorBuilder.newBuilder(c0)
.setVersionsWithTimeToLive(ttl, 2).build();
verifyVersionedCellKeyValues(ttl, cfd);
cfd = ColumnFamilyDescriptorBuilder.newBuilder(c0)
.setMinVersions(2)
.setMaxVersions(Integer.MAX_VALUE)
.setTimeToLive(ttl)
.setKeepDeletedCells(KeepDeletedCells.TTL)
.build();
verifyVersionedCellKeyValues(ttl, cfd);
}
/**
* Ensure that expired delete family markers don't override valid puts
*/
@Test
public void testExpiredDeleteFamily() throws Exception {
long now = System.currentTimeMillis();
KeyValue[] kvs = new KeyValue[] {
new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now-1000,
KeyValue.Type.DeleteFamily),
create("R1", "cf", "a", now-10, KeyValue.Type.Put,
"dont-care"),
};
List<KeyValueScanner> scanners = scanFixture(kvs);
Scan scan = new Scan();
scan.readVersions(1);
// scanner with ttl equal to 500
ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) {
List<Cell> results = new ArrayList<>();
assertEquals(true, scanner.next(results));
assertEquals(1, results.size());
assertEquals(kvs[1], results.get(0));
results.clear();
assertEquals(false, scanner.next(results));
}
}
@Test
public void testReadVersionWithRawAndFilter() throws IOException {
ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, Long.MAX_VALUE,
KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0
, CellComparator.getInstance(), false);
KeyValue [] kvs = new KeyValue[] {
create("R1", "cf", "a", 3, KeyValue.Type.Put, "dont-care"),
create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care")
};
List<KeyValueScanner> scanners = Arrays.asList(
new KeyValueScanner[]{
new KeyValueScanFixture(CellComparator.getInstance(), kvs)
});
BinaryComparator comp = new BinaryComparator(Bytes.toBytes("a"));
Filter filter = new QualifierFilter(CompareOperator.EQUAL, comp);
Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")).readVersions(2).setRaw(true);
scanSpec.setFilter(filter);
try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, null, scanners)) {
List<Cell> results = new ArrayList<>();
assertEquals(true, scan.next(results));
assertEquals(2, results.size());
}
}
/**
* Helper method to verify HBase column family properties
* @param tableName Physical HBase table whose properties are to be verified
* @param conn Phoenix connection
* @param propModified true if we have altered any of the properties to be kept in sync, false otherwise
* @param ignoreTTL We cannot modfiy a table level property when adding a column, so in those cases,
* ignore the check for TTL modification
* @throws Exception
*/
private void verifyHBaseColumnFamilyProperties(String tableName, Connection conn, boolean propModified,
boolean ignoreTTL) throws Exception {
final int expectedTTL = propModified ? MODIFIED_TTL_VALUE:INITIAL_TTL_VALUE;
final KeepDeletedCells expectedKeepDeletedCells = propModified ? MODIFIED_KEEP_DELETED_CELLS_VALUE: INITIAL_KEEP_DELETED_CELLS_VALUE;
final int expectedReplicationScope = propModified ? MODIFIED_REPLICATION_SCOPE_VALUE:INITIAL_REPLICATION_SCOPE_VALUE;
try (Admin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin()) {
// Note that this includes the local index column family as well
ColumnFamilyDescriptor[] columnFamilies = admin.getDescriptor(TableName.valueOf(tableName))
.getColumnFamilies();
for (ColumnFamilyDescriptor cfd: columnFamilies) {
if (!ignoreTTL) {
assertEquals("Mismatch in TTL", expectedTTL, cfd.getTimeToLive());
}
assertEquals("Mismatch in KEEP_DELETED_CELLS", expectedKeepDeletedCells, cfd.getKeepDeletedCells());
assertEquals("Mismatch in REPLICATION_SCOPE", expectedReplicationScope, cfd.getScope());
}
}
}
private static HColumnDescriptor createColumnFamily() {
return new HColumnDescriptor(CF_NAME)
.setMaxVersions(1)
.setBlockCacheEnabled(true)
.setBloomFilterType(BloomType.ROW)
.setCompressionType(DEFAULT_COMPRESSION_ALGORITHM)
.setDataBlockEncoding(DEFAULT_DATABLOCK_ENCODING)
.setCacheBloomsOnWrite(true)
.setCacheDataOnWrite(true)
.setCacheIndexesOnWrite(true)
.setKeepDeletedCells(KeepDeletedCells.FALSE)
.setValue(HTableDescriptor.MAX_FILESIZE, REGION_MAX_FILESIZE)
.setValue(HTableDescriptor.SPLIT_POLICY, REGION_SPLIT_POLICY);
}
public static void main(String[] args) throws Exception {
String quorum = "192.168.0.30,192.168.0.31,192.168.0.32";
//quorum = "192.168.8.191,192.168.1.192,192.168.1.193";
int port = 2181;
String znode = "/hyperbase1";
HBaseConnPool connPool = new HBaseClientManager(quorum, port, znode);
HBaseDDLHandler ddlHandler = new HBaseDDLHandler(connPool);
String tableName = "demo_test";
System.out.println("=============================== : delete");
ddlHandler.deleteTable(tableName);
String columnFamily = "cf";
System.out.println("=============================== : create");
ddlHandler.createTable(tableName, columnFamily, "cf2");
System.out.println("=============================== : desc");
HBaseUtils.printTableInfo(ddlHandler.getTable(tableName));
System.out.println("=============================== : alter");
HBaseAdmin admin = new HBaseAdmin(connPool.getConn());
admin.disableTable(tableName);
HTableInterface htable = ddlHandler.getTable(tableName);
HTableDescriptor tableDesc = admin.getTableDescriptor(htable.getTableName());
tableDesc.removeFamily(Bytes.toBytes("cf2"));
HColumnDescriptor newhcd = new HColumnDescriptor("cf3");
newhcd.setMaxVersions(2);
newhcd.setKeepDeletedCells(KeepDeletedCells.TRUE);
tableDesc.addFamily(newhcd);
admin.modifyTable(tableName, tableDesc);
admin.enableTable(tableName);
admin.close();
System.out.println("=============================== : desc");
HBaseUtils.printTableInfo(ddlHandler.getTable(tableName));
System.out.println("=============================== : delete");
ddlHandler.deleteTable(tableName);
connPool.closeConn();
}
public static KeepDeletedCells keepDeletedCellsFromThrift(TKeepDeletedCells in) {
switch (in.getValue()) {
case 0: return KeepDeletedCells.FALSE;
case 1: return KeepDeletedCells.TRUE;
case 2: return KeepDeletedCells.TTL;
default: return KeepDeletedCells.FALSE;
}
}
public static TKeepDeletedCells keepDeletedCellsFromHBase(KeepDeletedCells in) {
switch (in) {
case FALSE: return TKeepDeletedCells.FALSE;
case TRUE: return TKeepDeletedCells.TRUE;
case TTL: return TKeepDeletedCells.TTL;
default: return TKeepDeletedCells.FALSE;
}
}
protected NormalUserScanQueryMatcher(Scan scan, ScanInfo scanInfo, ColumnTracker columns,
boolean hasNullColumn, DeleteTracker deletes, long oldestUnexpiredTS, long now) {
super(scan, scanInfo, columns, hasNullColumn, oldestUnexpiredTS, now);
this.deletes = deletes;
this.get = scan.isGetScan();
this.seePastDeleteMarkers = scanInfo.getKeepDeletedCells() != KeepDeletedCells.FALSE;
}
protected final void trackDelete(Cell cell) {
// If keepDeletedCells is true, then we only remove cells by versions or TTL during
// compaction, so we do not need to track delete here.
// If keepDeletedCells is TTL and the delete marker is expired, then we can make sure that the
// minVerions is larger than 0(otherwise we will just return at preCheck). So here we still
// need to track the delete marker to see if it masks some cells.
if (keepDeletedCells == KeepDeletedCells.FALSE
|| (keepDeletedCells == KeepDeletedCells.TTL && cell.getTimestamp() < oldestUnexpiredTS)) {
deletes.add(cell);
}
}
private ScanInfo getScanInfo() {
int oldMaxVersions = 1;
int oldMinVersions = 0;
long oldTTL = 10000;
return new ScanInfo(conf, Bytes.toBytes("cf"), oldMinVersions, oldMaxVersions, oldTTL,
KeepDeletedCells.FALSE, HConstants.FOREVER, 1000,
CellComparator.getInstance(), true);
}
/**
* This is a cryptic test. It is checking that we don't include a fake cell, one that has a
* timestamp of {@link HConstants#OLDEST_TIMESTAMP}. See HBASE-16074 for background.
* @throws IOException
*/
@Test
public void testNeverIncludeFakeCell() throws IOException {
long now = EnvironmentEdgeManager.currentTime();
// Do with fam2 which has a col2 qualifier.
UserScanQueryMatcher qm = UserScanQueryMatcher.create(scan,
new ScanInfo(this.conf, fam2, 10, 1, ttl, KeepDeletedCells.FALSE,
HConstants.DEFAULT_BLOCKSIZE, 0, rowComparator, false),
get.getFamilyMap().get(fam2), now - ttl, now, null);
Cell kv = new KeyValue(row1, fam2, col2, 1, data);
Cell cell = PrivateCellUtil.createLastOnRowCol(kv);
qm.setToNewRow(kv);
MatchCode code = qm.match(cell);
assertFalse(code.compareTo(MatchCode.SEEK_NEXT_COL) != 0);
}
/**
* Verify that {@link ScanQueryMatcher} only skips expired KeyValue instances and does not exit
* early from the row (skipping later non-expired KeyValues). This version mimics a Get with
* explicitly specified column qualifiers.
* @throws IOException
*/
@Test
public void testMatch_ExpiredExplicit() throws IOException {
long testTTL = 1000;
MatchCode[] expected = new MatchCode[] { ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL,
ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL,
ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW, ScanQueryMatcher.MatchCode.DONE };
long now = EnvironmentEdgeManager.currentTime();
UserScanQueryMatcher qm = UserScanQueryMatcher.create(scan,
new ScanInfo(this.conf, fam2, 0, 1, testTTL, KeepDeletedCells.FALSE,
HConstants.DEFAULT_BLOCKSIZE, 0, rowComparator, false),
get.getFamilyMap().get(fam2), now - testTTL, now, null);
KeyValue[] kvs = new KeyValue[] { new KeyValue(row1, fam2, col1, now - 100, data),
new KeyValue(row1, fam2, col2, now - 50, data),
new KeyValue(row1, fam2, col3, now - 5000, data),
new KeyValue(row1, fam2, col4, now - 500, data),
new KeyValue(row1, fam2, col5, now - 10000, data),
new KeyValue(row2, fam1, col1, now - 10, data) };
KeyValue k = kvs[0];
qm.setToNewRow(k);
List<MatchCode> actual = new ArrayList<>(kvs.length);
for (KeyValue kv : kvs) {
actual.add(qm.match(kv));
}
assertEquals(expected.length, actual.size());
for (int i = 0; i < expected.length; i++) {
LOG.debug("expected " + expected[i] + ", actual " + actual.get(i));
assertEquals(expected[i], actual.get(i));
}
}
/**
* Verify that {@link ScanQueryMatcher} only skips expired KeyValue instances and does not exit
* early from the row (skipping later non-expired KeyValues). This version mimics a Get with
* wildcard-inferred column qualifiers.
* @throws IOException
*/
@Test
public void testMatch_ExpiredWildcard() throws IOException {
long testTTL = 1000;
MatchCode[] expected =
new MatchCode[] { ScanQueryMatcher.MatchCode.INCLUDE, ScanQueryMatcher.MatchCode.INCLUDE,
ScanQueryMatcher.MatchCode.SEEK_NEXT_COL, ScanQueryMatcher.MatchCode.INCLUDE,
ScanQueryMatcher.MatchCode.SEEK_NEXT_COL, ScanQueryMatcher.MatchCode.DONE };
long now = EnvironmentEdgeManager.currentTime();
UserScanQueryMatcher qm =
UserScanQueryMatcher.create(scan,
new ScanInfo(this.conf, fam2, 0, 1, testTTL, KeepDeletedCells.FALSE,
HConstants.DEFAULT_BLOCKSIZE, 0, rowComparator, false),
null, now - testTTL, now, null);
KeyValue[] kvs = new KeyValue[] { new KeyValue(row1, fam2, col1, now - 100, data),
new KeyValue(row1, fam2, col2, now - 50, data),
new KeyValue(row1, fam2, col3, now - 5000, data),
new KeyValue(row1, fam2, col4, now - 500, data),
new KeyValue(row1, fam2, col5, now - 10000, data),
new KeyValue(row2, fam1, col1, now - 10, data) };
KeyValue k = kvs[0];
qm.setToNewRow(k);
List<ScanQueryMatcher.MatchCode> actual = new ArrayList<>(kvs.length);
for (KeyValue kv : kvs) {
actual.add(qm.match(kv));
}
assertEquals(expected.length, actual.size());
for (int i = 0; i < expected.length; i++) {
LOG.debug("expected " + expected[i] + ", actual " + actual.get(i));
assertEquals(expected[i], actual.get(i));
}
}
@Test
public void testMatchWhenFilterReturnsIncludeAndSeekNextRow() throws IOException {
List<MatchCode> expected = new ArrayList<>();
expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW);
expected.add(ScanQueryMatcher.MatchCode.DONE);
Scan scanWithFilter = new Scan(scan).setFilter(new AlwaysIncludeAndSeekNextRowFilter());
long now = EnvironmentEdgeManager.currentTime();
// scan with column 2,4,5
UserScanQueryMatcher qm = UserScanQueryMatcher.create(
scanWithFilter, new ScanInfo(this.conf, fam2, 0, 1, ttl, KeepDeletedCells.FALSE,
HConstants.DEFAULT_BLOCKSIZE, 0, rowComparator, false),
get.getFamilyMap().get(fam2), now - ttl, now, null);
List<KeyValue> memstore = new ArrayList<>();
// ColumnTracker will return INCLUDE_AND_SEEK_NEXT_COL , and filter will return
// INCLUDE_AND_SEEK_NEXT_ROW, so final match code will be INCLUDE_AND_SEEK_NEXT_ROW.
memstore.add(new KeyValue(row1, fam2, col2, 1, data));
memstore.add(new KeyValue(row2, fam1, col1, data));
List<ScanQueryMatcher.MatchCode> actual = new ArrayList<>(memstore.size());
KeyValue k = memstore.get(0);
qm.setToNewRow(k);
for (KeyValue kv : memstore) {
actual.add(qm.match(kv));
}
assertEquals(expected.size(), actual.size());
for (int i = 0; i < expected.size(); i++) {
LOG.debug("expected " + expected.get(i) + ", actual " + actual.get(i));
assertEquals(expected.get(i), actual.get(i));
}
}
/**
* Here is the unit test for UserScanQueryMatcher#mergeFilterResponse: the match code may be
* changed to SEEK_NEXT_COL or INCLUDE_AND_SEEK_NEXT_COL after merging with filterResponse, even
* if the passed match code is neither SEEK_NEXT_COL nor INCLUDE_AND_SEEK_NEXT_COL. In that case,
* we need to make sure that the ColumnTracker has been switched to the next column. <br/>
* An effective test way is: we only need to check the cell from getKeyForNextColumn(). because
* that as long as the UserScanQueryMatcher returns SEEK_NEXT_COL or INCLUDE_AND_SEEK_NEXT_COL,
* UserScanQueryMatcher#getKeyForNextColumn should return an cell whose column is larger than the
* current cell's.
*/
@Test
public void testMergeFilterResponseCase2() throws Exception {
List<MatchCode> expected = new ArrayList<>();
expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
Scan scanWithFilter = new Scan(scan).setFilter(new AlwaysIncludeFilter()).readVersions(3);
long now = EnvironmentEdgeManager.currentTime();
// scan with column 2,4,5, the family with maxVersion = 5
UserScanQueryMatcher qm = UserScanQueryMatcher.create(
scanWithFilter, new ScanInfo(this.conf, fam2, 0, 5, ttl, KeepDeletedCells.FALSE,
HConstants.DEFAULT_BLOCKSIZE, 0, rowComparator, false),
get.getFamilyMap().get(fam2), now - ttl, now, null);
List<KeyValue> memstore = new ArrayList<>();
memstore.add(new KeyValue(row1, fam1, col2, 1, data)); // match code will be INCLUDE
memstore.add(new KeyValue(row1, fam1, col2, 2, data)); // match code will be INCLUDE
memstore.add(new KeyValue(row1, fam1, col2, 3, data)); // match code will be INCLUDE
memstore.add(new KeyValue(row1, fam1, col2, 4, data)); // match code will be SEEK_NEXT_COL
KeyValue k = memstore.get(0);
qm.setToNewRow(k);
for (int i = 0; i < memstore.size(); i++) {
assertEquals(expected.get(i), qm.match(memstore.get(i)));
}
// For last cell, the query matcher will return SEEK_NEXT_COL, and the
// ColumnTracker will skip to the next column, which is col4.
Cell lastCell = memstore.get(memstore.size() - 1);
Cell nextCell = qm.getKeyForNextColumn(lastCell);
assertArrayEquals(nextCell.getQualifierArray(), col4);
}
@Test
public void testWildCardTtlScan() throws IOException {
long now = System.currentTimeMillis();
KeyValue [] kvs = new KeyValue[] {
create("R1", "cf", "a", now-1000, KeyValue.Type.Put, "dont-care"),
create("R1", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"),
create("R1", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"),
create("R1", "cf", "d", now-10000, KeyValue.Type.Put, "dont-care"),
create("R2", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
create("R2", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"),
create("R2", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"),
create("R2", "cf", "c", now-1000, KeyValue.Type.Put, "dont-care")
};
List<KeyValueScanner> scanners = scanFixture(kvs);
Scan scan = new Scan();
scan.readVersions(1);
ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) {
List<Cell> results = new ArrayList<>();
assertEquals(true, scanner.next(results));
assertEquals(2, results.size());
assertEquals(kvs[1], results.get(0));
assertEquals(kvs[2], results.get(1));
results.clear();
assertEquals(true, scanner.next(results));
assertEquals(3, results.size());
assertEquals(kvs[4], results.get(0));
assertEquals(kvs[5], results.get(1));
assertEquals(kvs[6], results.get(2));
results.clear();
assertEquals(false, scanner.next(results));
}
}
@Test
public void testPreadNotEnabledForCompactionStoreScanners() throws Exception {
long now = System.currentTimeMillis();
KeyValue[] kvs = new KeyValue[] {
new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now - 1000,
KeyValue.Type.DeleteFamily),
create("R1", "cf", "a", now - 10, KeyValue.Type.Put, "dont-care"), };
List<KeyValueScanner> scanners = scanFixture(kvs);
ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
try (StoreScanner storeScanner = new StoreScanner(scanInfo, -1,
ScanType.COMPACT_RETAIN_DELETES, scanners)) {
assertFalse(storeScanner.isScanUsePread());
}
}
/**
* Retain all versions for a given TTL(retentionInterval), and then only a specific number
* of versions(versionAfterInterval) after that interval elapses.
*
* @param retentionInterval Retain all versions for this interval
* @param versionAfterInterval Retain no of versions to retain after retentionInterval
* @return this (for chained invocation)
*/
public ModifyableColumnFamilyDescriptor setVersionsWithTimeToLive(
final int retentionInterval, final int versionAfterInterval) {
ModifyableColumnFamilyDescriptor modifyableColumnFamilyDescriptor =
setVersions(versionAfterInterval, Integer.MAX_VALUE);
modifyableColumnFamilyDescriptor.setTimeToLive(retentionInterval);
modifyableColumnFamilyDescriptor.setKeepDeletedCells(KeepDeletedCells.TTL);
return modifyableColumnFamilyDescriptor;
}
@Test
public void testDropViewKeepsHTable() throws Exception {
Connection conn = getConnection();
Admin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
String hbaseNativeViewName = generateUniqueName();
byte[] hbaseNativeBytes = SchemaUtil.getTableNameAsBytes(HBASE_NATIVE_SCHEMA_NAME, hbaseNativeViewName);
try {
TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(hbaseNativeBytes));
ColumnFamilyDescriptor columnDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY_NAME)
.setKeepDeletedCells(KeepDeletedCells.TRUE).build();
builder.addColumnFamily(columnDescriptor);
admin.createTable(builder.build());
} finally {
admin.close();
}
conn.createStatement().execute("create view " + hbaseNativeViewName+
" (uint_key unsigned_int not null," +
" ulong_key unsigned_long not null," +
" string_key varchar not null,\n" +
" \"1\".uint_col unsigned_int," +
" \"1\".ulong_col unsigned_long" +
" CONSTRAINT pk PRIMARY KEY (uint_key, ulong_key, string_key))\n" +
ColumnFamilyDescriptorBuilder.DATA_BLOCK_ENCODING + "='" + DataBlockEncoding.NONE + "'");
conn.createStatement().execute("drop view " + hbaseNativeViewName);
conn.close();
}
/**
* @param admin to create the table
* @param index descriptor to update before creating table
*/
public static void createIndexTable(Admin admin, TableDescriptorBuilder indexBuilder) throws IOException {
indexBuilder.addColumnFamily(
ColumnFamilyDescriptorBuilder.newBuilder(CoveredColumnIndexCodec.INDEX_ROW_COLUMN_FAMILY)
.setKeepDeletedCells(KeepDeletedCells.TRUE).build());
admin.createTable(indexBuilder.build());
}
@Test
public void testSetHTableHColumnAndPhoenixTableProperties() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
String ddl = "CREATE TABLE T3 (\n"
+"ID1 VARCHAR(15) NOT NULL,\n"
+"ID2 VARCHAR(15) NOT NULL,\n"
+"CREATED_DATE DATE,\n"
+"CF1.CREATION_TIME BIGINT,\n"
+"CF2.LAST_USED DATE,\n"
+"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) IMMUTABLE_ROWS=true";
Connection conn = DriverManager.getConnection(getUrl(), props);
conn.createStatement().execute(ddl);
assertImmutableRows(conn, "T3", true);
ddl = "ALTER TABLE T3 SET COMPACTION_ENABLED = FALSE, VERSIONS = 10";
conn.createStatement().execute(ddl);
ddl = "ALTER TABLE T3 SET COMPACTION_ENABLED = FALSE, CF1.MIN_VERSIONS = 1, CF2.MIN_VERSIONS = 3, MIN_VERSIONS = 8, IMMUTABLE_ROWS=false, CF1.KEEP_DELETED_CELLS = true, KEEP_DELETED_CELLS = false";
conn.createStatement().execute(ddl);
assertImmutableRows(conn, "T3", false);
try (HBaseAdmin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin()) {
HTableDescriptor tableDesc = admin.getTableDescriptor(Bytes.toBytes("T3"));
HColumnDescriptor[] columnFamilies = tableDesc.getColumnFamilies();
assertEquals(3, columnFamilies.length);
assertEquals("0", columnFamilies[0].getNameAsString());
assertEquals(8, columnFamilies[0].getMinVersions());
assertEquals(10, columnFamilies[0].getMaxVersions());
assertEquals(KeepDeletedCells.FALSE, columnFamilies[0].getKeepDeletedCellsAsEnum());
assertEquals("CF1", columnFamilies[1].getNameAsString());
assertEquals(1, columnFamilies[1].getMinVersions());
assertEquals(10, columnFamilies[1].getMaxVersions());
assertEquals(KeepDeletedCells.TRUE, columnFamilies[1].getKeepDeletedCellsAsEnum());
assertEquals("CF2", columnFamilies[2].getNameAsString());
assertEquals(3, columnFamilies[2].getMinVersions());
assertEquals(10, columnFamilies[2].getMaxVersions());
assertEquals(KeepDeletedCells.FALSE, columnFamilies[2].getKeepDeletedCellsAsEnum());
assertEquals(Boolean.toString(false), tableDesc.getValue(HTableDescriptor.COMPACTION_ENABLED));
}
}
@Override
public void setKeepDeletedCells(KeepDeletedCells keepDeletedCells) {
this.keepDeletedCells = keepDeletedCells;
}
@Override
public KeepDeletedCells getKeepDeletedCells() {
return keepDeletedCells != null ? keepDeletedCells : scanInfo.getKeepDeletedCells();
}
public KeepDeletedCells getKeepDeletedCells() {
return keepDeletedCells;
}
/**
* Used for CP users for customizing max versions, ttl and keepDeletedCells.
*/
ScanInfo customize(int maxVersions, long ttl, KeepDeletedCells keepDeletedCells) {
return customize(maxVersions, ttl, keepDeletedCells, minVersions);
}