下面列出了org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder#newBuilder ( ) 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
private HRegion updateTtl(HRegion region, byte[] family, long ttl) throws Exception {
region.close();
TableDescriptorBuilder tableBuilder =
TableDescriptorBuilder.newBuilder(region.getTableDescriptor());
ColumnFamilyDescriptorBuilder cfd =
ColumnFamilyDescriptorBuilder.newBuilder(tableBuilder.build().getColumnFamily(family));
if (ttl > 0) {
cfd.setValue(Bytes.toBytes(TxConstants.PROPERTY_TTL), Bytes.toBytes(String.valueOf(ttl)));
}
cfd.setMaxVersions(10);
tableBuilder.removeColumnFamily(family);
tableBuilder.addColumnFamily(cfd.build());
return HRegion
.openHRegion(region.getRegionInfo(), tableBuilder.build(), region.getWAL(), conf,
new LocalRegionServerServices(conf, ServerName
.valueOf(InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())),
null);
}
/**
* Get backup system table descriptor
* @return table's descriptor
*/
public static TableDescriptor getSystemTableDescriptor(Configuration conf) {
TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(getTableName(conf));
ColumnFamilyDescriptorBuilder colBuilder =
ColumnFamilyDescriptorBuilder.newBuilder(SESSIONS_FAMILY);
colBuilder.setMaxVersions(1);
Configuration config = HBaseConfiguration.create();
int ttl = config.getInt(BackupRestoreConstants.BACKUP_SYSTEM_TTL_KEY,
BackupRestoreConstants.BACKUP_SYSTEM_TTL_DEFAULT);
colBuilder.setTimeToLive(ttl);
ColumnFamilyDescriptor colSessionsDesc = colBuilder.build();
builder.setColumnFamily(colSessionsDesc);
colBuilder = ColumnFamilyDescriptorBuilder.newBuilder(META_FAMILY);
colBuilder.setTimeToLive(ttl);
builder.setColumnFamily(colBuilder.build());
return builder.build();
}
/**
* Get backup system table descriptor
* @return table's descriptor
*/
public static TableDescriptor getSystemTableForBulkLoadedDataDescriptor(Configuration conf) {
TableDescriptorBuilder builder =
TableDescriptorBuilder.newBuilder(getTableNameForBulkLoadedData(conf));
ColumnFamilyDescriptorBuilder colBuilder =
ColumnFamilyDescriptorBuilder.newBuilder(SESSIONS_FAMILY);
colBuilder.setMaxVersions(1);
Configuration config = HBaseConfiguration.create();
int ttl = config.getInt(BackupRestoreConstants.BACKUP_SYSTEM_TTL_KEY,
BackupRestoreConstants.BACKUP_SYSTEM_TTL_DEFAULT);
colBuilder.setTimeToLive(ttl);
ColumnFamilyDescriptor colSessionsDesc = colBuilder.build();
builder.setColumnFamily(colSessionsDesc);
colBuilder = ColumnFamilyDescriptorBuilder.newBuilder(META_FAMILY);
colBuilder.setTimeToLive(ttl);
builder.setColumnFamily(colBuilder.build());
return builder.build();
}
/**
* Create a set of column descriptors with the combination of compression,
* encoding, bloom codecs available.
* @param prefix family names prefix
* @return the list of column descriptors
*/
public static List<ColumnFamilyDescriptor> generateColumnDescriptors(final String prefix) {
List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>();
long familyId = 0;
for (Compression.Algorithm compressionType: getSupportedCompressionAlgorithms()) {
for (DataBlockEncoding encodingType: DataBlockEncoding.values()) {
for (BloomType bloomType: BloomType.values()) {
String name = String.format("%[email protected]#&-%[email protected]#", prefix, familyId);
ColumnFamilyDescriptorBuilder columnFamilyDescriptorBuilder =
ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(name));
columnFamilyDescriptorBuilder.setCompressionType(compressionType);
columnFamilyDescriptorBuilder.setDataBlockEncoding(encodingType);
columnFamilyDescriptorBuilder.setBloomFilterType(bloomType);
columnFamilyDescriptors.add(columnFamilyDescriptorBuilder.build());
familyId++;
}
}
}
return columnFamilyDescriptors;
}
private void createTable(TableName tableName) throws Exception {
long startTime, endTime;
TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
TableDescriptor desc = builder.build();
ColumnFamilyDescriptorBuilder cbuilder =
ColumnFamilyDescriptorBuilder.newBuilder(COLUMN_NAME.getBytes(Charset.defaultCharset()));
ColumnFamilyDescriptor[] columns = new ColumnFamilyDescriptor[] { cbuilder.build() };
LOG.info("Creating table {} with {} splits.", tableName,
regionsCountPerServer * regionServerCount);
startTime = System.currentTimeMillis();
HBaseTestingUtility.createPreSplitLoadTestTable(util.getConfiguration(), desc, columns,
regionsCountPerServer);
util.waitTableAvailable(tableName);
endTime = System.currentTimeMillis();
LOG.info("Pre-split table created successfully in {}ms.", (endTime - startTime));
}
/**
* Apply a transform to all columns in a given table. If there are no columns in a table
* or if the context is stopping does nothing.
* @param tableName the table to modify
* @param transform the modification to perform. Callers will have the
* column name as a string and a column family builder available to them
*/
protected void modifyAllTableColumns(TableName tableName,
BiConsumer<String, ColumnFamilyDescriptorBuilder> transform) throws IOException {
HBaseTestingUtility util = this.context.getHBaseIntegrationTestingUtility();
Admin admin = util.getAdmin();
TableDescriptor tableDescriptor = admin.getDescriptor(tableName);
ColumnFamilyDescriptor[] columnDescriptors = tableDescriptor.getColumnFamilies();
if (columnDescriptors == null || columnDescriptors.length == 0) {
return;
}
TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableDescriptor);
for (ColumnFamilyDescriptor descriptor : columnDescriptors) {
ColumnFamilyDescriptorBuilder cfd = ColumnFamilyDescriptorBuilder.newBuilder(descriptor);
transform.accept(descriptor.getNameAsString(), cfd);
builder.modifyColumnFamily(cfd.build());
}
// Don't try the modify if we're stopping
if (this.context.isStopping()) {
return;
}
admin.modifyTable(builder.build());
}
/**
* Synchronize column family properties using the default cf properties for a given table
* @param tableDesc table descriptor of table to modify
* @param defaultColFam default column family used as the baseline for property synchronization
* @param syncedProps Map of properties to be kept in sync as read from the default column family descriptor
* @return modified table descriptor builder
*/
private static TableDescriptorBuilder syncColFamProperties(TableDescriptor tableDesc, ColumnFamilyDescriptor defaultColFam,
Map<String, Object> syncedProps) {
TableDescriptorBuilder tableDescBuilder = TableDescriptorBuilder.newBuilder(tableDesc);
// Ensure that all column families have necessary properties in sync (including local index cf if present)
for (ColumnFamilyDescriptor currentColFam: tableDesc.getColumnFamilies()) {
if (!currentColFam.equals(defaultColFam)) {
ColumnFamilyDescriptorBuilder colFamDescBuilder = ColumnFamilyDescriptorBuilder.newBuilder(currentColFam);
for (String prop: MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
String existingPropVal = Bytes.toString(currentColFam.getValue(Bytes.toBytes(prop)));
String expectedPropVal = syncedProps.get(prop).toString();
if (existingPropVal == null || !existingPropVal.toLowerCase().equals(expectedPropVal.toLowerCase())) {
// Need to synchronize this property for the current column family descriptor
colFamDescBuilder.setValue(prop, expectedPropVal);
}
}
if (!colFamDescBuilder.equals(ColumnFamilyDescriptorBuilder.newBuilder(currentColFam))) {
tableDescBuilder.modifyColumnFamily(colFamDescBuilder.build());
}
}
}
return tableDescBuilder;
}
@Test
public void testCreateOnExistingHBaseTable() throws Exception {
String schemaName = generateUniqueName();
String tableName = generateUniqueName();
String indexName = generateUniqueName();
byte[] cf = Bytes.toBytes("f");
try (PhoenixConnection conn = getConnection()){
TableName table = TableName.valueOf(SchemaUtil.getPhysicalHBaseTableName(schemaName,
tableName, isNamespaceMapped).getString());
TableDescriptorBuilder originalDescBuilder = TableDescriptorBuilder.newBuilder(table);
ColumnFamilyDescriptorBuilder familyDescBuilder =
ColumnFamilyDescriptorBuilder.newBuilder(cf);
originalDescBuilder.setColumnFamily(familyDescBuilder.build());
Admin admin = conn.getQueryServices().getAdmin();
admin.createTable(originalDescBuilder.build());
createBaseTable(schemaName, tableName, isMultiTenant, 0, null);
TableDescriptor baseDescriptor = admin.getDescriptor(table);
assertUsingNewCoprocs(baseDescriptor);
createIndexTable(schemaName, tableName, indexName);
baseDescriptor = admin.getDescriptor(table);
TableName indexTable = TableName.valueOf(SchemaUtil.getPhysicalHBaseTableName(schemaName,
indexName, isNamespaceMapped).getString());
TableDescriptor indexDescriptor = admin.getDescriptor(indexTable);
assertUsingNewCoprocs(baseDescriptor, indexDescriptor);
}
}
public static void setCompaction(Connection conn, TableName table, byte[] columnFamily, String key, String value)
throws IOException {
try(Admin admin = conn.getAdmin()) {
TableDescriptor desc = admin.getDescriptor(table);
ColumnFamilyDescriptor cfDesc = desc.getColumnFamily(columnFamily);
ColumnFamilyDescriptorBuilder cfBuilder = ColumnFamilyDescriptorBuilder.newBuilder(cfDesc);
cfBuilder.setValue(Bytes.toBytes(key),Bytes.toBytes(value));
admin.modifyColumnFamily(table, cfBuilder.build());
}
}
private void createTable(TableName tableName, int maxVersions) throws IOException {
ColumnFamilyDescriptorBuilder builder = ColumnFamilyDescriptorBuilder.newBuilder(fam);
if (maxVersions > 0) {
builder.setMaxVersions(maxVersions);
}
TEST_UTIL.getAdmin().createTable(
TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(builder.build()).build());
}
@Test
public void testPreWALAppendNotCalledOnMetaEdit() throws Exception {
final TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() +
"." + name.getMethodName());
TableDescriptorBuilder tdBuilder = TableDescriptorBuilder.newBuilder(tableName);
ColumnFamilyDescriptorBuilder cfBuilder = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY);
tdBuilder.setColumnFamily(cfBuilder.build());
tdBuilder.setCoprocessor(SimpleRegionObserver.class.getName());
TableDescriptor td = tdBuilder.build();
Table table = util.createTable(td, new byte[][] { A, B, C });
PreWALAppendWALActionsListener listener = new PreWALAppendWALActionsListener();
List<HRegion> regions = util.getHBaseCluster().getRegions(tableName);
//should be only one region
HRegion region = regions.get(0);
region.getWAL().registerWALActionsListener(listener);
//flushing should write to the WAL
region.flush(true);
//so should compaction
region.compact(false);
//and so should closing the region
region.close();
//but we still shouldn't have triggered preWALAppend because no user data was written
String[] methods = new String[] {"getCtPreWALAppend"};
Object[] expectedResult = new Integer[]{0};
verifyMethodResult(SimpleRegionObserver.class, methods, tableName, expectedResult);
}
protected void modifyTTL(TableName tableName) throws IOException, InterruptedException {
// Set the TTL to 5 secs, so all the files just written above will get cleaned up on compact.
admin.disableTable(tableName);
utility.waitTableDisabled(tableName.getName());
TableDescriptor descriptor = admin.getDescriptor(tableName);
ColumnFamilyDescriptor colDesc = descriptor.getColumnFamily(FAMILY);
ColumnFamilyDescriptorBuilder cFDB = ColumnFamilyDescriptorBuilder.newBuilder(colDesc);
cFDB.setTimeToLive(5);
admin.modifyColumnFamily(tableName, cFDB.build());
admin.enableTable(tableName);
utility.waitTableEnabled(tableName);
}
@BeforeClass
public static void setUp() throws Exception {
conf.setInt("hfile.format.version", 3);
conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
// Create the table schema
// Specify an encryption algorithm without a key
tdb = TableDescriptorBuilder.newBuilder(TableName.valueOf("default",
"TestEncryptionRandomKeying"));
ColumnFamilyDescriptorBuilder columnFamilyDescriptorBuilder =
ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf"));
String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
columnFamilyDescriptorBuilder.setEncryptionType(algorithm);
tdb.setColumnFamily(columnFamilyDescriptorBuilder.build());
// Start the minicluster
TEST_UTIL.startMiniCluster(1);
// Create the test table
TEST_UTIL.getAdmin().createTable(tdb.build());
TEST_UTIL.waitTableAvailable(tdb.build().getTableName(), 5000);
// Create a store file
Table table = TEST_UTIL.getConnection().getTable(tdb.build().getTableName());
try {
table.put(new Put(Bytes.toBytes("testrow"))
.addColumn(columnFamilyDescriptorBuilder.build().getName(),
Bytes.toBytes("q"), Bytes.toBytes("value")));
} finally {
table.close();
}
TEST_UTIL.getAdmin().flush(tdb.build().getTableName());
}
private void prepareTest(String testId) throws IOException {
tableName = TableName.valueOf("test_table_" + testId);
TableDescriptorBuilder tableDescriptorBuilder =
TableDescriptorBuilder.newBuilder(tableName);
columnFamilyDescriptorBuilder =
ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(CF));
tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptorBuilder.build());
try (Admin admin = TEST_UTIL.getConnection().getAdmin()) {
admin.createTable(tableDescriptorBuilder.build());
}
numBatchesWritten = 0;
}
private static void setupTables() throws IOException {
// Get the table name.
tableName = TableName.valueOf(util.getConfiguration()
.get("hbase.IntegrationTestMTTR.tableName", "IntegrationTestMTTR"));
loadTableName = TableName.valueOf(util.getConfiguration()
.get("hbase.IntegrationTestMTTR.loadTableName", "IntegrationTestMTTRLoadTestTool"));
if (util.getAdmin().tableExists(tableName)) {
util.deleteTable(tableName);
}
if (util.getAdmin().tableExists(loadTableName)) {
util.deleteTable(loadTableName);
}
// Create the table. If this fails then fail everything.
TableDescriptor tableDescriptor = util.getAdmin().getDescriptor(tableName);
TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableDescriptor);
// Make the max file size huge so that splits don't happen during the test.
builder.setMaxFileSize(Long.MAX_VALUE);
ColumnFamilyDescriptorBuilder colDescriptorBldr =
ColumnFamilyDescriptorBuilder.newBuilder(FAMILY);
colDescriptorBldr.setMaxVersions(1);
builder.setColumnFamily(colDescriptorBldr.build());
util.getAdmin().createTable(builder.build());
// Setup the table for LoadTestTool
int ret = loadTool.run(new String[]{"-tn", loadTableName.getNameAsString(), "-init_only"});
assertEquals("Failed to initialize LoadTestTool", 0, ret);
}
/**
* Converts a ColumnFamilySchema to ColumnFamilyDescriptor
* @param cfs the ColumnFamilySchema
* @return An {@link ColumnFamilyDescriptor} made from the passed in <code>cfs</code>
*/
public static ColumnFamilyDescriptor toColumnFamilyDescriptor(final ColumnFamilySchema cfs) {
// Use the empty constructor so we preserve the initial values set on construction for things
// like maxVersion. Otherwise, we pick up wrong values on deserialization which makes for
// unrelated-looking test failures that are hard to trace back to here.
ColumnFamilyDescriptorBuilder builder
= ColumnFamilyDescriptorBuilder.newBuilder(cfs.getName().toByteArray());
cfs.getAttributesList().forEach(a -> builder.setValue(a.getFirst().toByteArray(), a.getSecond().toByteArray()));
cfs.getConfigurationList().forEach(a -> builder.setConfiguration(a.getName(), a.getValue()));
return builder.build();
}
public static ColumnFamilyDescriptor columnFamilyDescriptorFromThrift(
TColumnFamilyDescriptor in) {
ColumnFamilyDescriptorBuilder builder = ColumnFamilyDescriptorBuilder
.newBuilder(in.getName());
if (in.isSetAttributes()) {
for (Map.Entry<ByteBuffer, ByteBuffer> attribute : in.getAttributes().entrySet()) {
builder.setValue(attribute.getKey().array(), attribute.getValue().array());
}
}
if (in.isSetConfiguration()) {
for (Map.Entry<String, String> conf : in.getConfiguration().entrySet()) {
builder.setConfiguration(conf.getKey(), conf.getValue());
}
}
if (in.isSetBlockSize()) {
builder.setBlocksize(in.getBlockSize());
}
if (in.isSetBloomnFilterType()) {
builder.setBloomFilterType(bloomFilterFromThrift(in.getBloomnFilterType()));
}
if (in.isSetCompressionType()) {
builder.setCompressionType(compressionAlgorithmFromThrift(in.getCompressionType()));
}
if (in.isSetDfsReplication()) {
builder.setDFSReplication(in.getDfsReplication());
}
if (in.isSetDataBlockEncoding()) {
builder.setDataBlockEncoding(dataBlockEncodingFromThrift(in.getDataBlockEncoding()));
}
if (in.isSetKeepDeletedCells()) {
builder.setKeepDeletedCells(keepDeletedCellsFromThrift(in.getKeepDeletedCells()));
}
if (in.isSetMaxVersions()) {
builder.setMaxVersions(in.getMaxVersions());
}
if (in.isSetMinVersions()) {
builder.setMinVersions(in.getMinVersions());
}
if (in.isSetScope()) {
builder.setScope(in.getScope());
}
if (in.isSetTimeToLive()) {
builder.setTimeToLive(in.getTimeToLive());
}
if (in.isSetBlockCacheEnabled()) {
builder.setBlockCacheEnabled(in.isBlockCacheEnabled());
}
if (in.isSetCacheBloomsOnWrite()) {
builder.setCacheBloomsOnWrite(in.isCacheBloomsOnWrite());
}
if (in.isSetCacheDataOnWrite()) {
builder.setCacheDataOnWrite(in.isCacheDataOnWrite());
}
if (in.isSetCacheIndexesOnWrite()) {
builder.setCacheIndexesOnWrite(in.isCacheIndexesOnWrite());
}
if (in.isSetCompressTags()) {
builder.setCompressTags(in.isCompressTags());
}
if (in.isSetEvictBlocksOnClose()) {
builder.setEvictBlocksOnClose(in.isEvictBlocksOnClose());
}
if (in.isSetInMemory()) {
builder.setInMemory(in.isInMemory());
}
return builder.build();
}
private void doCopyTableTestWithMob(boolean bulkload) throws Exception {
final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1");
final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2");
final byte[] FAMILY = Bytes.toBytes("mob");
final byte[] COLUMN1 = Bytes.toBytes("c1");
ColumnFamilyDescriptorBuilder cfd = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY);
cfd.setMobEnabled(true);
cfd.setMobThreshold(5);
TableDescriptor desc1 = TableDescriptorBuilder.newBuilder(tableName1)
.setColumnFamily(cfd.build())
.build();
TableDescriptor desc2 = TableDescriptorBuilder.newBuilder(tableName2)
.setColumnFamily(cfd.build())
.build();
try (Table t1 = TEST_UTIL.createTable(desc1, null);
Table t2 = TEST_UTIL.createTable(desc2, null);) {
// put rows into the first table
for (int i = 0; i < 10; i++) {
Put p = new Put(Bytes.toBytes("row" + i));
p.addColumn(FAMILY, COLUMN1, COLUMN1);
t1.put(p);
}
CopyTable copy = new CopyTable();
int code;
if (bulkload) {
code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
copy, new String[] { "--new.name=" + tableName2.getNameAsString(),
"--bulkload", tableName1.getNameAsString() });
} else {
code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
copy, new String[] { "--new.name=" + tableName2.getNameAsString(),
tableName1.getNameAsString() });
}
assertEquals("copy job failed", 0, code);
// verify the data was copied into table 2
for (int i = 0; i < 10; i++) {
Get g = new Get(Bytes.toBytes("row" + i));
Result r = t2.get(g);
assertEquals(1, r.size());
assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
assertEquals("compare row values between two tables",
t1.getDescriptor().getValue("row" + i),
t2.getDescriptor().getValue("row" + i));
}
assertEquals("compare count of mob rows after table copy", MobTestUtil.countMobRows(t1),
MobTestUtil.countMobRows(t2));
assertEquals("compare count of mob row values between two tables",
t1.getDescriptor().getValues().size(),
t2.getDescriptor().getValues().size());
assertTrue("The mob row count is 0 but should be > 0",
MobTestUtil.countMobRows(t2) > 0);
} finally {
TEST_UTIL.deleteTable(tableName1);
TEST_UTIL.deleteTable(tableName2);
}
}
/**
* Apply column family options such as Bloom filters, compression, and data
* block encoding.
*/
protected void applyColumnFamilyOptions(TableName tableName,
byte[][] columnFamilies) throws IOException {
try (Connection conn = ConnectionFactory.createConnection(conf);
Admin admin = conn.getAdmin()) {
TableDescriptor tableDesc = admin.getDescriptor(tableName);
LOG.info("Disabling table " + tableName);
admin.disableTable(tableName);
for (byte[] cf : columnFamilies) {
ColumnFamilyDescriptor columnDesc = tableDesc.getColumnFamily(cf);
boolean isNewCf = columnDesc == null;
ColumnFamilyDescriptorBuilder columnDescBuilder = isNewCf ?
ColumnFamilyDescriptorBuilder.newBuilder(cf) :
ColumnFamilyDescriptorBuilder.newBuilder(columnDesc);
if (bloomType != null) {
columnDescBuilder.setBloomFilterType(bloomType);
}
if (compressAlgo != null) {
columnDescBuilder.setCompressionType(compressAlgo);
}
if (dataBlockEncodingAlgo != null) {
columnDescBuilder.setDataBlockEncoding(dataBlockEncodingAlgo);
}
if (inMemoryCF) {
columnDescBuilder.setInMemory(inMemoryCF);
}
if (cipher != null) {
byte[] keyBytes = new byte[cipher.getKeyLength()];
new SecureRandom().nextBytes(keyBytes);
columnDescBuilder.setEncryptionType(cipher.getName());
columnDescBuilder.setEncryptionKey(
EncryptionUtil.wrapKey(conf,
User.getCurrent().getShortName(),
new SecretKeySpec(keyBytes,
cipher.getName())));
}
if (mobThreshold >= 0) {
columnDescBuilder.setMobEnabled(true);
columnDescBuilder.setMobThreshold(mobThreshold);
}
if (isNewCf) {
admin.addColumnFamily(tableName, columnDescBuilder.build());
} else {
admin.modifyColumnFamily(tableName, columnDescBuilder.build());
}
}
LOG.info("Enabling table " + tableName);
admin.enableTable(tableName);
}
}