org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder#ModifyableColumnFamilyDescriptor ( )源码实例Demo

/**
 * Before.
 *
 * @throws Exception the exception
 */
@PostConstruct
public void before() throws Exception {
    if (!hbaseTemplate.getConnection().getAdmin().tableExists(TableName.valueOf(TABLE_NAME))) {
        TableDescriptorBuilder.ModifyableTableDescriptor tableDesc =
                new TableDescriptorBuilder.ModifyableTableDescriptor(TableName.valueOf(TABLE_NAME));
        ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor columnDesc =
                new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(Bytes.toBytes(FAMILY_NAME));
        tableDesc.setColumnFamily(columnDesc);
        hbaseTemplate.getConnection().getAdmin().createTable(tableDesc);
    }
    hbaseTemplate.execute(TABLE_NAME, tb -> {
        Put put = new Put(Bytes.toBytes(ROW_KEY));
        put.addColumn(Bytes.toBytes(FAMILY_NAME), Bytes.toBytes(QUALIFIER_NAME), Bytes.toBytes(VALUE));
        tb.put(put);
        return null;
    });
}
 

/**
 * Creates a pre-split table for load testing. If the table already exists,
 * logs a warning and continues.
 * @return the number of regions the table was split into
 */
public static int createPreSplitLoadTestTable(Configuration conf,
    TableName tableName, byte[] columnFamily, Algorithm compression,
    DataBlockEncoding dataBlockEncoding, int numRegionsPerServer, int regionReplication,
    Durability durability)
        throws IOException {
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
  tableDescriptor.setDurability(durability);
  tableDescriptor.setRegionReplication(regionReplication);
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(columnFamily);
  familyDescriptor.setDataBlockEncoding(dataBlockEncoding);
  familyDescriptor.setCompressionType(compression);
  return createPreSplitLoadTestTable(conf, tableDescriptor, familyDescriptor,
    numRegionsPerServer);
}
 

/**
 * Create a table.
 * @param tableName
 * @param family
 * @param splitRows
 * @return A Table instance for the created table.
 * @throws IOException
 */
public Table createTable(TableName tableName, byte[] family, byte[][] splitRows)
    throws IOException {
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family);
  if (isNewVersionBehaviorEnabled()) {
    familyDescriptor.setNewVersionBehavior(true);
  }
  tableDescriptor.setColumnFamily(familyDescriptor);
  getAdmin().createTable(tableDescriptor, splitRows);
  // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are
  // assigned
  waitUntilAllRegionsAssigned(tableName);
  return getConnection().getTable(tableName);
}
 

private static void createTable() {
  assertNotNull("HBaseAdmin is not initialized successfully.", admin);
  if (admin != null) {

    TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
      new TableDescriptorBuilder.ModifyableTableDescriptor(name);

    ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
      new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(Bytes.toBytes("f1"));
    tableDescriptor.setColumnFamily(familyDescriptor);

    try {
      admin.createTable(tableDescriptor);
      assertTrue("Fail to create the table", admin.tableExists(name));
    } catch (IOException e) {
      assertNull("Exception found while creating table", e);
    }

  }
}
 

private void initHRegion (byte [] tableName, String callingMethod, int [] maxVersions,
  byte[] ... families)
throws IOException {
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TableName.valueOf(tableName));

  int i=0;
  for(byte [] family : families) {
    ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
      new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family);
    familyDescriptor.setMaxVersions(maxVersions != null ? maxVersions[i++] : 1);
    tableDescriptor.setColumnFamily(familyDescriptor);
  }
  RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
  region = TEST_UTIL.createLocalHRegion(info, tableDescriptor);
}
 

@Override
 public void run() {
   // create a table : master coprocessor will throw an exception and not
   // catch it.
   TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
     new TableDescriptorBuilder.ModifyableTableDescriptor(TableName.valueOf(TEST_TABLE));
   ColumnFamilyDescriptor familyDescriptor =
     new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(TEST_FAMILY);
   tableDescriptor.setColumnFamily(familyDescriptor);
   try {
     Admin admin = UTIL.getAdmin();
     admin.createTable(tableDescriptor);
     fail("BuggyMasterObserver failed to throw an exception.");
   } catch (IOException e) {
     assertEquals("HBaseAdmin threw an interrupted IOException as expected.",
         "java.io.InterruptedIOException", e.getClass().getName());
   }
}
 

private static void create(Admin admin, TableName tableName, byte[]... families)
    throws IOException {
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
  for (byte[] family : families) {
    ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
      new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family);
    familyDescriptor.setMaxVersions(1);
    familyDescriptor.setCompressionType(Algorithm.GZ);
    tableDescriptor.setColumnFamily(familyDescriptor);
  }
  try {
    admin.createTable(tableDescriptor);
  } catch (TableExistsException tee) {
    /* Ignore */
  }
}
 

@Before
public void setUp() throws Exception {
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(testTable.getTableName());
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(TEST_FAMILY1);
  familyDescriptor.setMaxVersions(4);
  tableDescriptor.setOwner(USER_OWNER);
  tableDescriptor.setColumnFamily(familyDescriptor);
  familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(TEST_FAMILY2);
  familyDescriptor.setMaxVersions(4);
  tableDescriptor.setOwner(USER_OWNER);
  tableDescriptor.setColumnFamily(familyDescriptor);
  // Create the test table (owner added to the _acl_ table)
  try (Connection connection = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) {
    try (Admin admin = connection.getAdmin()) {
      admin.createTable(tableDescriptor, new byte[][] { Bytes.toBytes("s") });
    }
  }
  TEST_UTIL.waitTableEnabled(testTable.getTableName());
  LOG.info("Sleeping a second because of HBASE-12581");
  Threads.sleep(1000);
}
 

/**
 * Create a table.
 * @param tableName
 * @param families
 * @param numVersions
 * @return A Table instance for the created table.
 * @throws IOException
 */
public Table createTable(TableName tableName, byte[][] families,
    int[] numVersions) throws IOException {
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
  int i = 0;
  for (byte[] family : families) {
    ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
      new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family)
        .setMaxVersions(numVersions[i]);
    if (isNewVersionBehaviorEnabled()) {
      familyDescriptor.setNewVersionBehavior(true);
    }
    tableDescriptor.setColumnFamily(familyDescriptor);
    i++;
  }
  getAdmin().createTable(tableDescriptor);
  // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are
  // assigned
  waitUntilAllRegionsAssigned(tableName);
  return getConnection().getTable(tableName);
}
 

@Test
public void testModifyFamily() {
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(
      TableName.valueOf(name.getMethodName()));
  byte[] familyName = Bytes.toBytes("cf");
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(familyName)
      .setBlocksize(1000)
      .setDFSReplication((short) 3);
  tableDescriptor.setColumnFamily(familyDescriptor);
  assertEquals(1000, tableDescriptor.getColumnFamily(familyName).getBlocksize());
  assertEquals(3, tableDescriptor.getColumnFamily(familyName).getDFSReplication());
  familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(familyName)
      .setBlocksize(2000)
      .setDFSReplication((short) 1);
  tableDescriptor.modifyColumnFamily(familyDescriptor);
  assertEquals(2000, tableDescriptor.getColumnFamily(familyName).getBlocksize());
  assertEquals(1, tableDescriptor.getColumnFamily(familyName).getDFSReplication());
}
 

public HTableDescriptor createTableDescriptor(final TableName name,
    final int minVersions, final int versions, final int ttl, KeepDeletedCells keepDeleted) {
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(name);
  for (byte[] cfName : new byte[][]{fam1, fam2, fam3}) {
    ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
      new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(cfName)
        .setMinVersions(minVersions)
        .setMaxVersions(versions)
        .setKeepDeletedCells(keepDeleted)
        .setBlockCacheEnabled(false)
        .setTimeToLive(ttl);
    if (isNewVersionBehaviorEnabled()) {
      familyDescriptor.setNewVersionBehavior(true);
    }
    tableDescriptor.setColumnFamily(familyDescriptor);
  }
  return new HTableDescriptor(tableDescriptor);
}
 

@Before
public void setUp() throws Exception {
  // Create the test table (owner added to the _acl_ table)
  Admin admin = TEST_UTIL.getAdmin();
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(testTable.getTableName());
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(TEST_FAMILY);
  familyDescriptor.setMaxVersions(4);
  tableDescriptor.setOwner(USER_OWNER);
  tableDescriptor.setColumnFamily(familyDescriptor);
  admin.createTable(tableDescriptor, new byte[][] { Bytes.toBytes("s") });
  TEST_UTIL.waitTableEnabled(testTable.getTableName());
  LOG.info("Sleeping a second because of HBASE-12581");
  Threads.sleep(1000);
}
 

@Override
protected void initTable() throws IOException {
  // Do the same as the LoadTestTool does, but with different table configuration.
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(getTablename());
  tableDescriptor.setValue(StoreEngine.STORE_ENGINE_CLASS_KEY,
    StripeStoreEngine.class.getName());
  tableDescriptor.setValue(HStore.BLOCKING_STOREFILES_KEY, "100");
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(
      HFileTestUtil.DEFAULT_COLUMN_FAMILY);
  HBaseTestingUtility.createPreSplitLoadTestTable(util.getConfiguration(),
    tableDescriptor, familyDescriptor);
}
 

@Before
public void setUp() throws Exception {
  tableDescriptor = HTU.createModifyableTableDescriptor(test.getMethodName());
  admin = HTU.getAdmin();
  cleanerChore = new MobFileCleanerChore();
  familyDescriptor = new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(fam);
  familyDescriptor.setMobEnabled(true);
  familyDescriptor.setMobThreshold(mobLen);
  familyDescriptor.setMaxVersions(1);
  tableDescriptor.setColumnFamily(familyDescriptor);
  RegionSplitter.UniformSplit splitAlgo = new RegionSplitter.UniformSplit();
  byte[][] splitKeys = splitAlgo.split(numRegions);
  table = HTU.createTable(tableDescriptor, splitKeys).getName();
}
 

@BeforeClass
public static void beforeClass() throws Exception {
  UTIL.getConfiguration().set("hbase.client.retries.number", "3");
  UTIL.startMiniCluster();
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TableName.valueOf(tableAname));
  for (ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor family : families) {
    tableDescriptor.setColumnFamily(family);
  }
  try (Admin admin = UTIL.getAdmin()) {
    admin.createTable(tableDescriptor);
  }
}
 

private static void createTable(Admin admin, TableName tableName, String[] columns)
    throws IOException {
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
  Set<String> cfSet = getColumnFamilies(columns);
  for (String cf : cfSet) {
    ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
      new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(Bytes.toBytes(cf));
    tableDescriptor.setColumnFamily(familyDescriptor);
  }
  LOG.warn(format("Creating table '%s' with '%s' columns and default descriptors.",
    tableName, cfSet));
  admin.createTable(tableDescriptor);
}
 

/**
 * Usecase:
 *
 *  - create a row with 1M cells, 10 bytes in each
 *  - flush & run major compaction
 *  - try to Get whole row.
 *
 *  OOME happened in StoreScanner.next(..).
 *
 * @throws IOException
 */
@Test(expected = RowTooBigException.class)
public void testScanAcrossManySmallColumns() throws IOException {
  byte[] row1 = Bytes.toBytes("row1");
  byte[] fam1 = Bytes.toBytes("fam1");

  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor = TEST_TD;
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor hcd =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(fam1);
  if (tableDescriptor.hasColumnFamily(hcd.getName())) {
    tableDescriptor.modifyColumnFamily(hcd);
  } else {
    tableDescriptor.setColumnFamily(hcd);
  }

  final RegionInfo hri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
  HRegion region = HBaseTestingUtility.createRegionAndWAL(hri, rootRegionDir,
    HTU.getConfiguration(), tableDescriptor);
  try {
    // Add to memstore
    for (int i = 0; i < 10; i++) {
      Put put = new Put(row1);
      for (int j = 0; j < 10 * 10000; j++) {
        byte[] value = new byte[10];
        put.addColumn(fam1, Bytes.toBytes("col_" + i + "_" + j), value);
      }
      region.put(put);
      region.flush(true);
    }
    region.compact(true);

    Get get = new Get(row1);
    region.get(get);
  } finally {
    HBaseTestingUtility.closeRegionAndWAL(region);
  }
}
 

@Test
public void testMutateRow() throws Exception {
  final byte[] qual2 = Bytes.toBytes("qual2");
  TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(fam);
  tableDescriptor.setColumnFamily(familyDescriptor);
  TEST_UTIL.getAdmin().createTable(tableDescriptor);
  try (Table table = TEST_UTIL.getConnection().getTable(tableName)){
    Put p1 = new Put(row1);
    p1.addColumn(fam, qual, value);
    p1.setCellVisibility(new CellVisibility(CONFIDENTIAL));

    Put p2 = new Put(row1);
    p2.addColumn(fam, qual2, value);
    p2.setCellVisibility(new CellVisibility(SECRET));

    RowMutations rm = new RowMutations(row1);
    rm.add(p1);
    rm.add(p2);

    table.mutateRow(rm);

    Get get = new Get(row1);
    get.setAuthorizations(new Authorizations(CONFIDENTIAL));
    Result result = table.get(get);
    assertTrue(result.containsColumn(fam, qual));
    assertFalse(result.containsColumn(fam, qual2));

    get.setAuthorizations(new Authorizations(SECRET));
    result = table.get(get);
    assertFalse(result.containsColumn(fam, qual));
    assertTrue(result.containsColumn(fam, qual2));
  }
}
 

@Test
public void testFlushedFileWithVisibilityTags() throws Exception {
  final byte[] qual2 = Bytes.toBytes("qual2");
  TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(fam);
  tableDescriptor.setColumnFamily(familyDescriptor);
  TEST_UTIL.getAdmin().createTable(tableDescriptor);
  try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
    Put p1 = new Put(row1);
    p1.addColumn(fam, qual, value);
    p1.setCellVisibility(new CellVisibility(CONFIDENTIAL));

    Put p2 = new Put(row1);
    p2.addColumn(fam, qual2, value);
    p2.setCellVisibility(new CellVisibility(SECRET));

    RowMutations rm = new RowMutations(row1);
    rm.add(p1);
    rm.add(p2);

    table.mutateRow(rm);
  }
  TEST_UTIL.getAdmin().flush(tableName);
  List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(tableName);
  HStore store = regions.get(0).getStore(fam);
  Collection<HStoreFile> storefiles = store.getStorefiles();
  assertTrue(storefiles.size() > 0);
  for (HStoreFile storeFile : storefiles) {
    assertTrue(storeFile.getReader().getHFileReader().getFileContext().isIncludesTags());
  }
}
 

@Test
public void testTags() throws Exception {
  Table table = null;
  try {
    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
    byte[] fam = Bytes.toBytes("info");
    byte[] row = Bytes.toBytes("rowa");
    // column names
    byte[] qual = Bytes.toBytes("qual");

    byte[] row1 = Bytes.toBytes("rowb");

    byte[] row2 = Bytes.toBytes("rowc");

    TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
      new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
    ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
      new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(fam);
    familyDescriptor.setBlockCacheEnabled(true);
    familyDescriptor.setDataBlockEncoding(DataBlockEncoding.NONE);
    tableDescriptor.setColumnFamily(familyDescriptor);
    Admin admin = TEST_UTIL.getAdmin();
    admin.createTable(tableDescriptor);
    byte[] value = Bytes.toBytes("value");
    table = TEST_UTIL.getConnection().getTable(tableName);
    Put put = new Put(row);
    put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value);
    put.setAttribute("visibility", Bytes.toBytes("myTag"));
    table.put(put);
    admin.flush(tableName);
    // We are lacking an API for confirming flush request compaction.
    // Just sleep for a short time. We won't be able to confirm flush
    // completion but the test won't hang now or in the future if
    // default compaction policy causes compaction between flush and
    // when we go to confirm it.
    Thread.sleep(1000);

    Put put1 = new Put(row1);
    byte[] value1 = Bytes.toBytes("1000dfsdf");
    put1.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value1);
    // put1.setAttribute("visibility", Bytes.toBytes("myTag3"));
    table.put(put1);
    admin.flush(tableName);
    Thread.sleep(1000);

    Put put2 = new Put(row2);
    byte[] value2 = Bytes.toBytes("1000dfsdf");
    put2.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value2);
    put2.setAttribute("visibility", Bytes.toBytes("myTag3"));
    table.put(put2);
    admin.flush(tableName);
    Thread.sleep(1000);

    result(fam, row, qual, row2, table, value, value2, row1, value1);

    admin.compact(tableName);
    while (admin.getCompactionState(tableName) != CompactionState.NONE) {
      Thread.sleep(10);
    }
    result(fam, row, qual, row2, table, value, value2, row1, value1);
  } finally {
    if (table != null) {
      table.close();
    }
  }
}