下面列出了org.apache.hadoop.hbase.client.Connection#getAdmin ( ) 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
private void alter() throws IOException {
Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
Admin hbaseAdmin = null;
try {
hbaseAdmin = conn.getAdmin();
HTableDescriptor table = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
hbaseAdmin.disableTable(table.getTableName());
table.setValue(metadataKey, metadataValue);
hbaseAdmin.modifyTable(table.getTableName(), table);
hbaseAdmin.enableTable(table.getTableName());
} finally {
if (hbaseAdmin != null) {
hbaseAdmin.close();
}
}
}
private static void startThirdCluster() throws Exception {
LOG.info("Setup Zk to same one from UTIL1 and UTIL2");
UTIL3.setZkCluster(UTIL1.getZkCluster());
UTIL3.startMiniCluster(NUM_SLAVES1);
TableDescriptor table = TableDescriptorBuilder.newBuilder(tableName)
.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(famName)
.setMobEnabled(true)
.setMobThreshold(4000)
.setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build())
.setColumnFamily(ColumnFamilyDescriptorBuilder.of(noRepfamName)).build();
Connection connection3 = ConnectionFactory.createConnection(CONF3);
try (Admin admin3 = connection3.getAdmin()) {
admin3.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
}
UTIL3.waitUntilAllRegionsAssigned(tableName);
htable3 = connection3.getTable(tableName);
}
private static void show() throws IOException {
Map<String, List<String>> envs = Maps.newHashMap();
// get all kylin hbase tables
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix();
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
for (HTableDescriptor desc : tableDescriptors) {
String host = desc.getValue(IRealizationConstants.HTableTag);
if (StringUtils.isEmpty(host)) {
add("unknown", desc.getNameAsString(), envs);
} else {
add(host, desc.getNameAsString(), envs);
}
}
for (Map.Entry<String, List<String>> entry : envs.entrySet()) {
System.out.println(entry.getKey() + " has htable count: " + entry.getValue().size());
}
hbaseAdmin.close();
}
private void clean() throws IOException {
Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
Admin hbaseAdmin = conn.getAdmin();
for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
String name = descriptor.getNameAsString().toLowerCase(Locale.ROOT);
if (name.startsWith("kylin") || name.startsWith("_kylin")) {
String x = descriptor.getValue(IRealizationConstants.HTableTag);
System.out.println("table name " + descriptor.getNameAsString() + " host: " + x);
System.out.println(descriptor);
System.out.println();
descriptor.setValue(IRealizationConstants.HTableOwner, "[email protected]");
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), descriptor);
}
}
hbaseAdmin.close();
}
public static void main(String[] args) throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
// tag::MERGE1[]
Configuration conf = HBaseConfiguration.create();
Connection connection = ConnectionFactory.createConnection(conf);
HBaseAdmin admin = (HBaseAdmin)connection.getAdmin();
List<HRegionInfo> regions = admin.getTableRegions(TableName.valueOf("t1")); //<1>
LOG.info("testtable contains " + regions.size() + " regions.");
for (int index = 0; index < regions.size() / 2; index++) {
HRegionInfo region1 = regions.get(index*2);
HRegionInfo region2 = regions.get(index*2+1);
LOG.info("Merging regions " + region1 + " and " + region2);
admin.mergeRegions(region1.getEncodedNameAsBytes(),
region2.getEncodedNameAsBytes(), false); //<2>
}
admin.close();
// end::MERGE1[]
}
public static boolean tableExists(Connection conn, String tableName) throws IOException {
Admin hbase = conn.getAdmin();
try {
return hbase.tableExists(TableName.valueOf(tableName));
} finally {
hbase.close();
}
}
@VisibleForTesting
HRegionFileSystem getFileSystem(Connection connection) throws IOException {
Admin admin = connection.getAdmin();
return HRegionFileSystem.openRegionFromFileSystem(admin.getConfiguration(),
CommonFSUtils.getCurrentFileSystem(admin.getConfiguration()), CommonFSUtils.getTableDir(
CommonFSUtils.getRootDir(admin.getConfiguration()), region.getTable()),
region, true);
}
/**
*
* @param sourceTableName
* @param sourceTable
* @param kylinConfig
* @return Pair of HTableName and shard number
* @throws IOException
*/
private Pair<String, Integer> createHTable(String sourceTableName, IReadableTable sourceTable,
KylinConfig kylinConfig) throws IOException {
TableSignature signature = sourceTable.getSignature();
int shardNum = calculateShardNum(kylinConfig, signature.getSize());
Connection conn = getHBaseConnection(kylinConfig);
Admin admin = conn.getAdmin();
String hTableName = genHTableName(kylinConfig, admin, sourceTableName);
TableName tableName = TableName.valueOf(hTableName);
HTableDescriptor hTableDesc = new HTableDescriptor(tableName);
hTableDesc.setCompactionEnabled(false);
hTableDesc.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName());
hTableDesc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix());
hTableDesc.setValue(IRealizationConstants.HTableCreationTime, String.valueOf(System.currentTimeMillis()));
String commitInfo = KylinVersion.getGitCommitInfo();
if (!StringUtils.isEmpty(commitInfo)) {
hTableDesc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
}
HColumnDescriptor cf = CubeHTableUtil.createColumnFamily(kylinConfig, HBaseLookupRowEncoder.CF_STRING, false);
hTableDesc.addFamily(cf);
try {
if (shardNum > 1) {
admin.createTable(hTableDesc, getSplitsByShardNum(shardNum));
} else {
admin.createTable(hTableDesc);
}
} finally {
IOUtils.closeQuietly(admin);
}
return new Pair<>(hTableName, shardNum);
}
private static void startClusters() throws Exception {
UTIL1.startMiniZKCluster();
MiniZooKeeperCluster miniZK = UTIL1.getZkCluster();
LOG.info("Setup first Zk");
UTIL2.setZkCluster(miniZK);
LOG.info("Setup second Zk");
CONF_WITH_LOCALFS = HBaseConfiguration.create(CONF1);
UTIL1.startMiniCluster(NUM_SLAVES1);
// Have a bunch of slave servers, because inter-cluster shipping logic uses number of sinks
// as a component in deciding maximum number of parallel batches to send to the peer cluster.
UTIL2.startMiniCluster(NUM_SLAVES2);
hbaseAdmin = ConnectionFactory.createConnection(CONF1).getAdmin();
TableDescriptor table = TableDescriptorBuilder.newBuilder(tableName)
.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(famName).setMaxVersions(100)
.setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build())
.setColumnFamily(ColumnFamilyDescriptorBuilder.of(noRepfamName)).build();
Connection connection1 = ConnectionFactory.createConnection(CONF1);
Connection connection2 = ConnectionFactory.createConnection(CONF2);
try (Admin admin1 = connection1.getAdmin()) {
admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
}
try (Admin admin2 = connection2.getAdmin()) {
admin2.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
}
UTIL1.waitUntilAllRegionsAssigned(tableName);
UTIL2.waitUntilAllRegionsAssigned(tableName);
htable1 = connection1.getTable(tableName);
htable2 = connection2.getTable(tableName);
}
@BeforeClass
public static void setUpBeforeClass() throws Exception {
TestReplicationBase.setUpBeforeClass();
TableDescriptor peerTable = TableDescriptorBuilder.newBuilder(peerTableName).setColumnFamily(
ColumnFamilyDescriptorBuilder.newBuilder(noRepfamName).setMaxVersions(100)
.build()).build();
Connection connection2 = ConnectionFactory.createConnection(CONF2);
try (Admin admin2 = connection2.getAdmin()) {
admin2.createTable(peerTable, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
}
htable3 = connection2.getTable(peerTableName);
}
@BeforeClass
public static void setUpBeforeClass() throws Exception {
TestReplicationBase.setUpBeforeClass();
TableDescriptor peerTable = TableDescriptorBuilder.newBuilder(peerTableName).setColumnFamily(
ColumnFamilyDescriptorBuilder.newBuilder(noRepfamName).setMaxVersions(100)
.build()).build();
Connection connection2 = ConnectionFactory.createConnection(CONF2);
try (Admin admin2 = connection2.getAdmin()) {
admin2.createTable(peerTable, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
}
htable3 = connection2.getTable(peerTableName);
}
@Before
public void setUp() throws IOException {
Configuration conf = HBaseConfiguration.create();
// conf.set("hbase.master", "node-02:16010;node-03:16010");
conf.set("hbase.zookeeper.quorum", "node-01:2181,node-02:2181,node-03:2181");
Connection connection = ConnectionFactory.createConnection(conf);
admin = connection.getAdmin();
}
/**
* Test a table creation including a coprocessor path
* which is not whitelisted. Table will not be created due to the
* offending coprocessor.
*/
@Test
public void testCreationNonWhitelistedCoprocessorPath() throws Exception {
Configuration conf = UTIL.getConfiguration();
// load coprocessor under test
conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
CoprocessorWhitelistMasterObserver.class.getName());
conf.setStrings(CoprocessorWhitelistMasterObserver.CP_COPROCESSOR_WHITELIST_PATHS_KEY,
new String[]{});
// set retries low to raise exception quickly
conf.setInt("hbase.client.retries.number", 5);
UTIL.startMiniCluster();
TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
new TableDescriptorBuilder.ModifyableTableDescriptor(TEST_TABLE);
ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(TEST_FAMILY);
tableDescriptor.setColumnFamily(familyDescriptor);
tableDescriptor.setCoprocessor(
CoprocessorDescriptorBuilder.newBuilder("net.clayb.hbase.coprocessor.NotWhitelisted")
.setJarPath("file:///notpermitted/couldnotpossiblyexist.jar")
.setPriority(Coprocessor.PRIORITY_USER)
.setProperties(Collections.emptyMap())
.build());
Connection connection = ConnectionFactory.createConnection(conf);
Admin admin = connection.getAdmin();
LOG.info("Creating Table");
try {
admin.createTable(tableDescriptor);
fail("Expected coprocessor to raise IOException");
} catch (IOException e) {
// swallow exception from coprocessor
}
LOG.info("Done Creating Table");
// ensure table was not created
assertEquals(0,
admin.listTableDescriptors(Pattern.compile("^" + TEST_TABLE.getNameAsString() + "$")).size());
}
/**
* List all the userPermissions matching the given table pattern and user name.
* @param connection Connection
* @param tableRegex The regular expression string to match against
* @param userName User name, if empty then all user permissions will be retrieved.
* @return List of UserPermissions
* @throws Throwable on failure
*/
public static List<UserPermission> getUserPermissions(Connection connection, String tableRegex,
String userName) throws Throwable {
List<UserPermission> permList = new ArrayList<>();
try (Admin admin = connection.getAdmin()) {
if (tableRegex == null || tableRegex.isEmpty()) {
permList = admin.getUserPermissions(
GetUserPermissionsRequest.newBuilder().withUserName(userName).build());
} else if (tableRegex.charAt(0) == '@') { // Namespaces
String namespaceRegex = tableRegex.substring(1);
for (NamespaceDescriptor nsds : admin.listNamespaceDescriptors()) { // Read out all
// namespaces
String namespace = nsds.getName();
if (namespace.matches(namespaceRegex)) { // Match the given namespace regex?
permList.addAll(admin.getUserPermissions(
GetUserPermissionsRequest.newBuilder(namespace).withUserName(userName).build()));
}
}
} else { // Tables
List<TableDescriptor> htds = admin.listTableDescriptors(Pattern.compile(tableRegex), true);
for (TableDescriptor htd : htds) {
permList.addAll(admin.getUserPermissions(GetUserPermissionsRequest
.newBuilder(htd.getTableName()).withUserName(userName).build()));
}
}
}
return permList;
}
/**
* Allows subclasses to initialize the table information.
*
* @param connection The Connection to the HBase cluster. MUST be unmanaged. We will close.
* @param tableName The {@link TableName} of the table to process.
* @throws IOException
*/
protected void initializeTable(Connection connection, TableName tableName) throws IOException {
if (this.table != null || this.connection != null) {
LOG.warn("initializeTable called multiple times. Overwriting connection and table " +
"reference; TableInputFormatBase will not close these old references when done.");
}
this.table = connection.getTable(tableName);
this.regionLocator = connection.getRegionLocator(tableName);
this.admin = connection.getAdmin();
this.connection = connection;
}
private static void dropTable() {
Connection conn = BigtableConfiguration.connect(projectID, instanceID);
try {
Admin admin = conn.getAdmin();
admin.disableTable(TableName.valueOf(tableName));
admin.deleteTable(TableName.valueOf(tableName));
} catch (Exception ex) {
LOG.info("dropTable(): exception {} ", ex.toString());
}
}
@Test
public void testNamespaceSpaceAndRPCQuotaRemoved() throws Exception {
final Connection conn = TEST_UTIL.getConnection();
final Admin admin = conn.getAdmin();
final String ns = testName.getMethodName();
// Drop the ns if it somehow exists
if (namespaceExists(ns)) {
admin.deleteNamespace(ns);
}
// Create the ns
NamespaceDescriptor desc = NamespaceDescriptor.create(ns).build();
admin.createNamespace(desc);
assertEquals(0, getNumSpaceQuotas());
assertEquals(0, getThrottleQuotas());
// Set Both quotas
QuotaSettings settings =
QuotaSettingsFactory.limitNamespaceSpace(ns, 1024L, SpaceViolationPolicy.NO_INSERTS);
admin.setQuota(settings);
settings =
QuotaSettingsFactory.throttleNamespace(ns, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
admin.setQuota(settings);
assertEquals(1, getNumSpaceQuotas());
assertEquals(1, getThrottleQuotas());
// Remove Space quota
settings = QuotaSettingsFactory.removeNamespaceSpaceLimit(ns);
admin.setQuota(settings);
assertEquals(0, getNumSpaceQuotas());
assertEquals(1, getThrottleQuotas());
// Set back the space quota
settings = QuotaSettingsFactory.limitNamespaceSpace(ns, 1024L, SpaceViolationPolicy.NO_INSERTS);
admin.setQuota(settings);
assertEquals(1, getNumSpaceQuotas());
assertEquals(1, getThrottleQuotas());
// Remove the throttle quota
settings = QuotaSettingsFactory.unthrottleNamespace(ns);
admin.setQuota(settings);
assertEquals(1, getNumSpaceQuotas());
assertEquals(0, getThrottleQuotas());
// Set back the throttle quota
settings =
QuotaSettingsFactory.throttleNamespace(ns, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
admin.setQuota(settings);
assertEquals(1, getNumSpaceQuotas());
assertEquals(1, getThrottleQuotas());
// Delete the namespace and check that both the quotas have been dropped as well
admin.deleteNamespace(ns);
assertEquals(0, getNumSpaceQuotas());
assertEquals(0, getThrottleQuotas());
}
/**
* Computes size of each region for table and given column families.
* */
public HBaseRegionSizeCalculator(String tableName, Connection hbaseConnection) throws IOException {
Table table = null;
Admin admin = null;
try {
table = hbaseConnection.getTable(TableName.valueOf(tableName));
admin = hbaseConnection.getAdmin();
if (!enabled(table.getConfiguration())) {
logger.info("Region size calculation disabled.");
return;
}
logger.info("Calculating region sizes for table \"" + table.getName() + "\".");
// Get regions for table.
RegionLocator regionLocator = hbaseConnection.getRegionLocator(table.getName());
List<HRegionLocation> regionLocationList = regionLocator.getAllRegionLocations();
Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
for (HRegionLocation hRegionLocation : regionLocationList) {
tableRegions.add(hRegionLocation.getRegionInfo().getRegionName());
}
ClusterStatus clusterStatus = admin.getClusterStatus();
Collection<ServerName> servers = clusterStatus.getServers();
final long megaByte = 1024L * 1024L;
// Iterate all cluster regions, filter regions from our table and
// compute their size.
for (ServerName serverName : servers) {
ServerLoad serverLoad = clusterStatus.getLoad(serverName);
for (RegionLoad regionLoad : serverLoad.getRegionsLoad().values()) {
byte[] regionId = regionLoad.getName();
if (tableRegions.contains(regionId)) {
long regionSizeBytes = regionLoad.getStorefileSizeMB() * megaByte;
sizeMap.put(regionId, regionSizeBytes);
countMap.put(regionId, new Pair<>(regionLoad.getStores(), regionLoad.getStorefiles()));
if (regionSizeBytes == 0L) {
logger.info("Region " + regionLoad.getNameAsString() + " has size " + regionSizeBytes);
}
}
}
}
} finally {
IOUtils.closeQuietly(admin);
}
}
public static boolean isAccessControllerRunning(Connection connection)
throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
try (Admin admin = connection.getAdmin()) {
return admin.isTableAvailable(ACL_TABLE_NAME);
}
}
@BeforeClass
public static void setupBeforeClass() throws Exception {
MR_TEST_UTIL = new MRTestUtil(HBASE_TEST_UTILITY);
HBASE_TEST_UTILITY.startMiniCluster();
MR_TEST_UTIL.startMrCluster();
FileSystem fs = FileSystem.get(HBASE_TEST_UTILITY.getConfiguration());
int zkClientPort = HBASE_TEST_UTILITY.getZkCluster().getClientPort();
SOLR_TEST_UTILITY = new SolrTestingUtility(zkClientPort, NetUtils.getFreePort(),
ImmutableMap.of(
"solr.hdfs.blockcache.enabled", "false",
"solr.directoryFactory", "HdfsDirectoryFactory",
"solr.hdfs.home", fs.makeQualified(new Path("/solrdata")).toString()));
SOLR_TEST_UTILITY.start();
SOLR_TEST_UTILITY.uploadConfig("config1", new File(MINIMR_CONF_DIR, "conf"));
SOLR_TEST_UTILITY.createCollection("collection1", "config1", 2);
SOLR_TEST_UTILITY.createCollection("collection2", "config1", 2);
COLLECTION1 = new CloudSolrClient.Builder().withZkHost(SOLR_TEST_UTILITY.getZkConnectString()).build();
COLLECTION1.setDefaultCollection("collection1");
SOLR_ZK = "127.0.0.1:" + zkClientPort + "/solr";
INDEXER_ZK = "localhost:" + zkClientPort;
ZooKeeperItf zkItf = ZkUtil.connect(INDEXER_ZK, 15000);
INDEXER_MODEL = new IndexerModelImpl(zkItf, "/ngdata/hbaseindexer");
IndexerDefinition indexerDef = new IndexerDefinitionBuilder()
.name("zkindexerdef")
.indexerComponentFactory(DefaultIndexerComponentFactory.class.getName())
.configuration(Resources.toByteArray(Resources.getResource(
HBaseMapReduceIndexerToolGoLiveTest.class, "user_indexer.xml")))
.connectionParams(ImmutableMap.of(
"solr.zk", SOLR_ZK,
"solr.collection", "collection1"))
.build();
addAndWaitForIndexer(indexerDef);
Closer.close(zkItf);
HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf(TEST_TABLE_NAME));
tableDescriptor.addFamily(new HColumnDescriptor(TEST_COLFAM_NAME));
Connection connection = ConnectionFactory.createConnection(HBASE_TEST_UTILITY.getConfiguration());
HBASE_ADMIN = connection.getAdmin();
HBASE_ADMIN.createTable(tableDescriptor, new byte[][]{Bytes.toBytes("row0800"), Bytes.toBytes("row1600")});
RECORD_TABLE = connection.getTable(TableName.valueOf(TEST_TABLE_NAME));
for (int i = 0; i < RECORD_COUNT; i++) {
writeHBaseRecord(String.format("row%04d", i), ImmutableMap.of(
"firstname", String.format("John%04d", i),
"lastname", String.format("Doe%04d", i)));
}
}