下面列出了怎么用org.apache.hadoop.hbase.HRegionInfo的API类实例代码及写法,或者点击链接到github查看源代码。
/**
* If the table is spread across multiple region servers, then we parallelize the scan by making each region server a split.
*
* @see GlueMetadataHandler
*/
@Override
public GetSplitsResponse doGetSplits(BlockAllocator blockAllocator, GetSplitsRequest request)
throws IOException
{
Set<Split> splits = new HashSet<>();
//We can read each region in parallel
for (HRegionInfo info : getOrCreateConn(request).getTableRegions(HbaseSchemaUtils.getQualifiedTable(request.getTableName()))) {
Split.Builder splitBuilder = Split.newBuilder(makeSpillLocation(request), makeEncryptionKey())
.add(HBASE_CONN_STR, getConnStr(request))
.add(START_KEY_FIELD, new String(info.getStartKey()))
.add(END_KEY_FIELD, new String(info.getEndKey()))
.add(REGION_ID_FIELD, String.valueOf(info.getRegionId()))
.add(REGION_NAME_FIELD, info.getRegionNameAsString());
splits.add(splitBuilder.build());
}
return new GetSplitsResponse(request.getCatalogName(), splits, null);
}
private void clean(NavigableMap<HRegionInfo, ServerName> regionServerMap) throws InterruptedException, IOException {
long timestamp = System.currentTimeMillis();
Set<String> tableNameSet = tableNameSet(regionServerMap);
ExecutorService executorService = Executors.newFixedThreadPool(RegionLocationCleaner.THREAD_POOL_SIZE);
try {
for (String tableName : tableNameSet)
executorService.execute(
new RegionLocationCleaner(tableName, admin.getConfiguration(), regionServerMap));
} finally {
executorService.shutdown();
executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);
}
Util.printVerboseMessage(args, "TableInfo.clean", timestamp);
}
@Test
public void testMergeEmptyFast2() throws Exception {
makeTestData2();
List<HRegionInfo> regionInfoList;
// merge
String[] argsParam = {"zookeeper", tableName, "empty-FAST", "--force-proceed", "--test"};
Args args = new ManagerArgs(argsParam);
Merge command = new Merge(admin, args);
command.run();
// check
Thread.sleep(Constant.SMALL_WAIT_INTERVAL_MS);
regionInfoList = getRegionInfoList(tableName);
assertEquals(2, regionInfoList.size());
assertArrayEquals("".getBytes(), regionInfoList.get(0).getStartKey());
assertArrayEquals("a".getBytes(), regionInfoList.get(1).getStartKey());
}
@Test
public void testMergeEmpty6() throws Exception {
makeTestData6();
List<HRegionInfo> regionInfoList;
// merge
String[] argsParam = {"zookeeper", tableName, "empty", "--force-proceed", "--max-iteration=4", "--test"};
Args args = new ManagerArgs(argsParam);
Merge command = new Merge(admin, args);
command.run();
Thread.sleep(Constant.WAIT_INTERVAL_MS);
// check
Thread.sleep(Constant.SMALL_WAIT_INTERVAL_MS);
regionInfoList = getRegionInfoList(tableName);
assertEquals(1, regionInfoList.size());
}
@Test
public void testMergeEmptyFast3() throws Exception {
makeTestData3();
List<HRegionInfo> regionInfoList;
// merge
String[] argsParam = {"zookeeper", tableName, "empty-FAST", "--force-proceed", "--test"};
Args args = new ManagerArgs(argsParam);
Merge command = new Merge(admin, args);
command.run();
// check
Thread.sleep(Constant.SMALL_WAIT_INTERVAL_MS);
regionInfoList = getRegionInfoList(tableName);
assertEquals(3, regionInfoList.size());
assertArrayEquals("".getBytes(), regionInfoList.get(0).getStartKey());
assertArrayEquals("b".getBytes(), regionInfoList.get(1).getStartKey());
assertArrayEquals("c".getBytes(), regionInfoList.get(2).getStartKey());
}
@Test
public void testMergeEmptyFast5() throws Exception {
makeTestData5();
List<HRegionInfo> regionInfoList;
// merge
String[] argsParam = {"zookeeper", tableName, "empty-FAST", "--force-proceed", "--test"};
Args args = new ManagerArgs(argsParam);
Merge command = new Merge(admin, args);
command.run();
// check
Thread.sleep(Constant.SMALL_WAIT_INTERVAL_MS);
regionInfoList = getRegionInfoList(tableName);
assertEquals(5, regionInfoList.size());
}
@Test
public void testBalanceDefault() throws Exception {
splitTable("a".getBytes());
NavigableMap<HRegionInfo, ServerName> regionLocations;
List<Map.Entry<HRegionInfo, ServerName>> hRegionInfoList;
try (HTable table = getTable(tableName)) {
regionLocations = table.getRegionLocations();
hRegionInfoList = new ArrayList<>(regionLocations.entrySet());
Assert.assertEquals(2, regionLocations.size());
Assert.assertEquals(hRegionInfoList.get(0).getValue(), hRegionInfoList.get(1).getValue());
String[] argsParam = {"zookeeper", tableName, "default", "--force-proceed"};
Args args = new ManagerArgs(argsParam);
Assert.assertEquals("zookeeper", args.getZookeeperQuorum());
Balance command = new Balance(admin, args);
command.run();
regionLocations = table.getRegionLocations();
hRegionInfoList = new ArrayList<>(regionLocations.entrySet());
Assert.assertEquals(2, hRegionInfoList.size());
}
}
private void clean(NavigableMap<HRegionInfo, ServerName> regionServerMap) throws InterruptedException, IOException {
long timestamp = System.currentTimeMillis();
Set<String> tableNameSet = tableNameSet(regionServerMap);
ExecutorService executorService = Executors.newFixedThreadPool(RegionLocationCleaner.THREAD_POOL_SIZE);
try {
for (String tableName : tableNameSet)
executorService.execute(
new RegionLocationCleaner(tableName, admin.getConfiguration(), regionServerMap));
} finally {
executorService.shutdown();
executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);
}
Util.printVerboseMessage(args, "TableInfo.clean", timestamp);
}
protected void makeTestData7() throws Exception {
List<HRegionInfo> regionInfoList;
// create tables
String tableName2 = createAdditionalTable(TestBase.tableName + "2");
String tableName3 = createAdditionalTable(TestBase.tableName + "3");
// split tables
splitTable(tableName, "a".getBytes());
splitTable(tableName, "b".getBytes());
regionInfoList = getRegionInfoList(tableName);
assertEquals(3, regionInfoList.size());
splitTable(tableName2, "a".getBytes());
splitTable(tableName2, "b".getBytes());
regionInfoList = getRegionInfoList(tableName2);
assertEquals(3, regionInfoList.size());
splitTable(tableName3, "a".getBytes());
splitTable(tableName3, "b".getBytes());
regionInfoList = getRegionInfoList(tableName3);
assertEquals(3, regionInfoList.size());
}
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
HColumnDescriptor cfd = new HColumnDescriptor(family);
if (ttl > 0) {
cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
}
cfd.setMaxVersions(10);
htd.addFamily(cfd);
htd.addCoprocessor(TransactionProcessor.class.getName());
Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
FileSystem fs = FileSystem.get(conf);
assertTrue(fs.mkdirs(tablePath));
WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
WAL hLog = walFactory.getWAL(new byte[]{1}, null);
HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
return new HRegion(regionFS, hLog, conf, htd,
new LocalRegionServerServices(conf, ServerName.valueOf(
InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
@Test
public void testColumnStatsMerge() throws Exception {
// split a region
try (Admin admin = connection.getAdmin()) {
HRegionInfo regionInfo = regionInfoList.get(1);
admin.splitRegion(regionInfo.getEncodedNameAsBytes());
regionInfoList = admin.getTableRegions(hTableName);
long totalWaitingTime = 1000 * 60;
long waitUnit = 2000;
while (regionInfoList.size() == 2 && totalWaitingTime > 0) {
totalWaitingTime -= waitUnit;
Thread.sleep(waitUnit);
regionInfoList = admin.getTableRegions(hTableName);
}
PreparedStatement ps = methodWatcher.prepareStatement("explain select * from t a, t b where a.c2=b.c2");
ResultSet rs = ps.executeQuery();
int count = 0;
while (rs.next()) {
count++;
}
Assert.assertTrue(count > 0);
}
}
@Test
public void testMergeEmpty6() throws Exception {
makeTestData6();
List<HRegionInfo> regionInfoList;
// merge
String[] argsParam = {"zookeeper", tableName, "empty", "--force-proceed", "--max-iteration=4", "--test"};
Args args = new ManagerArgs(argsParam);
Merge command = new Merge(admin, args);
command.run();
Thread.sleep(Constant.WAIT_INTERVAL_MS);
// check
Thread.sleep(Constant.SMALL_WAIT_INTERVAL_MS);
regionInfoList = getRegionInfoList(tableName);
assertEquals(1, regionInfoList.size());
}
private static NavigableMap<HRegionInfo, ServerName> regionServerMap(Args args, Configuration conf,
HConnection connection, final String tableNameParam, final boolean offlined) throws IOException {
long timestamp = System.currentTimeMillis();
final NavigableMap<HRegionInfo, ServerName> regions = new TreeMap<>();
TableName tableName = TableName.valueOf(tableNameParam);
MetaScanner.MetaScannerVisitor visitor = new MetaScanner.TableMetaScannerVisitor(tableName) {
@Override
public boolean processRowInternal(Result rowResult) throws IOException {
HRegionInfo info = HRegionInfo.getHRegionInfo(rowResult);
ServerName serverName = HRegionInfo.getServerName(rowResult);
if (info.isOffline() && !offlined) return true;
regions.put(info, serverName);
return true;
}
};
MetaScanner.metaScan(conf, connection, visitor, tableName);
Util.printVerboseMessage(args, "CommandAdapter.regionServerMap", timestamp);
return regions;
}
@Test
public void testMergeEmpty6() throws Exception {
makeTestData6();
List<HRegionInfo> regionInfoList;
// merge
String[] argsParam = {"zookeeper", tableName, "empty", "--force-proceed", "--max-iteration=4", "--test"};
Args args = new ManagerArgs(argsParam);
Merge command = new Merge(admin, args);
command.run();
Thread.sleep(Constant.WAIT_INTERVAL_MS);
// check
Thread.sleep(Constant.SMALL_WAIT_INTERVAL_MS);
regionInfoList = getRegionInfoList(tableName);
assertEquals(1, regionInfoList.size());
}
@Test
public void testGetOnlineRegions() throws Exception {
createAdditionalTable(tableName + "2");
int regionCount = 0;
for (ServerName serverName : admin.getClusterStatus().getServers()) {
for (HRegionInfo hRegionInfo : admin.getOnlineRegions(serverName)) {
System.out.println(hRegionInfo);
regionCount++;
}
}
if (miniCluster) {
if (securedCluster) {
Assert.assertEquals(5, regionCount);
} else {
Assert.assertEquals(4, regionCount);
}
}
}
protected SortedSet<byte[]> getTransactionalRegions() throws IOException {
SortedSet<byte[]> regions = new TreeSet<>(Bytes.BYTES_COMPARATOR);
try (Admin admin = connection.getAdmin()) {
HTableDescriptor[] tableDescriptors = admin.listTables();
LOG.debug("Got {} tables to process", tableDescriptors == null ? 0 : tableDescriptors.length);
if (tableDescriptors != null) {
for (HTableDescriptor tableDescriptor : tableDescriptors) {
if (isTransactionalTable(tableDescriptor)) {
List<HRegionInfo> tableRegions = admin.getTableRegions(tableDescriptor.getTableName());
LOG.debug("Regions for table {}: {}", tableDescriptor.getTableName(), tableRegions);
if (tableRegions != null) {
for (HRegionInfo region : tableRegions) {
regions.add(region.getRegionName());
}
}
} else {
LOG.debug("{} is not a transactional table", tableDescriptor.getTableName());
}
}
}
}
return regions;
}
@Test
public void testMergeEmptyFast6() throws Exception {
makeTestData6();
List<HRegionInfo> regionInfoList;
// merge
String[] argsParam = {"zookeeper", tableName, "empty-FAST", "--force-proceed", "--test"
, "--verbose", "--debug"};
Args args = new ManagerArgs(argsParam);
Merge command = new Merge(admin, args);
command.run();
// check
Thread.sleep(Constant.SMALL_WAIT_INTERVAL_MS);
regionInfoList = getRegionInfoList(tableName);
assertEquals(1, regionInfoList.size());
}
/**
* Ensures each region of SYSTEM.CATALOG is on a different region server
*/
private static void moveRegion(HRegionInfo regionInfo, ServerName srcServerName, ServerName dstServerName) throws Exception {
Admin admin = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin();
HBaseTestingUtility util = getUtility();
MiniHBaseCluster cluster = util.getHBaseCluster();
HMaster master = cluster.getMaster();
AssignmentManager am = master.getAssignmentManager();
HRegionServer dstServer = util.getHBaseCluster().getRegionServer(dstServerName);
HRegionServer srcServer = util.getHBaseCluster().getRegionServer(srcServerName);
byte[] encodedRegionNameInBytes = regionInfo.getEncodedNameAsBytes();
admin.move(encodedRegionNameInBytes, Bytes.toBytes(dstServer.getServerName().getServerName()));
while (dstServer.getOnlineRegion(regionInfo.getRegionName()) == null
|| dstServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameInBytes)
|| srcServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameInBytes)) {
// wait for the move to be finished
Thread.sleep(100);
}
}
protected void makeTestData7() throws Exception {
List<HRegionInfo> regionInfoList;
// create tables
String tableName2 = createAdditionalTable(TestBase.tableName + "2");
String tableName3 = createAdditionalTable(TestBase.tableName + "3");
// split tables
splitTable(tableName, "a".getBytes());
splitTable(tableName, "b".getBytes());
regionInfoList = getRegionInfoList(tableName);
assertEquals(3, regionInfoList.size());
splitTable(tableName2, "a".getBytes());
splitTable(tableName2, "b".getBytes());
regionInfoList = getRegionInfoList(tableName2);
assertEquals(3, regionInfoList.size());
splitTable(tableName3, "a".getBytes());
splitTable(tableName3, "b".getBytes());
regionInfoList = getRegionInfoList(tableName3);
assertEquals(3, regionInfoList.size());
}
public static List<RegionPlan> makePlan(HBaseAdmin admin, List<RegionPlan> newRegionPlan) throws IOException {
// snapshot current region assignment
Map<HRegionInfo, ServerName> regionAssignmentMap = createRegionAssignmentMap(admin);
// update with new plan
for (RegionPlan regionPlan : newRegionPlan) {
regionAssignmentMap.put(regionPlan.getRegionInfo(), regionPlan.getDestination());
}
Map<ServerName, List<HRegionInfo>> clusterState = initializeRegionMap(admin);
for (Map.Entry<HRegionInfo, ServerName> entry : regionAssignmentMap.entrySet())
clusterState.get(entry.getValue()).add(entry.getKey());
StochasticLoadBalancer balancer = new StochasticLoadBalancer();
Configuration conf = admin.getConfiguration();
conf.setFloat("hbase.regions.slop", 0.2f);
balancer.setConf(conf);
return balancer.balanceCluster(clusterState);
}
@Test
public void testMergeEmptyFast3() throws Exception {
makeTestData3();
List<HRegionInfo> regionInfoList;
// merge
String[] argsParam = {"zookeeper", tableName, "empty-FAST", "--force-proceed", "--test"};
Args args = new ManagerArgs(argsParam);
Merge command = new Merge(admin, args);
command.run();
// check
Thread.sleep(Constant.SMALL_WAIT_INTERVAL_MS);
regionInfoList = getRegionInfoList(tableName);
assertEquals(3, regionInfoList.size());
assertArrayEquals("".getBytes(), regionInfoList.get(0).getStartKey());
assertArrayEquals("b".getBytes(), regionInfoList.get(1).getStartKey());
assertArrayEquals("c".getBytes(), regionInfoList.get(2).getStartKey());
}
@Test
public void testMergeEmpty2() throws Exception {
makeTestData2();
List<HRegionInfo> regionInfoList;
// merge
String[] argsParam = {"zookeeper", tableName, "empty", "--force-proceed"};
Args args = new ManagerArgs(argsParam);
Merge command = new Merge(admin, args);
command.setTest(true);
command.run();
Thread.sleep(Constant.WAIT_INTERVAL_MS);
// check
Thread.sleep(Constant.SMALL_WAIT_INTERVAL_MS);
regionInfoList = getRegionInfoList(tableName);
assertEquals(1, regionInfoList.size());
assertArrayEquals("".getBytes(), regionInfoList.get(0).getStartKey());
}
public static boolean isReallyEmptyRegion(HConnection connection,
String tableName, HRegionInfo regionInfo) throws IOException {
boolean emptyRegion = false;
// verify really empty region by scanning records
try (HTableInterface table = connection.getTable(tableName)) {
Scan scan = new Scan(regionInfo.getStartKey(), regionInfo.getEndKey());
FilterList filterList = new FilterList();
filterList.addFilter(new KeyOnlyFilter());
filterList.addFilter(new FirstKeyOnlyFilter());
scan.setFilter(filterList);
scan.setCacheBlocks(false);
scan.setSmall(true);
scan.setCaching(1);
try (ResultScanner scanner = table.getScanner(scan)) {
if (scanner.next() == null) emptyRegion = true;
}
}
return emptyRegion;
}
@Test
public void testSplitWithDecimalString() throws Exception {
List<HRegionInfo> regionInfoList;
regionInfoList = getRegionInfoList(tableName);
assertEquals(1, regionInfoList.size());
String[] argsParam = {"zookeeper", tableName, "Rule", "decimalString", "3", "10", "--force-proceed"};
Args args = new ManagerArgs(argsParam);
assertEquals("zookeeper", args.getZookeeperQuorum());
Split command = new Split(admin, args);
command.run();
waitForSplitting(3);
regionInfoList = getRegionInfoList(tableName);
assertEquals(3, regionInfoList.size());
for (HRegionInfo hRegionInfo : regionInfoList) {
byte[] startKey = hRegionInfo.getStartKey();
if (startKey.length > 0) {
assertTrue(Bytes.toString(startKey).matches("[0-9]*"));
assertFalse(Bytes.toString(startKey).matches("[A-Za-z]*"));
}
}
}
public static boolean isReallyEmptyRegion(HConnection connection,
String tableName, HRegionInfo regionInfo) throws IOException {
boolean emptyRegion = false;
// verify really empty region by scanning records
try (HTableInterface table = connection.getTable(tableName)) {
Scan scan = new Scan(regionInfo.getStartKey(), regionInfo.getEndKey());
FilterList filterList = new FilterList();
filterList.addFilter(new KeyOnlyFilter());
filterList.addFilter(new FirstKeyOnlyFilter());
scan.setFilter(filterList);
scan.setCacheBlocks(false);
scan.setSmall(true);
scan.setCaching(1);
try (ResultScanner scanner = table.getScanner(scan)) {
if (scanner.next() == null) emptyRegion = true;
}
}
return emptyRegion;
}
@Test
public void testSplitWithHexStringDowncaseRule() throws Exception {
List<HRegionInfo> regionInfoList;
regionInfoList = getRegionInfoList(tableName);
assertEquals(1, regionInfoList.size());
String[] argsParam = {"zookeeper", tableName, "Rule", "hexString_down", "5", "--force-proceed"};
Args args = new ManagerArgs(argsParam);
assertEquals("zookeeper", args.getZookeeperQuorum());
Split command = new Split(admin, args);
command.run();
waitForSplitting(5);
regionInfoList = getRegionInfoList(tableName);
assertEquals(5, regionInfoList.size());
for (HRegionInfo hRegionInfo : regionInfoList) {
byte[] startKey = hRegionInfo.getStartKey();
if (startKey.length > 0) {
assertTrue(Bytes.toString(startKey).matches("[a-z0-9]*"));
assertFalse(Bytes.toString(startKey).matches("[A-Z]*"));
}
}
}
@SuppressWarnings("UnusedParameters")
private static NavigableMap<HRegionInfo, ServerName> regionServerMap(Args args, Configuration conf, HConnection connection, final String tableNameParam, final boolean offlined) throws IOException {
long timestamp = System.currentTimeMillis();
final NavigableMap<HRegionInfo, ServerName> regions = new TreeMap<>();
TableName tableName = TableName.valueOf(tableNameParam);
MetaScanner.MetaScannerVisitor visitor = new MetaScanner.TableMetaScannerVisitor(tableName) {
@Override
public boolean processRowInternal(Result rowResult) throws IOException {
HRegionInfo info = HRegionInfo.getHRegionInfo(rowResult);
ServerName serverName = HRegionInfo.getServerName(rowResult);
if (info.isOffline() && !offlined) return true;
regions.put(info, serverName);
return true;
}
};
MetaScanner.metaScan(connection, visitor, tableName);
Util.printVerboseMessage(args, "CommandAdapter.allTableRegions", timestamp);
return regions;
}
@Test
public void testMergeEmpty4() throws Exception {
makeTestData4();
List<HRegionInfo> regionInfoList;
// merge
String[] argsParam = {"zookeeper", tableName, "empty", "--force-proceed", "--test"};
Args args = new ManagerArgs(argsParam);
Merge command = new Merge(admin, args);
command.setTest(true);
command.run();
Thread.sleep(Constant.WAIT_INTERVAL_MS);
// check
Thread.sleep(Constant.SMALL_WAIT_INTERVAL_MS);
regionInfoList = getRegionInfoList(tableName);
assertEquals(1, regionInfoList.size());
assertArrayEquals("".getBytes(), regionInfoList.get(0).getStartKey());
}
private boolean isRegionBoundaryOfPhoenixSaltingTable(HRegionInfo regionInfo) {
byte[] endKey = regionInfo.getEndKey();
boolean boundaryRegionForPhoenix = true;
if (endKey.length > 0) {
for (int i = 1, limit = endKey.length; i < limit; i++) {
if (endKey[i] != 0) {
boundaryRegionForPhoenix = false;
break;
}
}
}
if (boundaryRegionForPhoenix) {
Util.printVerboseMessage(args, regionInfo.getEncodedName() + " is boundary region of phoenix : "
+ Bytes.toStringBinary(regionInfo.getStartKey()) + " ~ " + Bytes.toStringBinary(regionInfo.getEndKey()));
}
return boundaryRegionForPhoenix;
}
@Override
public void regionOffline(HRegionInfo regionInfo) {
TableName tableName = regionInfo.getTable();
synchronized (this.colocationInfo) {
Map<ImmutableBytesWritable, ServerName> tableKeys = this.colocationInfo.get(tableName);
if (null == tableKeys) {
if (LOG.isDebugEnabled()) {
LOG.debug("No regions of table " + tableName + " in the colocationInfo.");
}
} else {
tableKeys.remove(new ImmutableBytesWritable(regionInfo.getStartKey()));
if (LOG.isDebugEnabled()) {
LOG.debug("The regioninfo " + regionInfo + " removed from the colocationInfo");
}
}
}
}