下面列出了怎么用org.apache.hadoop.hbase.util.PairOfSameType的API类实例代码及写法,或者点击链接到github查看源代码。
@Override
public PairOfSameType<Integer> getRegionCounts() {
try {
if (!master.isInitialized()) {
return new PairOfSameType<>(0, 0);
}
Integer onlineRegionCount = 0;
Integer offlineRegionCount = 0;
List<TableDescriptor> descriptors = master.listTableDescriptors(null, null,
null, false);
for (TableDescriptor htDesc : descriptors) {
TableName tableName = htDesc.getTableName();
Map<RegionState.State, List<RegionInfo>> tableRegions =
master.getAssignmentManager().getRegionStates()
.getRegionByStateOfTable(tableName);
onlineRegionCount += tableRegions.get(RegionState.State.OPEN).size();
offlineRegionCount += tableRegions.get(RegionState.State.OFFLINE).size();
}
return new PairOfSameType<>(onlineRegionCount, offlineRegionCount);
} catch (IOException e) {
return new PairOfSameType<>(0, 0);
}
}
/**
* Splits a region
* @param r Region to split.
* @return List of region locations
*/
private List<HRegionLocation> splitRegion(final RegionInfo r)
throws IOException, InterruptedException, ExecutionException {
List<HRegionLocation> locations = new ArrayList<>();
// Split this table in two.
Admin admin = TEST_UTIL.getAdmin();
Connection connection = TEST_UTIL.getConnection();
admin.splitRegionAsync(r.getEncodedNameAsBytes()).get();
admin.close();
PairOfSameType<RegionInfo> regions = waitOnDaughters(r);
if (regions != null) {
try (RegionLocator rl = connection.getRegionLocator(r.getTable())) {
locations.add(rl.getRegionLocation(regions.getFirst().getEncodedNameAsBytes()));
locations.add(rl.getRegionLocation(regions.getSecond().getEncodedNameAsBytes()));
}
return locations;
}
return locations;
}
public PairOfSameType<HRegion> stepsAfterPONR(final Server server,
final RegionServerServices services, PairOfSameType<HRegion> regions)
throws IOException {
openDaughters(server, services, regions.getFirst(), regions.getSecond());
transitionZKNode(server, services, regions.getFirst(), regions.getSecond());
return regions;
}
/**
* If daughters no longer hold reference to the parents, delete the parent.
* @param parent RegionInfo of split offlined parent
* @param rowContent Content of <code>parent</code> row in
* <code>metaRegionName</code>
* @return True if we removed <code>parent</code> from meta table and from
* the filesystem.
*/
boolean cleanParent(final RegionInfo parent, Result rowContent)
throws IOException {
// Check whether it is a merged region and if it is clean of references.
if (MetaTableAccessor.hasMergeRegions(rowContent.rawCells())) {
// Wait until clean of merge parent regions first
return false;
}
// Run checks on each daughter split.
PairOfSameType<RegionInfo> daughters = MetaTableAccessor.getDaughterRegions(rowContent);
Pair<Boolean, Boolean> a = checkDaughterInFs(parent, daughters.getFirst());
Pair<Boolean, Boolean> b = checkDaughterInFs(parent, daughters.getSecond());
if (hasNoReferences(a) && hasNoReferences(b)) {
String daughterA = daughters.getFirst() != null?
daughters.getFirst().getShortNameToLog(): "null";
String daughterB = daughters.getSecond() != null?
daughters.getSecond().getShortNameToLog(): "null";
LOG.debug("Deleting region " + parent.getShortNameToLog() +
" because daughters -- " + daughterA + ", " + daughterB +
" -- no longer hold references");
ProcedureExecutor<MasterProcedureEnv> pe = this.services.getMasterProcedureExecutor();
pe.submitProcedure(new GCRegionProcedure(pe.getEnvironment(), parent));
// Remove from in-memory states
this.services.getAssignmentManager().getRegionStates().deleteRegion(parent);
this.services.getServerManager().removeRegion(parent);
return true;
}
return false;
}
private RegionReplicaInfo(final Result result, final HRegionLocation location) {
this.row = result != null ? result.getRow() : null;
this.regionInfo = location != null ? location.getRegion() : null;
this.regionState = (result != null && regionInfo != null)
? RegionStateStore.getRegionState(result, regionInfo)
: null;
this.serverName = location != null ? location.getServerName() : null;
this.seqNum = (location != null) ? location.getSeqNum() : HConstants.NO_SEQNUM;
this.targetServerName = (result != null && regionInfo != null)
? MetaTableAccessor.getTargetServerName(result, regionInfo.getReplicaId())
: null;
this.mergeRegionInfo = (result != null)
? MetaTableAccessor.getMergeRegionsWithName(result.rawCells())
: null;
if (result != null) {
PairOfSameType<RegionInfo> daughterRegions = MetaTableAccessor.getDaughterRegions(result);
this.splitRegionInfo = new LinkedHashMap<>();
if (daughterRegions.getFirst() != null) {
splitRegionInfo.put(HConstants.SPLITA_QUALIFIER_STR, daughterRegions.getFirst());
}
if (daughterRegions.getSecond() != null) {
splitRegionInfo.put(HConstants.SPLITB_QUALIFIER_STR, daughterRegions.getSecond());
}
} else {
this.splitRegionInfo = null;
}
}
private PairOfSameType<RegionInfo> waitOnDaughters(final RegionInfo r)
throws IOException {
long start = System.currentTimeMillis();
PairOfSameType<RegionInfo> pair = null;
try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
Table metaTable = conn.getTable(TableName.META_TABLE_NAME)) {
Result result = null;
RegionInfo region = null;
while ((System.currentTimeMillis() - start) < 60000) {
result = metaTable.get(new Get(r.getRegionName()));
if (result == null) {
break;
}
region = CatalogFamilyFormat.getRegionInfo(result);
if (region.isSplitParent()) {
LOG.debug(region.toString() + " IS a parent!");
pair = MetaTableAccessor.getDaughterRegions(result);
break;
}
Threads.sleep(100);
}
if (pair.getFirst() == null || pair.getSecond() == null) {
throw new IOException("Failed to get daughters, for parent region: " + r);
}
return pair;
}
}
@Test
public void testWholesomeMerge() throws Exception {
LOG.info("Starting " + name.getMethodName());
final TableName tableName = TableName.valueOf(name.getMethodName());
try {
// Create table and load data.
Table table = createTableAndLoadData(MASTER, tableName);
// Merge 1st and 2nd region
mergeRegionsAndVerifyRegionNum(MASTER, tableName, 0, 1, INITIAL_REGION_NUM - 1);
// Merge 2nd and 3th region
PairOfSameType<RegionInfo> mergedRegions =
mergeRegionsAndVerifyRegionNum(MASTER, tableName, 1, 2, INITIAL_REGION_NUM - 2);
verifyRowCount(table, ROWSIZE);
// Randomly choose one of the two merged regions
RegionInfo hri = RandomUtils.nextBoolean() ? mergedRegions.getFirst() : mergedRegions.getSecond();
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
AssignmentManager am = cluster.getMaster().getAssignmentManager();
RegionStates regionStates = am.getRegionStates();
// We should not be able to assign it again
am.assign(hri);
assertFalse("Merged region can't be assigned", regionStates.isRegionInTransition(hri));
// We should not be able to unassign it either
am.unassign(hri);
assertFalse("Merged region can't be unassigned", regionStates.isRegionInTransition(hri));
table.close();
} finally {
TEST_UTIL.deleteTable(tableName);
}
}
private PairOfSameType<RegionInfo> mergeRegionsAndVerifyRegionNum(
HMaster master, TableName tablename,
int regionAnum, int regionBnum, int expectedRegionNum) throws Exception {
PairOfSameType<RegionInfo> mergedRegions =
requestMergeRegion(master, tablename, regionAnum, regionBnum);
waitAndVerifyRegionNum(master, tablename, expectedRegionNum);
return mergedRegions;
}
private PairOfSameType<RegionInfo> requestMergeRegion(
HMaster master, TableName tablename,
int regionAnum, int regionBnum) throws Exception {
List<Pair<RegionInfo, ServerName>> tableRegions = MetaTableAccessor
.getTableRegionsAndLocations(
TEST_UTIL.getConnection(), tablename);
RegionInfo regionA = tableRegions.get(regionAnum).getFirst();
RegionInfo regionB = tableRegions.get(regionBnum).getFirst();
ADMIN.mergeRegionsAsync(
regionA.getEncodedNameAsBytes(),
regionB.getEncodedNameAsBytes(), false);
return new PairOfSameType<>(regionA, regionB);
}
/**
* Prepare the regions and region files.
* @param server Hosting server instance. Can be null when testing (won't try
* and update in zk if a null server)
* @param services Used to online/offline regions.
* @throws IOException If thrown, transaction failed.
* Call {@link #rollback(Server, RegionServerServices)}
* @return Regions created
*/
/* package */PairOfSameType<HRegion> createDaughters(final Server server,
final RegionServerServices services) throws IOException {
LOG.info("Starting split of region " + this.parent);
if ((server != null && server.isStopped()) ||
(services != null && services.isStopping())) {
throw new IOException("Server is stopped or stopping");
}
assert !this.parent.lock.writeLock().isHeldByCurrentThread():
"Unsafe to hold write lock while performing RPCs";
// Coprocessor callback
if (this.parent.getCoprocessorHost() != null) {
this.parent.getCoprocessorHost().preSplit();
}
// Coprocessor callback
if (this.parent.getCoprocessorHost() != null) {
this.parent.getCoprocessorHost().preSplit(this.splitrow);
}
// If true, no cluster to write meta edits to or to update znodes in.
boolean testing = server == null? true:
server.getConfiguration().getBoolean("hbase.testing.nocluster", false);
this.fileSplitTimeout = testing ? this.fileSplitTimeout :
server.getConfiguration().getLong("hbase.regionserver.fileSplitTimeout",
this.fileSplitTimeout);
PairOfSameType<HRegion> daughterRegions = stepsBeforePONR(server, services, testing);
List<Mutation> metaEntries = new ArrayList<Mutation>();
if (this.parent.getCoprocessorHost() != null) {
if (this.parent.getCoprocessorHost().
preSplitBeforePONR(this.splitrow, metaEntries)) {
throw new IOException("Coprocessor bypassing region "
+ this.parent.getRegionNameAsString() + " split.");
}
try {
for (Mutation p : metaEntries) {
HRegionInfo.parseRegionName(p.getRow());
}
} catch (IOException e) {
LOG.error("Row key of mutation from coprossor is not parsable as region name."
+ "Mutations from coprocessor should only for hbase:meta table.");
throw e;
}
}
// This is the point of no return. Adding subsequent edits to .META. as we
// do below when we do the daughter opens adding each to .META. can fail in
// various interesting ways the most interesting of which is a timeout
// BUT the edits all go through (See HBASE-3872). IF we reach the PONR
// then subsequent failures need to crash out this regionserver; the
// server shutdown processing should be able to fix-up the incomplete split.
// The offlined parent will have the daughters as extra columns. If
// we leave the daughter regions in place and do not remove them when we
// crash out, then they will have their references to the parent in place
// still and the server shutdown fixup of .META. will point to these
// regions.
// We should add PONR JournalEntry before offlineParentInMeta,so even if
// OfflineParentInMeta timeout,this will cause regionserver exit,and then
// master ServerShutdownHandler will fix daughter & avoid data loss. (See
// HBase-4562).
this.journal.add(JournalEntry.PONR);
// Edit parent in meta. Offlines parent region and adds splita and splitb
// as an atomic update. See HBASE-7721. This update to META makes the region
// will determine whether the region is split or not in case of failures.
// If it is successful, master will roll-forward, if not, master will rollback
// and assign the parent region.
if (!testing) {
if (metaEntries == null || metaEntries.isEmpty()) {
MetaTableAccessor.splitRegion(server.getConnection(), parent.getRegionInfo(),
daughterRegions.getFirst().getRegionInfo(),
daughterRegions.getSecond().getRegionInfo(), server.getServerName(), 1);
} else {
offlineParentInMetaAndputMetaEntries(server.getConnection(),
parent.getRegionInfo(), daughterRegions.getFirst().getRegionInfo(), daughterRegions
.getSecond().getRegionInfo(), server.getServerName(), metaEntries);
}
}
return daughterRegions;
}
/**
* Returns the daughter regions by reading the corresponding columns of the catalog table Result.
* @param data a Result object from the catalog table scan
* @return pair of RegionInfo or PairOfSameType(null, null) if region is not a split parent
*/
public static PairOfSameType<RegionInfo> getDaughterRegions(Result data) {
RegionInfo splitA = CatalogFamilyFormat.getRegionInfo(data, HConstants.SPLITA_QUALIFIER);
RegionInfo splitB = CatalogFamilyFormat.getRegionInfo(data, HConstants.SPLITB_QUALIFIER);
return new PairOfSameType<>(splitA, splitB);
}
/**
* Blocks until the region split is complete in hbase:meta and region server opens the daughters
*/
public static void blockUntilRegionSplit(Configuration conf, long timeout,
final byte[] regionName, boolean waitForDaughters) throws IOException, InterruptedException {
long start = System.currentTimeMillis();
log("blocking until region is split:" + Bytes.toStringBinary(regionName));
RegionInfo daughterA = null, daughterB = null;
try (Connection conn = ConnectionFactory.createConnection(conf);
Table metaTable = conn.getTable(TableName.META_TABLE_NAME)) {
Result result = null;
RegionInfo region = null;
while ((System.currentTimeMillis() - start) < timeout) {
result = metaTable.get(new Get(regionName));
if (result == null) {
break;
}
region = CatalogFamilyFormat.getRegionInfo(result);
if (region.isSplitParent()) {
log("found parent region: " + region.toString());
PairOfSameType<RegionInfo> pair = MetaTableAccessor.getDaughterRegions(result);
daughterA = pair.getFirst();
daughterB = pair.getSecond();
break;
}
Threads.sleep(100);
}
if (daughterA == null || daughterB == null) {
throw new IOException("Failed to get daughters, daughterA=" + daughterA + ", daughterB=" +
daughterB + ", timeout=" + timeout + ", result=" + result + ", regionName=" +
Bytes.toString(regionName) + ", region=" + region);
}
// if we are here, this means the region split is complete or timed out
if (waitForDaughters) {
long rem = timeout - (System.currentTimeMillis() - start);
blockUntilRegionIsInMeta(conn, rem, daughterA);
rem = timeout - (System.currentTimeMillis() - start);
blockUntilRegionIsInMeta(conn, rem, daughterB);
rem = timeout - (System.currentTimeMillis() - start);
blockUntilRegionIsOpened(conf, rem, daughterA);
rem = timeout - (System.currentTimeMillis() - start);
blockUntilRegionIsOpened(conf, rem, daughterB);
// Compacting the new region to make sure references can be cleaned up
compactAndBlockUntilDone(TEST_UTIL.getAdmin(),
TEST_UTIL.getMiniHBaseCluster().getRegionServer(0), daughterA.getRegionName());
compactAndBlockUntilDone(TEST_UTIL.getAdmin(),
TEST_UTIL.getMiniHBaseCluster().getRegionServer(0), daughterB.getRegionName());
removeCompactedFiles(conn, timeout, daughterA);
removeCompactedFiles(conn, timeout, daughterB);
}
}
}
@Override
public void getMetrics(MetricsCollector metricsCollector, boolean all) {
MetricsRecordBuilder metricsRecordBuilder = metricsCollector.addRecord(metricsName);
// masterWrapper can be null because this function is called inside of init.
// If the master is already stopped or has initiated a shutdown, no point in registering the
// metrics again.
if (masterWrapper != null && masterWrapper.isRunning()) {
// Pair<online region number, offline region number>
PairOfSameType<Integer> regionNumberPair = masterWrapper.getRegionCounts();
metricsRecordBuilder
.addGauge(Interns.info(MERGE_PLAN_COUNT_NAME, MERGE_PLAN_COUNT_DESC),
masterWrapper.getMergePlanCount())
.addGauge(Interns.info(SPLIT_PLAN_COUNT_NAME, SPLIT_PLAN_COUNT_DESC),
masterWrapper.getSplitPlanCount())
.addGauge(Interns.info(MASTER_ACTIVE_TIME_NAME,
MASTER_ACTIVE_TIME_DESC), masterWrapper.getActiveTime())
.addGauge(Interns.info(MASTER_START_TIME_NAME,
MASTER_START_TIME_DESC), masterWrapper.getStartTime())
.addGauge(Interns.info(MASTER_FINISHED_INITIALIZATION_TIME_NAME,
MASTER_FINISHED_INITIALIZATION_TIME_DESC),
masterWrapper.getMasterInitializationTime())
.addGauge(Interns.info(AVERAGE_LOAD_NAME, AVERAGE_LOAD_DESC),
masterWrapper.getAverageLoad())
.addGauge(Interns.info(ONLINE_REGION_COUNT_NAME, ONLINE_REGION_COUNT_DESC),
regionNumberPair.getFirst())
.addGauge(Interns.info(OFFLINE_REGION_COUNT_NAME, OFFLINE_REGION_COUNT_DESC),
regionNumberPair.getSecond())
.tag(Interns.info(LIVE_REGION_SERVERS_NAME, LIVE_REGION_SERVERS_DESC),
masterWrapper.getRegionServers())
.addGauge(Interns.info(NUM_REGION_SERVERS_NAME,
NUMBER_OF_REGION_SERVERS_DESC), masterWrapper.getNumRegionServers())
.tag(Interns.info(DEAD_REGION_SERVERS_NAME, DEAD_REGION_SERVERS_DESC),
masterWrapper.getDeadRegionServers())
.addGauge(Interns.info(NUM_DEAD_REGION_SERVERS_NAME,
NUMBER_OF_DEAD_REGION_SERVERS_DESC),
masterWrapper.getNumDeadRegionServers())
.tag(Interns.info(ZOOKEEPER_QUORUM_NAME, ZOOKEEPER_QUORUM_DESC),
masterWrapper.getZookeeperQuorum())
.tag(Interns.info(SERVER_NAME_NAME, SERVER_NAME_DESC), masterWrapper.getServerName())
.tag(Interns.info(CLUSTER_ID_NAME, CLUSTER_ID_DESC), masterWrapper.getClusterId())
.tag(Interns.info(IS_ACTIVE_MASTER_NAME, IS_ACTIVE_MASTER_DESC),
String.valueOf(masterWrapper.getIsActiveMaster()));
}
metricsRegistry.snapshot(metricsRecordBuilder, all);
if(metricsAdapter != null) {
metricsAdapter.snapshotAllMetrics(registry, metricsRecordBuilder);
}
}
/**
* Run the transaction.
* @param server Hosting server instance. Can be null when testing (won't try
* and update in zk if a null server)
* @param services Used to online/offline regions.
* @throws IOException If thrown, transaction failed.
* Call {@link #rollback(Server, RegionServerServices)}
* @return Regions created
* @throws IOException
* @see #rollback(Server, RegionServerServices)
*/
public PairOfSameType<HRegion> execute(final Server server,
final RegionServerServices services)
throws IOException {
PairOfSameType<HRegion> regions = createDaughters(server, services);
if (this.parent.getCoprocessorHost() != null) {
this.parent.getCoprocessorHost().preSplitAfterPONR();
}
return stepsAfterPONR(server, services, regions);
}
/**
* Returns the daughter regions by reading the corresponding columns of the catalog table
* Result.
* (Copied from MetaTableAccessor)
* @param data a Result object from the catalog table scan
* @return pair of RegionInfo or PairOfSameType(null, null) if region is not a split parent
*/
public static PairOfSameType<RegionInfo> getDaughterRegions(Result data) {
RegionInfo splitA = getRegionInfo(data, HConstants.SPLITA_QUALIFIER);
RegionInfo splitB = getRegionInfo(data, HConstants.SPLITB_QUALIFIER);
return new PairOfSameType<>(splitA, splitB);
}
/**
* Get the online and offline region counts
*
* @return pair of count for online regions and offline regions
*/
PairOfSameType<Integer> getRegionCounts();