下面列出了怎么用org.apache.hadoop.hbase.ServerName的API类实例代码及写法,或者点击链接到github查看源代码。
Set<Integer> getServerIndexes(Args args) {
if (indexRSs == null) {
indexRSs = new HashSet<>();
if (args.has(Args.OPTION_REGION_SERVER)) {
Object arg = args.valueOf(Args.OPTION_REGION_SERVER);
if (arg != null) {
int i = 0;
for (ServerName serverName : serverNameSet) {
if (serverName.getServerName().matches((String) arg)) {
indexRSs.add(i);
}
i++;
}
if (indexRSs.size() == 0)
throw new IllegalStateException(arg + " is invalid");
}
}
}
return indexRSs;
}
private void validateST2(HTable table1) throws IOException {
NavigableMap<HRegionInfo, ServerName> regionLocations;
List<Map.Entry<HRegionInfo, ServerName>> hRegionInfoList;
regionLocations = table1.getRegionLocations();
hRegionInfoList = new ArrayList<>(regionLocations.entrySet());
Map<ServerName, Integer> serverCountMap = new HashMap<>();
for (Map.Entry<HRegionInfo, ServerName> entry : hRegionInfoList) {
if (serverCountMap.get(entry.getValue()) == null) {
serverCountMap.put(entry.getValue(), 1);
} else {
serverCountMap.put(entry.getValue(), serverCountMap.get(entry.getValue()) + 1);
}
}
Assert.assertEquals(Math.min(getServerNameList().size(), regionLocations.size()), serverCountMap.size());
int regionCount;
regionCount = 0;
for (ServerName serverName : getServerNameList()) {
List<HRegionInfo> regionInfoList = getRegionInfoList(serverName, Bytes.toString(table1.getTableName()));
Assert.assertNotEquals(4, regionInfoList.size());
Assert.assertEquals(2, regionInfoList.size());
regionCount += regionInfoList.size();
}
Assert.assertEquals(4, regionCount);
}
/**
* Create round-robin assigns. Use on table creation to distribute out regions across cluster.
* @return AssignProcedures made out of the passed in <code>hris</code> and a call to the balancer
* to populate the assigns with targets chosen using round-robin (default balancer
* scheme). If at assign-time, the target chosen is no longer up, thats fine, the
* AssignProcedure will ask the balancer for a new target, and so on.
*/
public TransitRegionStateProcedure[] createRoundRobinAssignProcedures(List<RegionInfo> hris,
List<ServerName> serversToExclude) {
if (hris.isEmpty()) {
return new TransitRegionStateProcedure[0];
}
if (serversToExclude != null
&& this.master.getServerManager().getOnlineServersList().size() == 1) {
LOG.debug("Only one region server found and hence going ahead with the assignment");
serversToExclude = null;
}
try {
// Ask the balancer to assign our regions. Pass the regions en masse. The balancer can do
// a better job if it has all the assignments in the one lump.
Map<ServerName, List<RegionInfo>> assignments = getBalancer().roundRobinAssignment(hris,
this.master.getServerManager().createDestinationServersList(serversToExclude));
// Return mid-method!
return createAssignProcedures(assignments);
} catch (IOException hioe) {
LOG.warn("Failed roundRobinAssignment", hioe);
}
// If an error above, fall-through to this simpler assign. Last resort.
return createAssignProcedures(hris);
}
public long storeSize(String table) throws IOException {
long total = 0;
try (Admin admin = this.hbase.getAdmin()) {
for (ServerName rs : admin.getRegionServers()) {
// NOTE: we can use getLoad() before hbase 2.0
//ServerLoad load = admin.getClusterStatus().getLoad(rs);
//total += load.getStorefileSizeMB() * Bytes.MB;
//total += load.getMemStoreSizeMB() * Bytes.MB;
TableName tableName = TableName.valueOf(this.namespace, table);
for (RegionMetrics m : admin.getRegionMetrics(rs, tableName)) {
total += m.getStoreFileSize().get(Size.Unit.BYTE);
total += m.getMemStoreSize().get(Size.Unit.BYTE);
}
}
}
return total;
}
private void savePlan(Map<ServerName, List<HRegionInfo>> bulkPlan) {
synchronized (this.colocationInfo) {
for (Entry<ServerName, List<HRegionInfo>> e : bulkPlan.entrySet()) {
if (LOG.isDebugEnabled()) {
LOG.debug("Saving user regions' plans for server " + e.getKey() + '.');
}
for (HRegionInfo hri : e.getValue()) {
if (!isTableColocated(hri.getTable())) continue;
regionOnline(hri, e.getKey());
}
if (LOG.isDebugEnabled()) {
LOG.debug("Saved user regions' plans for server " + e.getKey() + '.');
}
}
}
}
protected void localPerform() throws IOException {
getLogger().info("Starting to execute DuplicatePacketsCommandAction");
ServerName server = PolicyBasedChaosMonkey.selectRandomItem(getCurrentServers());
String hostname = server.getHostname();
try {
clusterManager.execSudoWithRetries(hostname, timeout, getCommand(ADD));
Thread.sleep(duration);
} catch (InterruptedException e) {
getLogger().debug("Failed to run the command for the full duration", e);
} finally {
clusterManager.execSudoWithRetries(hostname, timeout, getCommand(DELETE));
}
getLogger().info("Finished to execute DuplicatePacketsCommandAction");
}
@Test
public void testSortExtract(){
ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
EnvironmentEdgeManager.injectEdge(mee);
mee.setValue(1);
DeadServer d = new DeadServer();
d.putIfAbsent(hostname123);
mee.incValue(1);
d.putIfAbsent(hostname1234);
mee.incValue(1);
d.putIfAbsent(hostname12345);
List<Pair<ServerName, Long>> copy = d.copyDeadServersSince(2L);
Assert.assertEquals(2, copy.size());
Assert.assertEquals(hostname1234, copy.get(0).getFirst());
Assert.assertEquals(new Long(2L), copy.get(0).getSecond());
Assert.assertEquals(hostname12345, copy.get(1).getFirst());
Assert.assertEquals(new Long(3L), copy.get(1).getSecond());
EnvironmentEdgeManager.reset();
}
/**
* The caller will block until all the log files of the given region server have been processed -
* successfully split or an error is encountered - by an available worker region server. This
* method must only be called after the region servers have been brought online.
* @param logDirs List of log dirs to split
* @throws IOException If there was an error while splitting any log file
* @return cumulative size of the logfiles split
*/
public long splitLogDistributed(final List<Path> logDirs) throws IOException {
if (logDirs.isEmpty()) {
return 0;
}
Set<ServerName> serverNames = new HashSet<>();
for (Path logDir : logDirs) {
try {
ServerName serverName = AbstractFSWALProvider.getServerNameFromWALDirectoryName(logDir);
if (serverName != null) {
serverNames.add(serverName);
}
} catch (IllegalArgumentException e) {
// ignore invalid format error.
LOG.warn("Cannot parse server name from " + logDir);
}
}
return splitLogDistributed(serverNames, logDirs, null);
}
private void checkIfDaughterInherits2FN(List<ServerName> parentFN, List<ServerName> daughterFN) {
assertNotNull(parentFN);
assertNotNull(daughterFN);
List<ServerName> favoredNodes = Lists.newArrayList(daughterFN);
favoredNodes.removeAll(parentFN);
/*
* With a small cluster its likely some FN might accidentally get shared. Its likely the
* 3rd FN the balancer chooses might still belong to the parent in which case favoredNodes
* size would be 0.
*/
assertTrue("Daughter FN:" + daughterFN + " should have inherited 2 FN from parent FN:"
+ parentFN, favoredNodes.size() <= 1);
}
@Test
public void testReplicationStatusAfterLagging() throws Exception {
UTIL2.shutdownMiniHBaseCluster();
restartSourceCluster(1);
// add some values to cluster 1
for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
Put p = new Put(Bytes.toBytes("row" + i));
p.addColumn(famName, Bytes.toBytes("col1"), Bytes.toBytes("val" + i));
htable1.put(p);
}
UTIL2.startMiniHBaseCluster();
Thread.sleep(10000);
Admin hbaseAdmin = UTIL1.getAdmin();
ServerName serverName = UTIL1.getHBaseCluster().getRegionServer(0).getServerName();
ClusterMetrics metrics = hbaseAdmin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS));
List<ReplicationLoadSource> loadSources =
metrics.getLiveServerMetrics().get(serverName).getReplicationLoadSourceList();
assertEquals(1, loadSources.size());
ReplicationLoadSource loadSource = loadSources.get(0);
assertTrue(loadSource.hasEditsSinceRestart());
assertTrue(loadSource.getTimestampOfLastShippedOp() > 0);
assertEquals(0, loadSource.getReplicationLag());
}
private void waitForRSShutdownToStartAndFinish(JVMClusterUtil.MasterThread activeMaster,
ServerName serverName) throws InterruptedException {
ServerManager sm = activeMaster.getMaster().getServerManager();
// First wait for it to be in dead list
while (!sm.getDeadServers().isDeadServer(serverName)) {
LOG.debug("Waiting for [" + serverName + "] to be listed as dead in master");
Thread.sleep(SLEEP_TIME);
}
LOG.debug("Server [" + serverName + "] marked as dead, waiting for it to " +
"finish dead processing");
while (sm.areDeadServersInProgress()) {
LOG.debug("Server [" + serverName + "] still being processed, waiting");
Thread.sleep(SLEEP_TIME);
}
LOG.debug("Server [" + serverName + "] done with server shutdown processing");
}
private List<ServerName> getOnlineFavoredNodes(List<ServerName> onlineServers,
List<ServerName> serversWithoutStartCodes) {
if (serversWithoutStartCodes == null) {
return null;
} else {
List<ServerName> result = Lists.newArrayList();
for (ServerName sn : serversWithoutStartCodes) {
for (ServerName online : onlineServers) {
if (ServerName.isSameAddress(sn, online)) {
result.add(online);
}
}
}
return result;
}
}
/**
* @param cluster
* @param indexTable
* @param primaryTable
* @return
* @throws Exception
*/
private ServerName getSharedServer(MiniHBaseCluster cluster, byte[] indexTable,
byte[] primaryTable) throws Exception {
Set<ServerName> indexServers = getServersForTable(cluster, indexTable);
Set<ServerName> primaryServers = getServersForTable(cluster, primaryTable);
Set<ServerName> joinSet = new HashSet<ServerName>(indexServers);
joinSet.addAll(primaryServers);
// if there is already an overlap, then find it and return it
if (joinSet.size() < indexServers.size() + primaryServers.size()) {
// find the first overlapping server
for (ServerName server : joinSet) {
if (indexServers.contains(server) && primaryServers.contains(server)) {
return server;
}
}
throw new RuntimeException(
"Couldn't find a matching server on which both the primary and index table live, "
+ "even though they have overlapping server sets");
}
return null;
}
@Test
public void testAcquireAndRelease() throws Exception {
List<FakeServerProcedure> testProcedures = new ArrayList<>();
for (int i = 0; i < 4; i++) {
testProcedures.add(new FakeServerProcedure(
TEST_UTIL.getHBaseCluster().getServerHoldingMeta()));
}
ServerName server = splitWALManager.acquireSplitWALWorker(testProcedures.get(0));
Assert.assertNotNull(server);
Assert.assertNotNull(splitWALManager.acquireSplitWALWorker(testProcedures.get(1)));
Assert.assertNotNull(splitWALManager.acquireSplitWALWorker(testProcedures.get(2)));
Exception e = null;
try {
splitWALManager.acquireSplitWALWorker(testProcedures.get(3));
} catch (ProcedureSuspendedException suspendException) {
e = suspendException;
}
Assert.assertNotNull(e);
Assert.assertTrue(e instanceof ProcedureSuspendedException);
splitWALManager.releaseSplitWALWorker(server, TEST_UTIL.getHBaseCluster().getMaster()
.getMasterProcedureExecutor().getEnvironment().getProcedureScheduler());
Assert.assertNotNull(splitWALManager.acquireSplitWALWorker(testProcedures.get(3)));
}
/**
* Wait until the meta region is available and is not in transition.
* @param zkw reference to the {@link ZKWatcher} which also contains configuration and constants
* @param replicaId the ID of the replica
* @param timeout maximum time to wait in millis
* @return ServerName or null if we timed out.
* @throws InterruptedException if waiting for the socket operation fails
*/
public static ServerName blockUntilAvailable(final ZKWatcher zkw, int replicaId,
final long timeout) throws InterruptedException {
if (timeout < 0) {
throw new IllegalArgumentException();
}
if (zkw == null) {
throw new IllegalArgumentException();
}
long startTime = System.currentTimeMillis();
ServerName sn = null;
while (true) {
sn = getMetaRegionLocation(zkw, replicaId);
if (sn != null ||
(System.currentTimeMillis() - startTime) > timeout - HConstants.SOCKET_RETRY_WAIT_MS) {
break;
}
Thread.sleep(HConstants.SOCKET_RETRY_WAIT_MS);
}
return sn;
}
@Test
public void testGetTopBlockLocations() throws Exception {
for (int i = 0; i < ServerNum; i++) {
HRegionServer server = cluster.getRegionServer(i);
for (HRegion region : server.getRegions(tableName)) {
List<ServerName> servers = finder.getTopBlockLocations(region
.getRegionInfo());
// test table may have empty region
if (region.getHDFSBlocksDistribution().getUniqueBlocksTotalWeight() == 0) {
continue;
}
List<String> topHosts = region.getHDFSBlocksDistribution().getTopHosts();
// rs and datanode may have different host in local machine test
if (!topHosts.contains(server.getServerName().getHostname())) {
continue;
}
for (int j = 0; j < ServerNum; j++) {
ServerName serverName = cluster.getRegionServer(j).getServerName();
assertTrue(servers.contains(serverName));
}
}
}
}
private ServerMetrics mockServerMetricsWithCpRequests(ServerName server,
List<RegionInfo> regionsOnServer,
long cpRequestCount) {
ServerMetrics serverMetrics = mock(ServerMetrics.class);
Map<byte[], RegionMetrics> regionLoadMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
for(RegionInfo info : regionsOnServer){
RegionMetrics rl = mock(RegionMetrics.class);
when(rl.getReadRequestCount()).thenReturn(0L);
when(rl.getCpRequestCount()).thenReturn(cpRequestCount);
when(rl.getWriteRequestCount()).thenReturn(0L);
when(rl.getMemStoreSize()).thenReturn(Size.ZERO);
when(rl.getStoreFileSize()).thenReturn(Size.ZERO);
regionLoadMap.put(info.getRegionName(), rl);
}
when(serverMetrics.getRegionMetrics()).thenReturn(regionLoadMap);
return serverMetrics;
}
public static List<RegionPlan> makePlan(HBaseAdmin admin, List<RegionPlan> newRegionPlan) throws IOException {
// snapshot current region assignment
Map<HRegionInfo, ServerName> regionAssignmentMap = createRegionAssignmentMap(admin);
// update with new plan
for (RegionPlan regionPlan : newRegionPlan) {
regionAssignmentMap.put(regionPlan.getRegionInfo(), regionPlan.getDestination());
}
Map<ServerName, List<HRegionInfo>> clusterState = initializeRegionMap(admin);
for (Map.Entry<HRegionInfo, ServerName> entry : regionAssignmentMap.entrySet())
clusterState.get(entry.getValue()).add(entry.getKey());
StochasticLoadBalancer balancer = new StochasticLoadBalancer();
Configuration conf = admin.getConfiguration();
conf.setFloat("hbase.regions.slop", 0.2f);
balancer.setConf(conf);
return balancer.balanceCluster(clusterState);
}
private Set<ServerName> getInheritedFNForDaughter(FavoredNodeAssignmentHelper helper,
List<ServerName> parentFavoredNodes, Position primary, Position secondary)
throws IOException {
Set<ServerName> daughterFN = Sets.newLinkedHashSet();
if (parentFavoredNodes.size() >= primary.ordinal()) {
daughterFN.add(parentFavoredNodes.get(primary.ordinal()));
}
if (parentFavoredNodes.size() >= secondary.ordinal()) {
daughterFN.add(parentFavoredNodes.get(secondary.ordinal()));
}
while (daughterFN.size() < FavoredNodeAssignmentHelper.FAVORED_NODES_NUM) {
ServerName newNode = helper.generateMissingFavoredNode(Lists.newArrayList(daughterFN));
daughterFN.add(newNode);
}
return daughterFN;
}
@Override
public CompletableFuture<ServerName> getActiveMaster() {
return this
.<GetActiveMasterResponse> call(
(c, s, d) -> s.getActiveMaster(c, GetActiveMasterRequest.getDefaultInstance(), d),
GetActiveMasterResponse::hasServerName, "getActiveMaster()")
.thenApply(this::transformServerName);
}
public boolean equalsName(String name) {
if (name == null) return false;
if (level instanceof RegionName) {
return ((RegionName) level).name().equals(name);
} else if (level instanceof ServerName) {
return ((ServerName) level).getServerName().equals(name);
} else if (level instanceof TableName) {
return ((TableName) level).getTableName().equals(name);
} else if (level instanceof String) {
return level.equals(name);
} else
throw new RuntimeException("can not equalsName");
}
private Map<ServerName, Set<RegionInfo>> mapRSToPrimaries(
Map<RegionInfo, ServerName> primaryRSMap) {
Map<ServerName, Set<RegionInfo>> primaryServerMap = new HashMap<>();
for (Entry<RegionInfo, ServerName> e : primaryRSMap.entrySet()) {
Set<RegionInfo> currentSet = primaryServerMap.get(e.getValue());
if (currentSet == null) {
currentSet = new HashSet<>();
}
currentSet.add(e.getKey());
primaryServerMap.put(e.getValue(), currentSet);
}
return primaryServerMap;
}
/**
* Contacts a region server and waits up to hbase.hbck.close.timeout ms (default 120s) to close
* the region. This bypasses the active hmaster.
*/
public static void closeRegionSilentlyAndWait(Connection connection, ServerName server,
RegionInfo region) throws IOException, InterruptedException {
long timeout = connection.getConfiguration()
.getLong("hbase.hbck.close.timeout", 120000);
ServerManager.closeRegionSilentlyAndWait((ClusterConnection)connection, server,
region, timeout);
}
@Override
public CompletableFuture<Map<TableName, SpaceQuotaSnapshot>> getRegionServerSpaceQuotaSnapshots(
ServerName serverName) {
return this.<Map<TableName, SpaceQuotaSnapshot>> newAdminCaller()
.action((controller, stub) -> this
.<GetSpaceQuotaSnapshotsRequest, GetSpaceQuotaSnapshotsResponse,
Map<TableName, SpaceQuotaSnapshot>> adminCall(controller, stub,
RequestConverter.buildGetSpaceQuotaSnapshotsRequest(),
(s, c, req, done) -> s.getSpaceQuotaSnapshots(controller, req, done),
resp -> resp.getSnapshotsList().stream()
.collect(Collectors.toMap(snapshot -> ProtobufUtil.toTableName(snapshot.getTableName()),
snapshot -> SpaceQuotaSnapshot.toSpaceQuotaSnapshot(snapshot.getSnapshot())))))
.serverName(serverName).call();
}
@Override
public String toString() {
if (level instanceof RegionName) {
return ((RegionName) level).name();
} else if (level instanceof ServerName) {
return ((ServerName) level).getServerName();
} else if (level instanceof TableName) {
return ((TableName) level).getTableName();
} else {
return level.toString();
}
}
private static List<HBaseProtos.ServerName> toProtoServerNames(List<ServerName> servers) {
List<HBaseProtos.ServerName> pbServers = new ArrayList<>(servers.size());
for (ServerName server : servers) {
pbServers.add(ProtobufUtil.toServerName(server));
}
return pbServers;
}
/**
* A helper to close a region given a region name
* using admin protocol.
*
* @param admin
* @param regionName
* @throws IOException
*/
public static void closeRegion(final RpcController controller,
final AdminService.BlockingInterface admin, final ServerName server, final byte[] regionName)
throws IOException {
CloseRegionRequest closeRegionRequest =
ProtobufUtil.buildCloseRegionRequest(server, regionName);
try {
admin.closeRegion(controller, closeRegionRequest);
} catch (ServiceException se) {
throw getRemoteException(se);
}
}
public List<FileStatus> getWALsToSplit(ServerName serverName, boolean splitMeta)
throws IOException {
List<Path> logDirs = master.getMasterWalManager().getLogDirs(Collections.singleton(serverName));
FileStatus[] fileStatuses =
SplitLogManager.getFileList(this.conf, logDirs, splitMeta ? META_FILTER : NON_META_FILTER);
LOG.info("{} WAL count={}, meta={}", serverName, fileStatuses.length, splitMeta);
return Lists.newArrayList(fileStatuses);
}
private void addAssignment(RegionInfo regionInfo, ServerName server) {
// Process the region to region server map
regionToRegionServerMap.put(regionInfo, server);
if (server == null) return;
// Process the region server to region map
List<RegionInfo> regionList = currentRSToRegionMap.get(server);
if (regionList == null) {
regionList = new ArrayList<>();
}
regionList.add(regionInfo);
currentRSToRegionMap.put(server, regionList);
}
private HRegion updateTtl(HRegion region, byte[] family, long ttl) throws Exception {
region.close();
HTableDescriptor htd = region.getTableDesc();
HColumnDescriptor cfd = htd.getFamily(family);
if (ttl > 0) {
cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
}
cfd.setMaxVersions(10);
return HRegion.openHRegion(region.getRegionInfo(), htd, region.getWAL(), conf,
new LocalRegionServerServices(conf, ServerName.valueOf(
InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())), null);
}