private List<RegionInfo> getOpenRegions(Connection connection, TableName table) throws Exception {
  List<RegionInfo> regions = new ArrayList<>();
  Table metaTbl = connection.getTable(META_TABLE_NAME);
  String tblName = table.getNameAsString();
  RowFilter rowFilter = new RowFilter(CompareOperator.EQUAL,
    new SubstringComparator(tblName+","));
  SingleColumnValueFilter colFilter = new SingleColumnValueFilter(CATALOG_FAMILY,
    STATE_QUALIFIER, CompareOperator.EQUAL, Bytes.toBytes("OPEN"));
  Scan scan = new Scan();
  FilterList filter = new FilterList(FilterList.Operator.MUST_PASS_ALL);
  filter.addFilter(rowFilter);
  filter.addFilter(colFilter);
  scan.setFilter(filter);
  try(ResultScanner rs = metaTbl.getScanner(scan)){
    Result r;
    while ((r = rs.next()) != null) {
      RegionInfo region = RegionInfo.parseFrom(r.getValue(CATALOG_FAMILY, REGIONINFO_QUALIFIER));
      regions.add(region);
    }
  }
  return regions;
}
 

private HRegion initHRegion(TableDescriptor htd, byte[] startKey, byte[] stopKey, int replicaId)
    throws IOException {
  Configuration conf = TEST_UTIL.getConfiguration();
  conf.set("hbase.wal.provider", walProvider);
  conf.setBoolean("hbase.hregion.mvcc.preassign", false);
  Path tableDir = CommonFSUtils.getTableDir(testDir, htd.getTableName());

  RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).setStartKey(startKey)
      .setEndKey(stopKey).setReplicaId(replicaId).setRegionId(0).build();
  fileSystem = tableDir.getFileSystem(conf);
  final Configuration walConf = new Configuration(conf);
  CommonFSUtils.setRootDir(walConf, tableDir);
  this.walConf = walConf;
  wals = new WALFactory(walConf, "log_" + replicaId);
  ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
  HRegion region = HRegion.createHRegion(info, TEST_UTIL.getDefaultRootDirPath(), conf, htd,
    wals.getWAL(info));
  return region;
}
 

private long getMaxSeqId(HRegionServer rs, RegionInfo region) throws IOException {
  Path walFile = ((AbstractFSWAL<?>) rs.getWAL(null)).getCurrentFileName();
  long maxSeqId = -1L;
  try (WAL.Reader reader =
    WALFactory.createReader(UTIL.getTestFileSystem(), walFile, UTIL.getConfiguration())) {
    for (;;) {
      WAL.Entry entry = reader.next();
      if (entry == null) {
        break;
      }
      if (Bytes.equals(region.getEncodedNameAsBytes(), entry.getKey().getEncodedRegionName())) {
        maxSeqId = Math.max(maxSeqId, entry.getKey().getSequenceId());
      }
    }
  }
  return maxSeqId;
}
 

/**
 * Generate a list of regions evenly distributed between the tables.
 *
 * @param numRegions The number of regions to be generated.
 * @return List of RegionInfo.
 */
protected List<RegionInfo> randomRegions(int numRegions) {
  List<RegionInfo> regions = new ArrayList<>(numRegions);
  byte[] start = new byte[16];
  byte[] end = new byte[16];
  rand.nextBytes(start);
  rand.nextBytes(end);
  int regionIdx = rand.nextInt(tables.length);
  for (int i = 0; i < numRegions; i++) {
    Bytes.putInt(start, 0, numRegions << 1);
    Bytes.putInt(end, 0, (numRegions << 1) + 1);
    int tableIndex = (i + regionIdx) % tables.length;
    regions.add(RegionInfoBuilder.newBuilder(tables[tableIndex])
        .setStartKey(start)
        .setEndKey(end)
        .setSplit(false)
        .setRegionId(regionId++)
        .build());
  }
  return regions;
}
 

@Test
public void testRecoveryAndDoubleExecution() throws Exception {
  final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecution");
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

  List<RegionInfo> tableRegions = createTable(tableName);

  ProcedureTestingUtility.waitNoProcedureRunning(procExec);
  ProcedureTestingUtility.setKillIfHasParent(procExec, false);
  ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

  RegionInfo[] regionsToMerge = new RegionInfo[2];
  regionsToMerge[0] = tableRegions.get(0);
  regionsToMerge[1] = tableRegions.get(1);

  long procId = procExec.submitProcedure(
    new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true));

  // Restart the executor and execute the step twice
  MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
  ProcedureTestingUtility.assertProcNotFailed(procExec, procId);

  assertRegionCount(tableName, initialRegionCount - 1);
}
 

/**
 * Create a protocol buffer UpdateFavoredNodesRequest to update a list of favorednode mappings
 * @param updateRegionInfos a list of favored node mappings
 * @return a protocol buffer UpdateFavoredNodesRequest
 */
public static UpdateFavoredNodesRequest buildUpdateFavoredNodesRequest(
    final List<Pair<RegionInfo, List<ServerName>>> updateRegionInfos) {
  UpdateFavoredNodesRequest.Builder ubuilder = UpdateFavoredNodesRequest.newBuilder();
  if (updateRegionInfos != null && !updateRegionInfos.isEmpty()) {
    RegionUpdateInfo.Builder builder = RegionUpdateInfo.newBuilder();
    for (Pair<RegionInfo, List<ServerName>> pair : updateRegionInfos) {
      builder.setRegion(ProtobufUtil.toRegionInfo(pair.getFirst()));
      for (ServerName server : pair.getSecond()) {
        builder.addFavoredNodes(ProtobufUtil.toServerName(server));
      }
      ubuilder.addUpdateInfo(builder.build());
      builder.clear();
    }
  }
  return ubuilder.build();
}
 

public void checkOnline() throws DoNotRetryRegionException {
  RegionInfo ri = getRegionInfo();
  State s = state;
  if (s != State.OPEN) {
    throw new DoNotRetryRegionException(ri.getEncodedName() + " is not OPEN; state=" + s);
  }
  if (ri.isSplitParent()) {
    throw new DoNotRetryRegionException(
      ri.getEncodedName() + " is not online (splitParent=true)");
  }
  if (ri.isSplit()) {
    throw new DoNotRetryRegionException(ri.getEncodedName() + " has split=true");
  }
  if (ri.isOffline()) {
    // RegionOfflineException is not instance of DNRIOE so wrap it.
    throw new DoNotRetryRegionException(new RegionOfflineException(ri.getEncodedName()));
  }
}
 

@Override
public boolean visit(Result r) {
  if (r == null || r.isEmpty()) {
    return true;
  }
  this.report.count++;
  RegionInfo regionInfo = null;
  try {
    regionInfo = metaTableConsistencyCheck(r);
  } catch(Throwable t) {
    LOG.warn("Failed consistency check on {}", Bytes.toStringBinary(r.getRow()), t);
  }
  if (regionInfo != null) {
    LOG.trace(regionInfo.toString());
    if (regionInfo.isSplitParent()) { // splitParent means split and offline.
      this.report.splitParents.put(regionInfo, r);
    }
    if (MetaTableAccessor.hasMergeRegions(r.rawCells())) {
      this.report.mergedRegions.put(regionInfo, r);
    }
  }
  // Returning true means "keep scanning"
  return true;
}
 

@Before
public void setUp() throws Exception {
  TableName tableName = TableName.valueOf(getClass().getSimpleName());
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
  tableDescriptor.setColumnFamily(
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(fam));
  RegionInfo info = RegionInfoBuilder.newBuilder(tableName).build();
  Path path = testUtil.getDataTestDir(getClass().getSimpleName());
  region = HBaseTestingUtility.createRegionAndWAL(info, path,
    testUtil.getConfiguration(), tableDescriptor);
  rss = mock(RegionServerServices.class);
  List<HRegion> regions = new ArrayList<>(1);
  regions.add(region);
  Mockito.doReturn(regions).when(rss).getRegions();
}
 

@Test
public void testSetRegionState() throws IOException {
  TEST_UTIL.createTable(REGION_STATES_TABLE_NAME, Bytes.toBytes("family1"));
  try (Admin admin = TEST_UTIL.getConnection().getAdmin()) {
    List<RegionInfo> regions = admin.getRegions(REGION_STATES_TABLE_NAME);
    RegionInfo info = regions.get(0);
    assertEquals(RegionState.State.OPEN, getCurrentRegionState(info));
    String region = info.getEncodedName();
    try (ClusterConnection connection = this.hbck2.connect()) {
      this.hbck2.setRegionState(connection, region, RegionState.State.CLOSING);
    }
    assertEquals(RegionState.State.CLOSING, getCurrentRegionState(info));
  } finally {
    TEST_UTIL.deleteTable(REGION_STATES_TABLE_NAME);
  }
}
 

/**
 * Adds a hbase:meta row for each of the specified new regions. Initial state for new regions is
 * CLOSED.
 * @param connection connection we're using
 * @param regionInfos region information list
 * @param ts desired timestamp
 * @throws IOException if problem connecting or updating meta
 */
private static void addRegionsToMeta(Connection connection, List<RegionInfo> regionInfos,
  int regionReplication, long ts) throws IOException {
  List<Put> puts = new ArrayList<>();
  for (RegionInfo regionInfo : regionInfos) {
    if (RegionReplicaUtil.isDefaultReplica(regionInfo)) {
      Put put = makePutFromRegionInfo(regionInfo, ts);
      // New regions are added with initial state of CLOSED.
      addRegionStateToPut(put, RegionState.State.CLOSED);
      // Add empty locations for region replicas so that number of replicas can be cached
      // whenever the primary region is looked up from meta
      for (int i = 1; i < regionReplication; i++) {
        addEmptyLocation(put, i);
      }
      puts.add(put);
    }
  }
  putsToMetaTable(connection, puts);
  LOG.info("Added {} regions to meta.", puts.size());
}
 

@Test
public void testCreateWithoutColumnFamily() throws Exception {
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
  final TableName tableName = TableName.valueOf(name.getMethodName());
  // create table with 0 families will fail
  final TableDescriptorBuilder builder =
    TableDescriptorBuilder.newBuilder(MasterProcedureTestingUtility.createHTD(tableName));

  // disable sanity check
  builder.setValue(TableDescriptorChecker.TABLE_SANITY_CHECKS, Boolean.FALSE.toString());
  TableDescriptor htd = builder.build();
  final RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);

  long procId =
      ProcedureTestingUtility.submitAndWait(procExec,
          new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
  final Procedure<?> result = procExec.getResult(procId);
  assertEquals(true, result.isFailed());
  Throwable cause = ProcedureTestingUtility.getExceptionCause(result);
  assertTrue("expected DoNotRetryIOException, got " + cause,
      cause instanceof DoNotRetryIOException);
}
 

private List<RegionInfo> readRegionsFromFile(String filename) throws IOException {
  List<RegionInfo> regions = new ArrayList<>();
  File f = new File(filename);
  if (!f.exists()) {
    return regions;
  }
  try (DataInputStream dis = new DataInputStream(
      new BufferedInputStream(new FileInputStream(f)))) {
    int numRegions = dis.readInt();
    int index = 0;
    while (index < numRegions) {
      regions.add(RegionInfo.parseFromOrNull(Bytes.readByteArray(dis)));
      index++;
    }
  } catch (IOException e) {
    LOG.error("Error while reading regions from file:" + filename, e);
    throw e;
  }
  return regions;
}
 

@Override
public SpaceQuotaSnapshot getTargetState(
    String subject, SpaceQuota spaceQuota) throws IOException {
  rlock.lock();
  try {
    final long sizeLimitInBytes = spaceQuota.getSoftLimit();
    long sum = 0L;
    for (Entry<RegionInfo,Long> entry : filterBySubject(subject)) {
      sum += entry.getValue();
    }
    // Add in the size for any snapshots against this table
    sum += QuotaTableUtil.getNamespaceSnapshotSize(conn, subject);
    // Observance is defined as the size of the table being less than the limit
    SpaceQuotaStatus status = sum <= sizeLimitInBytes ? SpaceQuotaStatus.notInViolation()
        : new SpaceQuotaStatus(ProtobufUtil.toViolationPolicy(spaceQuota.getViolationPolicy()));
    return new SpaceQuotaSnapshot(status, sum, sizeLimitInBytes);
  } finally {
    rlock.unlock();
  }
}
 

@Override
public long createSystemTable(final TableDescriptor tableDescriptor) throws IOException {
  if (isStopped()) {
    throw new MasterNotRunningException();
  }

  TableName tableName = tableDescriptor.getTableName();
  if (!(tableName.isSystemTable())) {
    throw new IllegalArgumentException(
      "Only system table creation can use this createSystemTable API");
  }

  RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, null);

  LOG.info(getClientIdAuditPrefix() + " create " + tableDescriptor);

  // This special create table is called locally to master.  Therefore, no RPC means no need
  // to use nonce to detect duplicated RPC call.
  long procId = this.procedureExecutor.submitProcedure(
    new CreateTableProcedure(procedureExecutor.getEnvironment(), tableDescriptor, newRegions));

  return procId;
}
 

/**
 * Add regions to in-memory states
 * @param regionInfos regions to add
 * @param env MasterProcedureEnv
 * @param regionReplication the number of region replications
 */
private void addRegionsToInMemoryStates(List<RegionInfo> regionInfos, MasterProcedureEnv env,
    int regionReplication) {
  AssignmentManager am = env.getAssignmentManager();
  for (RegionInfo regionInfo : regionInfos) {
    if (regionInfo.isSplit()) {
      am.getRegionStates().updateRegionState(regionInfo, RegionState.State.SPLIT);
    } else {
      am.getRegionStates().updateRegionState(regionInfo, RegionState.State.CLOSED);

      // For region replicas
      for (int i = 1; i < regionReplication; i++) {
        RegionInfo regionInfoForReplica =
            RegionReplicaUtil.getRegionInfoForReplica(regionInfo, i);
        am.getRegionStates().updateRegionState(regionInfoForReplica, RegionState.State.CLOSED);
      }
    }
  }
}
 

/**
 * Get region info from local cluster.
 */
Map<ServerName, List<String>> getDeployedHRIs(final Admin admin) throws IOException {
  ClusterMetrics status = admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS));
  Collection<ServerName> regionServers = status.getLiveServerMetrics().keySet();
  Map<ServerName, List<String>> mm = new HashMap<>();
  for (ServerName hsi : regionServers) {
    // list all online regions from this region server
    List<RegionInfo> regions = admin.getRegions(hsi);
    List<String> regionNames = new ArrayList<>(regions.size());
    for (RegionInfo hri : regions) {
      regionNames.add(hri.getRegionNameAsString());
    }
    mm.put(hsi, regionNames);
  }
  return mm;
}
 

/**
 * Test the cluster startup bulk assignment which attempts to retain assignment info.
 */
@Test
public void testRetainAssignment() throws Exception {
  // Test simple case where all same servers are there
  Map<ServerName, List<RegionInfo>> currentAssignments = mockClusterServers();
  Map<RegionInfo, ServerName> inputForTest = new HashMap<>();
  for (ServerName sn : currentAssignments.keySet()) {
    for (RegionInfo region : currentAssignments.get(sn)) {
      inputForTest.put(region, sn);
    }
  }
  //verify region->null server assignment is handled
  inputForTest.put(randomRegions(1).get(0), null);
  Map<ServerName, List<RegionInfo>> newAssignment = loadBalancer
      .retainAssignment(inputForTest, servers);
  assertRetainedAssignment(inputForTest, servers, newAssignment);
}
 

private boolean canMerge(Path path, RegionInfo region1, RegionInfo region2,
    Collection<Pair<RegionInfo, RegionInfo>> alreadyMerging) throws IOException {
  if(alreadyMerging.stream().anyMatch(regionPair ->
      region1.equals(regionPair.getFirst()) ||
      region2.equals(regionPair.getFirst()) ||
      region1.equals(regionPair.getSecond()) ||
      region2.equals(regionPair.getSecond()))){
    return false;
  }
  if (RegionInfo.areAdjacent(region1, region2)) {
    long size1 = sumSizeInFS(new Path(path, region1.getEncodedName()));
    long size2 = sumSizeInFS(new Path(path, region2.getEncodedName()));
    boolean mergeable = (resultSizeThreshold > (size1 + size2));
    if (!mergeable) {
      LOG.warn("Not merging regions {} and {} because resulting region size would get close to " +
          "the {} limit. {} total size: {}; {} total size:{}", region1.getEncodedName(),
        region2.getEncodedName(), resultSizeThreshold, region1.getEncodedName(), size1,
        region2.getEncodedName(), size2);
    }
    return mergeable;
  } else {
    LOG.warn(
      "WARNING: Can't merge regions {} and {} because those are not adjacent.",
      region1.getEncodedName(),
      region2.getEncodedName());
    return false;
  }
}
 

@Override
public WAL getWAL(RegionInfo region) throws IOException {
  if (region == null) {
    return provider.getWAL(null);
  }
  WAL wal = null;
  Optional<Pair<String, String>> peerIdAndRemoteWALDir =
      peerInfoProvider.getPeerIdAndRemoteWALDir(region.getTable());
  if (peerIdAndRemoteWALDir.isPresent()) {
    Pair<String, String> pair = peerIdAndRemoteWALDir.get();
    wal = getWAL(pair.getFirst(), pair.getSecond());
  }
  return wal != null ? wal : provider.getWAL(region);
}
 

@Override
public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
                            TableDescriptor desc, RegionInfo[] regions) throws IOException {
  if (this.createTableStartTime > 0) {
    long time = System.currentTimeMillis() - this.createTableStartTime;
    LOG.info("Create table took: " + time);

    // Update the timer metric for the create table operation duration.
    createTableTimer.updateMillis(time);
  }
}
 

/**
 * @param info the RegionInfo for this region
 * @param edits the file of recovered edits
 * @throws IOException Exception
 */
public void postReplayWALs(final RegionInfo info, final Path edits) throws IOException {
  execOperation(coprocEnvironments.isEmpty()? null:
      new RegionObserverOperationWithoutResult() {
    @Override
    public void call(RegionObserver observer) throws IOException {
      observer.postReplayWALs(this, info, edits);
    }
  });
}
 

private void assignMetaReplica(int replicaId)
    throws IOException, KeeperException, InterruptedException {
  errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta, replicaId " +
      replicaId +" is not found on any region.");
  if (shouldFixAssignments()) {
    errors.print("Trying to fix a problem with hbase:meta..");
    setShouldRerun();
    // try to fix it (treat it as unassigned region)
    RegionInfo h = RegionReplicaUtil.getRegionInfoForReplica(
        RegionInfoBuilder.FIRST_META_REGIONINFO, replicaId);
    HBaseFsckRepair.fixUnassigned(admin, h);
    HBaseFsckRepair.waitUntilAssigned(admin, h);
  }
}
 

public static RegionInfo[] createRegionInfos(TableDescriptor tableDescriptor,
    byte[][] splitKeys) {
  long regionId = System.currentTimeMillis();
  RegionInfo[] hRegionInfos = null;
  if (splitKeys == null || splitKeys.length == 0) {
    hRegionInfos = new RegionInfo[]{
        RegionInfoBuilder.newBuilder(tableDescriptor.getTableName())
         .setStartKey(null)
         .setEndKey(null)
         .setSplit(false)
         .setRegionId(regionId)
         .build()
    };
  } else {
    int numRegions = splitKeys.length + 1;
    hRegionInfos = new RegionInfo[numRegions];
    byte[] startKey = null;
    byte[] endKey = null;
    for (int i = 0; i < numRegions; i++) {
      endKey = (i == splitKeys.length) ? null : splitKeys[i];
      hRegionInfos[i] =
          RegionInfoBuilder.newBuilder(tableDescriptor.getTableName())
              .setStartKey(startKey)
              .setEndKey(endKey)
              .setSplit(false)
              .setRegionId(regionId)
              .build();
      startKey = endKey;
    }
  }
  return hRegionInfos;
}
 

private HRegion createHRegion(TableDescriptor td, RegionInfo info, String dir, WAL wal,
    Durability durability) throws IOException {
  Path path = new Path(DIR, dir);
  if (FS.exists(path)) {
    if (!FS.delete(path, true)) {
      throw new IOException("Failed delete of " + path);
    }
  }
  ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
  return HRegion.createHRegion(info, path, CONF, td, wal);
}
 

@Test
public void testMerge()
    throws IOException, InterruptedException, ExecutionException, TimeoutException {
  TableName tableName = TableName.valueOf(name.getMethodName());
  Admin admin = UTIL.getAdmin();
  admin.createTable(
    TableDescriptorBuilder.newBuilder(tableName)
        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).build(),
    new byte[][] { Bytes.toBytes(0) });
  List<RegionInfo> regions = admin.getRegions(tableName);
  addStoreFileToKnownFamily(regions.get(0));
  admin.mergeRegionsAsync(regions.get(0).getEncodedNameAsBytes(),
    regions.get(1).getEncodedNameAsBytes(), false).get(30, TimeUnit.SECONDS);
}
 

@Test
public void testIsEnd() {
  assertTrue(RegionInfoBuilder.FIRST_META_REGIONINFO.isFirst());
  org.apache.hadoop.hbase.client.RegionInfo ri =
      org.apache.hadoop.hbase.client.RegionInfoBuilder.newBuilder(TableName.META_TABLE_NAME).
          setEndKey(Bytes.toBytes("not_end")).build();
  assertFalse(ri.isLast());
}
 

private static Map<byte[], Integer> createRegionSizesMap(final List<RegionInfo> regionInfos,
  int... sizes) {
  if (regionInfos.size() != sizes.length) {
    throw new IllegalStateException("Parameter lengths must match.");
  }

  final Map<byte[], Integer> ret = new HashMap<>(regionInfos.size());
  for (int i = 0; i < regionInfos.size(); i++) {
    ret.put(regionInfos.get(i).getRegionName(), sizes[i]);
  }
  return ret;
}
 

List<RegionInfo> findExtraRegionsInMETA(String table) throws IOException {
  InternalMetaChecker<RegionInfo> extraChecker = new InternalMetaChecker<>();
  return extraChecker.checkRegionsInMETA(table, (regions,dirs) -> {
    ListUtils<RegionInfo, Path> utils = new ListUtils<>();
    return utils.complement(regions, dirs, r -> r.getEncodedName(), d -> d.getName());
  });
}
 

/**
 * Remove merged region
 * @param mergedRegion {@link RegionInfo}
 * @throws IOException
 */
public void cleanupMergedRegion(final RegionInfo mergedRegion) throws IOException {
  Path regionDir = new Path(this.tableDir, mergedRegion.getEncodedName());
  if (this.fs.exists(regionDir) && !this.fs.delete(regionDir, true)) {
    throw new IOException("Failed delete of " + regionDir);
  }
}