private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1}, null);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 

private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  Path hlogPath = new Path(FSUtils.getRootDir(conf) + "/hlog");
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  HLog hLog = HLogFactory.createHLog(fs, hlogPath, tableName, conf);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd, new MockRegionServerServices(conf, null));
}
 

private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1}, null);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 

private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, tableName + ".hlog");
  HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
  WAL hLog = walFactory.getWAL(info);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 

private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1});
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
                     new LocalRegionServerServices(conf, ServerName.valueOf(
                       InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 

private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1});
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
                     new LocalRegionServerServices(conf, ServerName.valueOf(
                       InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 

private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1});
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 

private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  Path hlogPath = new Path(FSUtils.getRootDir(conf) + "/hlog");
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  HLog hLog = HLogFactory.createHLog(fs, hlogPath, tableName, conf);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd, new MockRegionServerServices(conf, null));
}
 

/**
 * Create reference file(s) to parent region hfiles in the <code>mergeDir</code>
 * @param regionFs merge parent region file system
 * @param mergeDir the temp directory in which we are accumulating references.
 */
private void mergeStoreFiles(final MasterProcedureEnv env, final HRegionFileSystem regionFs,
    final Path mergeDir) throws IOException {
  final TableDescriptor htd = env.getMasterServices().getTableDescriptors().get(getTableName());
  for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {
    String family = hcd.getNameAsString();
    final Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(family);
    if (storeFiles != null && storeFiles.size() > 0) {
      for (StoreFileInfo storeFileInfo : storeFiles) {
        // Create reference file(s) to parent region file here in mergedDir.
        // As this procedure is running on master, use CacheConfig.DISABLED means
        // don't cache any block.
        regionFs.mergeStoreFile(mergedRegion, family, new HStoreFile(
            storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED), mergeDir);
      }
    }
  }
}
 

/**
 * Create daughter regions
 */
@VisibleForTesting
public void createDaughterRegions(final MasterProcedureEnv env) throws IOException {
  final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
  final Path tabledir = CommonFSUtils.getTableDir(mfs.getRootDir(), getTableName());
  final FileSystem fs = mfs.getFileSystem();
  HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(
    env.getMasterConfiguration(), fs, tabledir, getParentRegion(), false);
  regionFs.createSplitsDir(daughterOneRI, daughterTwoRI);

  Pair<Integer, Integer> expectedReferences = splitStoreFiles(env, regionFs);

  assertReferenceFileCount(fs, expectedReferences.getFirst(),
    regionFs.getSplitsDir(daughterOneRI));
  //Move the files from the temporary .splits to the final /table/region directory
  regionFs.commitDaughterRegion(daughterOneRI);
  assertReferenceFileCount(fs, expectedReferences.getFirst(),
    new Path(tabledir, daughterOneRI.getEncodedName()));

  assertReferenceFileCount(fs, expectedReferences.getSecond(),
    regionFs.getSplitsDir(daughterTwoRI));
  regionFs.commitDaughterRegion(daughterTwoRI);
  assertReferenceFileCount(fs, expectedReferences.getSecond(),
    new Path(tabledir, daughterTwoRI.getEncodedName()));
}
 

private Pair<Path, Path> splitStoreFile(HRegionFileSystem regionFs, byte[] family, HStoreFile sf)
  throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("pid=" + getProcId() + " splitting started for store file: " +
        sf.getPath() + " for region: " + getParentRegion().getShortNameToLog());
  }

  final byte[] splitRow = getSplitRow();
  final String familyName = Bytes.toString(family);
  final Path path_first = regionFs.splitStoreFile(this.daughterOneRI, familyName, sf, splitRow,
      false, splitPolicy);
  final Path path_second = regionFs.splitStoreFile(this.daughterTwoRI, familyName, sf, splitRow,
     true, splitPolicy);
  if (LOG.isDebugEnabled()) {
    LOG.debug("pid=" + getProcId() + " splitting complete for store file: " +
        sf.getPath() + " for region: " + getParentRegion().getShortNameToLog());
  }
  return new Pair<Path,Path>(path_first, path_second);
}
 

/**
 * @return the set of the regions contained in the table
 */
private List<RegionInfo> getTableRegions() throws IOException {
  LOG.debug("get table regions: " + tableDir);
  FileStatus[] regionDirs =
    CommonFSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
  if (regionDirs == null) {
    return null;
  }

  List<RegionInfo> regions = new ArrayList<>(regionDirs.length);
  for (int i = 0; i < regionDirs.length; ++i) {
    RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDirs[i].getPath());
    regions.add(hri);
  }
  LOG.debug("found " + regions.size() + " regions for table=" +
    tableDesc.getTableName().getNameAsString());
  return regions;
}
 

@Override
protected boolean shouldIncludeStore(HRegionFileSystem fileSystem, String family,
    Collection<StoreFileInfo> storeFiles, long ts) throws IOException {

  for (StoreFileInfo storeFile : storeFiles) {
    // Lets only compact when all files are older than TTL
    if (storeFile.getModificationTime() >= ts) {
      LOG.info("There is atleast one file in store: " + family + " file: " + storeFile.getPath()
          + " with timestamp " + storeFile.getModificationTime()
          + " for region: " + fileSystem.getRegionInfo().getEncodedName()
          + " older than TTL: " + ts);
      return false;
    }
  }
  return true;
}
 

boolean shouldCFBeCompacted(HRegionFileSystem fileSystem, String family, long ts)
    throws IOException {

  // do we have any store files?
  Collection<StoreFileInfo> storeFiles = fileSystem.getStoreFiles(family);
  if (storeFiles == null) {
    LOG.info("Excluding store: " + family + " for compaction for region:  " + fileSystem
        .getRegionInfo().getEncodedName(), " has no store files");
    return false;
  }
  // check for reference files
  if (fileSystem.hasReferences(family) && familyHasReferenceFile(fileSystem, family, ts)) {
    LOG.info("Including store: " + family + " with: " + storeFiles.size()
        + " files for compaction for region: " + fileSystem.getRegionInfo().getEncodedName());
    return true;
  }
  // check store file timestamps
  boolean includeStore = this.shouldIncludeStore(fileSystem, family, storeFiles, ts);
  if (!includeStore) {
    LOG.info("Excluding store: " + family + " for compaction for region:  " + fileSystem
        .getRegionInfo().getEncodedName() + " already compacted");
  }
  return includeStore;
}
 

@Test public void testIfWeHaveNewReferenceFilesButOldStoreFiles() throws Exception {
  // this tests that reference files that are new, but have older timestamps for the files
  // they reference still will get compacted.
  TableName table = TableName.valueOf("TestMajorCompactor");
  TableDescriptor htd = UTILITY.createTableDescriptor(table, Bytes.toBytes(FAMILY));
  RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
  HRegion region =
      HBaseTestingUtility.createRegionAndWAL(hri, rootRegionDir, UTILITY.getConfiguration(), htd);

  Configuration configuration = mock(Configuration.class);
  // the reference file timestamp is newer
  List<StoreFileInfo> storeFiles = mockStoreFiles(regionStoreDir, 4, 101);
  List<Path> paths = storeFiles.stream().map(StoreFileInfo::getPath).collect(Collectors.toList());
  // the files that are referenced are older, thus we still compact.
  HRegionFileSystem fileSystem =
      mockFileSystem(region.getRegionInfo(), true, storeFiles, 50);
  MajorCompactionRequest majorCompactionRequest = spy(new MajorCompactionRequest(configuration,
      region.getRegionInfo(), Sets.newHashSet(FAMILY)));
  doReturn(mock(Connection.class)).when(majorCompactionRequest).getConnection(eq(configuration));
  doReturn(paths).when(majorCompactionRequest).getReferenceFilePaths(any(FileSystem.class),
      any(Path.class));
  doReturn(fileSystem).when(majorCompactionRequest).getFileSystem(any(Connection.class));
  Set<String> result =
      majorCompactionRequest.getStoresRequiringCompaction(Sets.newHashSet("a"), 100);
  assertEquals(FAMILY, Iterables.getOnlyElement(result));
}
 

private HRegionFileSystem mockFileSystem(RegionInfo info, boolean hasReferenceFiles,
    List<StoreFileInfo> storeFiles, long referenceFileTimestamp) throws IOException {
  FileSystem fileSystem = mock(FileSystem.class);
  if (hasReferenceFiles) {
    FileStatus fileStatus = mock(FileStatus.class);
    doReturn(referenceFileTimestamp).when(fileStatus).getModificationTime();
    doReturn(fileStatus).when(fileSystem).getFileLinkStatus(isA(Path.class));
  }
  HRegionFileSystem mockSystem = mock(HRegionFileSystem.class);
  doReturn(info).when(mockSystem).getRegionInfo();
  doReturn(regionStoreDir).when(mockSystem).getStoreDir(FAMILY);
  doReturn(hasReferenceFiles).when(mockSystem).hasReferences(anyString());
  doReturn(storeFiles).when(mockSystem).getStoreFiles(anyString());
  doReturn(fileSystem).when(mockSystem).getFileSystem();
  return mockSystem;
}
 

protected RegionInfo createRegion(Configuration conf, final Table htbl,
    byte[] startKey, byte[] endKey) throws IOException {
  Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
  RegionInfo hri = RegionInfoBuilder.newBuilder(htbl.getName())
      .setStartKey(startKey)
      .setEndKey(endKey)
      .build();

  LOG.info("manually adding regioninfo and hdfs data: " + hri.toString());
  Path rootDir = CommonFSUtils.getRootDir(conf);
  FileSystem fs = rootDir.getFileSystem(conf);
  Path p = new Path(CommonFSUtils.getTableDir(rootDir, htbl.getName()),
      hri.getEncodedName());
  fs.mkdirs(p);
  Path riPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
  FSDataOutputStream out = fs.create(riPath);
  out.write(RegionInfo.toDelimitedByteArray(hri));
  out.close();

  // add to meta.
  MetaTableAccessor.addRegionToMeta(TEST_UTIL.getConnection(), hri);
  meta.close();
  return hri;
}
 

/**
 * If merged region no longer holds reference to the merge regions, archive
 * merge region on hdfs and perform deleting references in hbase:meta
 * @return true if we delete references in merged region on hbase:meta and archive
 *   the files on the file system
 */
private boolean cleanMergeRegion(final RegionInfo mergedRegion, List<RegionInfo> parents)
    throws IOException {
  FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
  Path rootdir = this.services.getMasterFileSystem().getRootDir();
  Path tabledir = CommonFSUtils.getTableDir(rootdir, mergedRegion.getTable());
  TableDescriptor htd = getDescriptor(mergedRegion.getTable());
  HRegionFileSystem regionFs = null;
  try {
    regionFs = HRegionFileSystem.openRegionFromFileSystem(
        this.services.getConfiguration(), fs, tabledir, mergedRegion, true);
  } catch (IOException e) {
    LOG.warn("Merged region does not exist: " + mergedRegion.getEncodedName());
  }
  if (regionFs == null || !regionFs.hasReferences(htd)) {
    LOG.debug("Deleting parents ({}) from fs; merged child {} no longer holds references",
         parents.stream().map(r -> RegionInfo.getShortNameToLog(r)).
            collect(Collectors.joining(", ")),
        mergedRegion);
    ProcedureExecutor<MasterProcedureEnv> pe = this.services.getMasterProcedureExecutor();
    pe.submitProcedure(new GCMultipleMergedRegionsProcedure(pe.getEnvironment(),
        mergedRegion,  parents));
    for (RegionInfo ri:  parents) {
      // The above scheduled GCMultipleMergedRegionsProcedure does the below.
      // Do we need this?
      this.services.getAssignmentManager().getRegionStates().deleteRegion(ri);
      this.services.getServerManager().removeRegion(ri);
    }
    return true;
  }
  return false;
}
 

/**
 * Clean up a merged region on rollback after failure.
 */
private void cleanupMergedRegion(final MasterProcedureEnv env) throws IOException {
  final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
  TableName tn = this.regionsToMerge[0].getTable();
  final Path tabledir = CommonFSUtils.getTableDir(mfs.getRootDir(), tn);
  final FileSystem fs = mfs.getFileSystem();
  // See createMergedRegion above where we specify the merge dir as being in the
  // FIRST merge parent region.
  HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(
    env.getMasterConfiguration(), fs, tabledir, regionsToMerge[0], false);
  regionFs.cleanupMergedRegion(mergedRegion);
}
 

Map<String, Long> getStoresRequiringCompaction(TableDescriptor htd) throws IOException {
  try(Connection connection = getConnection(configuration)) {
    HRegionFileSystem fileSystem = getFileSystem(connection);
    Map<String, Long> familyTTLMap = Maps.newHashMap();
    for (ColumnFamilyDescriptor descriptor : htd.getColumnFamilies()) {
      long ts = getColFamilyCutoffTime(descriptor);
      // If the table's TTL is forever, lets not compact any of the regions.
      if (ts > 0 && shouldCFBeCompacted(fileSystem, descriptor.getNameAsString(), ts)) {
        familyTTLMap.put(descriptor.getNameAsString(), ts);
      }
    }
    return familyTTLMap;
  }
}
 

Set<String> getStoresRequiringCompaction(Set<String> requestedStores, long timestamp)
    throws IOException {
  try(Connection connection = getConnection(configuration)) {
    HRegionFileSystem fileSystem = getFileSystem(connection);
    Set<String> familiesToCompact = Sets.newHashSet();
    for (String family : requestedStores) {
      if (shouldCFBeCompacted(fileSystem, family, timestamp)) {
        familiesToCompact.add(family);
      }
    }
    return familiesToCompact;
  }
}
 

protected boolean shouldIncludeStore(HRegionFileSystem fileSystem, String family,
    Collection<StoreFileInfo> storeFiles, long ts) throws IOException {

  for (StoreFileInfo storeFile : storeFiles) {
    if (storeFile.getModificationTime() < ts) {
      LOG.info("Including store: " + family + " with: " + storeFiles.size()
          + " files for compaction for region: "
          + fileSystem.getRegionInfo().getEncodedName());
      return true;
    }
  }
  return false;
}
 

protected boolean familyHasReferenceFile(HRegionFileSystem fileSystem, String family, long ts)
    throws IOException {
  List<Path> referenceFiles =
      getReferenceFilePaths(fileSystem.getFileSystem(), fileSystem.getStoreDir(family));
  for (Path referenceFile : referenceFiles) {
    FileStatus status = fileSystem.getFileSystem().getFileLinkStatus(referenceFile);
    if (status.getModificationTime() < ts) {
      LOG.info("Including store: " + family + " for compaction for region:  " + fileSystem
          .getRegionInfo().getEncodedName() + " (reference store files)");
      return true;
    }
  }
  return false;

}
 

@VisibleForTesting
HRegionFileSystem getFileSystem(Connection connection) throws IOException {
  Admin admin = connection.getAdmin();
  return HRegionFileSystem.openRegionFromFileSystem(admin.getConfiguration(),
    CommonFSUtils.getCurrentFileSystem(admin.getConfiguration()), CommonFSUtils.getTableDir(
      CommonFSUtils.getRootDir(admin.getConfiguration()), region.getTable()),
    region, true);
}
 

private RegionData[] createTable(final TableDescriptor htd, final int nregions)
    throws IOException {
  Path tableDir = CommonFSUtils.getTableDir(rootDir, htd.getTableName());
  new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false);

  assertTrue(nregions % 2 == 0);
  RegionData[] regions = new RegionData[nregions];
  for (int i = 0; i < regions.length; i += 2) {
    byte[] startKey = Bytes.toBytes(0 + i * 2);
    byte[] endKey = Bytes.toBytes(1 + i * 2);

    // First region, simple with one plain hfile.
    RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName())
        .setStartKey(startKey)
        .setEndKey(endKey)
        .build();
    HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
    regions[i] = new RegionData(tableDir, hri, 3);
    for (int j = 0; j < regions[i].files.length; ++j) {
      Path storeFile = createStoreFile(rfs.createTempName());
      regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile);
    }

    // Second region, used to test the split case.
    // This region contains a reference to the hfile in the first region.
    startKey = Bytes.toBytes(2 + i * 2);
    endKey = Bytes.toBytes(3 + i * 2);
    hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
    rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
    regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length);
    for (int j = 0; j < regions[i].files.length; ++j) {
      String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName();
      Path refFile = createStoreFile(new Path(rootDir, refName));
      regions[i+1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile);
    }
  }
  return regions;
}
 

private MajorCompactionRequest makeMockRequest(List<StoreFileInfo> storeFiles,
    boolean references) throws IOException {
  Configuration configuration = mock(Configuration.class);
  RegionInfo regionInfo = mock(RegionInfo.class);
  when(regionInfo.getEncodedName()).thenReturn("HBase");
  when(regionInfo.getTable()).thenReturn(TableName.valueOf("foo"));
  MajorCompactionRequest request =
      new MajorCompactionRequest(configuration, regionInfo, Sets.newHashSet("a"));
  MajorCompactionRequest spy = spy(request);
  HRegionFileSystem fileSystem = mockFileSystem(regionInfo, references, storeFiles);
  doReturn(fileSystem).when(spy).getFileSystem(isA(Connection.class));
  doReturn(mock(Connection.class)).when(spy).getConnection(eq(configuration));
  return spy;
}
 

private MajorCompactionTTLRequest makeMockRequest(List<StoreFileInfo> storeFiles)
    throws IOException {
  Configuration configuration = mock(Configuration.class);
  RegionInfo regionInfo = mock(RegionInfo.class);
  when(regionInfo.getEncodedName()).thenReturn("HBase");
  when(regionInfo.getTable()).thenReturn(TableName.valueOf("foo"));
  MajorCompactionTTLRequest request = new MajorCompactionTTLRequest(configuration, regionInfo);
  MajorCompactionTTLRequest spy = spy(request);
  HRegionFileSystem fileSystem = mockFileSystem(regionInfo, false, storeFiles);
  doReturn(fileSystem).when(spy).getFileSystem(isA(Connection.class));
  doReturn(mock(Connection.class)).when(spy).getConnection(eq(configuration));
  return spy;
}
 

void putRegionInfoFromHdfsInMeta(Path region) throws IOException {
  RegionInfo info = HRegionFileSystem.loadRegionInfoFileContent(fs, region);
  HBCKMetaTableAccessor.addRegionToMeta(conn, info);
}
 

/**
 * Checks if a daughter region -- either splitA or splitB -- still holds
 * references to parent.
 * @param parent Parent region
 * @param daughter Daughter region
 * @return A pair where the first boolean says whether or not the daughter
 *   region directory exists in the filesystem and then the second boolean says
 *   whether the daughter has references to the parent.
 */
private Pair<Boolean, Boolean> checkDaughterInFs(final RegionInfo parent,
  final RegionInfo daughter)
throws IOException {
  if (daughter == null)  {
    return new Pair<>(Boolean.FALSE, Boolean.FALSE);
  }

  FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
  Path rootdir = this.services.getMasterFileSystem().getRootDir();
  Path tabledir = CommonFSUtils.getTableDir(rootdir, daughter.getTable());

  Path daughterRegionDir = new Path(tabledir, daughter.getEncodedName());

  HRegionFileSystem regionFs;

  try {
    if (!CommonFSUtils.isExists(fs, daughterRegionDir)) {
      return new Pair<>(Boolean.FALSE, Boolean.FALSE);
    }
  } catch (IOException ioe) {
    LOG.error("Error trying to determine if daughter region exists, " +
             "assuming exists and has references", ioe);
    return new Pair<>(Boolean.TRUE, Boolean.TRUE);
  }

  boolean references = false;
  TableDescriptor parentDescriptor = getDescriptor(parent.getTable());
  try {
    regionFs = HRegionFileSystem.openRegionFromFileSystem(
        this.services.getConfiguration(), fs, tabledir, daughter, true);

    for (ColumnFamilyDescriptor family: parentDescriptor.getColumnFamilies()) {
      if ((references = regionFs.hasReferences(family.getNameAsString()))) {
        break;
      }
    }
  } catch (IOException e) {
    LOG.error("Error trying to determine referenced files from : " + daughter.getEncodedName()
        + ", to: " + parent.getEncodedName() + " assuming has references", e);
    return new Pair<>(Boolean.TRUE, Boolean.TRUE);
  }
  return new Pair<>(Boolean.TRUE, references);
}
 

@VisibleForTesting
protected void addRegion(final Path tableDir, final RegionInfo regionInfo, RegionVisitor visitor)
    throws IOException {
  boolean isMobRegion = MobUtils.isMobRegionInfo(regionInfo);
  try {
    Path baseDir = tableDir;
    // Open the RegionFS
    if (isMobRegion) {
      baseDir = CommonFSUtils.getTableDir(MobUtils.getMobHome(conf), regionInfo.getTable());
    }
    HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, rootFs,
      baseDir, regionInfo, true);
    monitor.rethrowException();

    // 1. dump region meta info into the snapshot directory
    LOG.debug("Storing region-info for snapshot.");
    Object regionData = visitor.regionOpen(regionInfo);
    monitor.rethrowException();

    // 2. iterate through all the stores in the region
    LOG.debug("Creating references for hfiles");

    // This ensures that we have an atomic view of the directory as long as we have < ls limit
    // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files
    // in batches and may miss files being added/deleted. This could be more robust (iteratively
    // checking to see if we have all the files until we are sure), but the limit is currently
    // 1000 files/batch, far more than the number of store files under a single column family.
    Collection<String> familyNames = regionFs.getFamilies();
    if (familyNames != null) {
      for (String familyName: familyNames) {
        Object familyData = visitor.familyOpen(regionData, Bytes.toBytes(familyName));
        monitor.rethrowException();

        Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName);
        if (storeFiles == null) {
          if (LOG.isDebugEnabled()) {
            LOG.debug("No files under family: " + familyName);
          }
          continue;
        }

        // 2.1. build the snapshot reference for the store
        // iterate through all the store's files and create "references".
        addReferenceFiles(visitor, regionData, familyData, storeFiles, false);

        visitor.familyClose(regionData, familyData);
      }
    }
    visitor.regionClose(regionData);
  } catch (IOException e) {
    // the mob directory might not be created yet, so do nothing when it is a mob region
    if (!isMobRegion) {
      throw e;
    }
  }
}