下面列出了怎么用org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils的API类实例代码及写法,或者点击链接到github查看源代码。
/**
* get region infos
*
* @param zookeeperQuorum zookeeper quorum
* @param zookeeperClientPort zookeeper client port
* @param hBaseRootDir HBase root dir
* @param snapshotName snapshot name
* @return region info list
* @throws IOException IOException
*/
public static List<HRegionInfo> getRegionInfos(String zookeeperQuorum, String zookeeperClientPort,
String hBaseRootDir, String snapshotName) throws IOException {
try {
Configuration conf = Utils.getHadoopConf(zookeeperQuorum, zookeeperClientPort);
Path root = new Path(hBaseRootDir);
FileSystem fs = FileSystem.get(conf);
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, root);
HBaseProtos.SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
return Utils.getRegionInfosFromManifest(manifest);
} catch (IOException ex) {
logger.error("get region info error: " + ex.getMessage(), ex);
throw ex;
}
}
/**
* Returns value represent path for:
* ""/$USER/SBACKUP_ROOT/backup_id/namespace/table/.hbase-snapshot/
* snapshot_1396650097621_namespace_table"
* this path contains .snapshotinfo, .tabledesc (0.96 and 0.98) this path contains .snapshotinfo,
* .data.manifest (trunk)
* @param tableName table name
* @return path to table info
* @throws IOException exception
*/
Path getTableInfoPath(TableName tableName) throws IOException {
Path tableSnapShotPath = getTableSnapshotPath(backupRootPath, tableName, backupId);
Path tableInfoPath = null;
// can't build the path directly as the timestamp values are different
FileStatus[] snapshots = fs.listStatus(tableSnapShotPath,
new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
for (FileStatus snapshot : snapshots) {
tableInfoPath = snapshot.getPath();
// SnapshotManifest.DATA_MANIFEST_NAME = "data.manifest";
if (tableInfoPath.getName().endsWith("data.manifest")) {
break;
}
}
return tableInfoPath;
}
/**
* Get table descriptor
* @param tableName is the table backed up
* @return {@link TableDescriptor} saved in backup image of the table
*/
TableDescriptor getTableDesc(TableName tableName) throws IOException {
Path tableInfoPath = this.getTableInfoPath(tableName);
SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, tableInfoPath);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, tableInfoPath, desc);
TableDescriptor tableDescriptor = manifest.getTableDescriptor();
if (!tableDescriptor.getTableName().equals(tableName)) {
LOG.error("couldn't find Table Desc for table: " + tableName + " under tableInfoPath: "
+ tableInfoPath.toString());
LOG.error("tableDescriptor.getNameAsString() = "
+ tableDescriptor.getTableName().getNameAsString());
throw new FileNotFoundException("couldn't find Table Desc for table: " + tableName
+ " under tableInfoPath: " + tableInfoPath.toString());
}
return tableDescriptor;
}
@Override
public void preCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final SnapshotDescription snapshot, final TableDescriptor hTableDescriptor)
throws IOException {
User user = getActiveUser(ctx);
if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, user)
&& hTableDescriptor.getTableName().getNameAsString()
.equals(snapshot.getTableNameAsString())) {
// Snapshot owner is allowed to create a table with the same name as the snapshot he took
AuthResult result = AuthResult.allow("cloneSnapshot " + snapshot.getName(),
"Snapshot owner check allowed", user, null, hTableDescriptor.getTableName(), null);
AccessChecker.logResult(result);
} else {
accessChecker.requirePermission(user, "cloneSnapshot " + snapshot.getName(), null,
Action.ADMIN);
}
}
private void openWithoutRestoringSnapshot() throws IOException {
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
SnapshotProtos.SnapshotDescription snapshotDesc =
SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
if (regionManifests == null) {
throw new IllegalArgumentException("Snapshot seems empty, snapshotName: " + snapshotName);
}
regions = new ArrayList<>(regionManifests.size());
regionManifests.stream().map(r -> ProtobufUtil.toRegionInfo(r.getRegionInfo()))
.filter(this::isValidRegion).sorted().forEach(r -> regions.add(r));
htd = manifest.getTableDescriptor();
}
/**
* Action before cloning from snapshot.
* @param env MasterProcedureEnv
* @throws IOException
* @throws InterruptedException
*/
private void preCloneSnapshot(final MasterProcedureEnv env)
throws IOException, InterruptedException {
if (!getTableName().isSystemTable()) {
// Check and update namespace quota
final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
SnapshotManifest manifest = SnapshotManifest.open(
env.getMasterConfiguration(),
mfs.getFileSystem(),
SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, mfs.getRootDir()),
snapshot);
ProcedureSyncWait.getMasterQuotaManager(env)
.checkNamespaceTableAndRegionQuota(getTableName(), manifest.getRegionManifestsMap().size());
}
final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
if (cpHost != null) {
cpHost.preCreateTableAction(tableDescriptor, null, getUser());
}
}
/**
* Take a snapshot using the specified handler.
* On failure the snapshot temporary working directory is removed.
* NOTE: prepareToTakeSnapshot() called before this one takes care of the rejecting the
* snapshot request if the table is busy with another snapshot/restore operation.
* @param snapshot the snapshot description
* @param handler the snapshot handler
*/
private synchronized void snapshotTable(SnapshotDescription snapshot,
final TakeSnapshotHandler handler) throws IOException {
try {
handler.prepare();
this.executorService.submit(handler);
this.snapshotHandlers.put(TableName.valueOf(snapshot.getTable()), handler);
} catch (Exception e) {
// cleanup the working directory by trying to delete it from the fs.
Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir,
master.getConfiguration());
FileSystem workingDirFs = workingDir.getFileSystem(master.getConfiguration());
try {
if (!workingDirFs.delete(workingDir, true)) {
LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
ClientSnapshotDescriptionUtils.toString(snapshot));
}
} catch (IOException e1) {
LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
ClientSnapshotDescriptionUtils.toString(snapshot));
}
// fail the snapshot
throw new SnapshotCreationException("Could not build snapshot handler", e,
ProtobufUtil.createSnapshotDesc(snapshot));
}
}
@Override
public void setConf(final Configuration conf) {
super.setConf(conf);
try {
long cacheRefreshPeriod = conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY,
DEFAULT_HFILE_CACHE_REFRESH_PERIOD);
final FileSystem fs = CommonFSUtils.getCurrentFileSystem(conf);
Path rootDir = CommonFSUtils.getRootDir(conf);
Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir, conf);
FileSystem workingFs = workingDir.getFileSystem(conf);
cache = new SnapshotFileCache(fs, rootDir, workingFs, workingDir, cacheRefreshPeriod,
cacheRefreshPeriod, "snapshot-hfile-cleaner-cache-refresher",
new SnapshotFileCache.SnapshotFileInspector() {
@Override
public Collection<String> filesUnderSnapshot(final FileSystem fs,
final Path snapshotDir)
throws IOException {
return SnapshotReferenceUtil.getHFileNames(conf, fs, snapshotDir);
}
});
} catch (IOException e) {
LOG.error("Failed to create cleaner util", e);
}
}
private static void setupConf(Configuration conf) {
// disable the ui
conf.setInt("hbase.regionsever.info.port", -1);
// change the flush size to a small amount, regulating number of store files
conf.setInt("hbase.hregion.memstore.flush.size", 25000);
// so make sure we get a compaction when doing a load, but keep around some
// files in the store
conf.setInt("hbase.hstore.compaction.min", 10);
conf.setInt("hbase.hstore.compactionThreshold", 10);
// block writes if we get to 12 store files
conf.setInt("hbase.hstore.blockingStoreFiles", 12);
// Enable snapshot
conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
ConstantSizeRegionSplitPolicy.class.getName());
conf.set(SnapshotDescriptionUtils.SNAPSHOT_WORKING_DIR, "file://" + new Path(TEMP_DIR, ".tmpDir").toUri());
}
private static void setupConf(Configuration conf) throws IOException {
// disable the ui
conf.setInt("hbase.regionsever.info.port", -1);
// change the flush size to a small amount, regulating number of store files
conf.setInt("hbase.hregion.memstore.flush.size", 25000);
// so make sure we get a compaction when doing a load, but keep around some
// files in the store
conf.setInt("hbase.hstore.compaction.min", 10);
conf.setInt("hbase.hstore.compactionThreshold", 10);
// block writes if we get to 12 store files
conf.setInt("hbase.hstore.blockingStoreFiles", 12);
// Enable snapshot
conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
ConstantSizeRegionSplitPolicy.class.getName());
String snapshotPath = UTIL.getDefaultRootDirPath().toString() + Path.SEPARATOR +
UUID.randomUUID().toString() + Path.SEPARATOR + ".tmpdir" + Path.SEPARATOR;
conf.set(SnapshotDescriptionUtils.SNAPSHOT_WORKING_DIR, "file://" + new Path(snapshotPath).toUri());
}
private Set<String> getFilesReferencedBySnapshot(String snapshotName) throws IOException {
HashSet<String> files = new HashSet<>();
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
snapshotName, CommonFSUtils.getRootDir(conf));
SnapshotProtos.SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(
fs, snapshotDir);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
// For each region referenced by the snapshot
for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
// For each column family in this region
for (FamilyFiles ff : rm.getFamilyFilesList()) {
// And each store file in that family
for (StoreFile sf : ff.getStoreFilesList()) {
files.add(sf.getName());
}
}
}
return files;
}
/**
* If there is a corrupted region manifest, it should throw out CorruptedSnapshotException,
* instead of an IOException
*/
@Test
public void testCorruptedRegionManifest() throws IOException {
SnapshotTestingUtils.SnapshotMock
snapshotMock = new SnapshotTestingUtils.SnapshotMock(conf, fs, rootDir);
SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(
SNAPSHOT_NAME_STR, TABLE_NAME_STR);
builder.addRegionV2();
builder.corruptOneRegionManifest();
long period = Long.MAX_VALUE;
SnapshotFileCache cache = new SnapshotFileCache(conf, period, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles());
try {
cache.getSnapshotsInProgress();
} finally {
fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir, conf), true);
}
}
/**
* If there is a corrupted data manifest, it should throw out CorruptedSnapshotException,
* instead of an IOException
*/
@Test
public void testCorruptedDataManifest() throws IOException {
SnapshotTestingUtils.SnapshotMock
snapshotMock = new SnapshotTestingUtils.SnapshotMock(conf, fs, rootDir);
SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(
SNAPSHOT_NAME_STR, TABLE_NAME_STR);
builder.addRegionV2();
// consolidate to generate a data.manifest file
builder.consolidate();
builder.corruptDataManifest();
long period = Long.MAX_VALUE;
SnapshotFileCache cache = new SnapshotFileCache(conf, period, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles());
try {
cache.getSnapshotsInProgress();
} finally {
fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir,
TEST_UTIL.getConfiguration()), true);
}
}
@Override
public List<HRegionLocation> getRegionBoundaries(StatementContext context, byte[] tableName) throws SQLException {
String snapshotName;
Configuration conf = context.getConnection().getQueryServices().getConfiguration();
if((snapshotName = getSnapshotName(conf)) != null) {
try {
Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
FileSystem fs = rootDir.getFileSystem(conf);
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
SnapshotDescription snapshotDescription = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDescription);
return getRegionLocationsFromManifest(manifest);
}
catch (Exception e) {
throw new RuntimeException(e);
}
}
else {
return context.getConnection().getQueryServices().getAllTableRegions(tableName);
}
}
@Override
public void preListSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
final SnapshotDescription snapshot) throws IOException {
User user = getActiveUser(ctx);
if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, user)) {
// list it, if user is the owner of snapshot
AuthResult result = AuthResult.allow("listSnapshot " + snapshot.getName(),
"Snapshot owner check allowed", user, null, null, null);
AccessChecker.logResult(result);
} else {
accessChecker.requirePermission(user, "listSnapshot " + snapshot.getName(), null,
Action.ADMIN);
}
}
@Override
public void preRestoreSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final SnapshotDescription snapshot, final TableDescriptor hTableDescriptor)
throws IOException {
User user = getActiveUser(ctx);
if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, user)) {
accessChecker.requirePermission(user, "restoreSnapshot " + snapshot.getName(),
hTableDescriptor.getTableName(), null, null, null, Permission.Action.ADMIN);
} else {
accessChecker.requirePermission(user, "restoreSnapshot " + snapshot.getName(), null,
Action.ADMIN);
}
}
@Override
public void preDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final SnapshotDescription snapshot) throws IOException {
User user = getActiveUser(ctx);
if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, user)) {
// Snapshot owner is allowed to delete the snapshot
AuthResult result = AuthResult.allow("deleteSnapshot " + snapshot.getName(),
"Snapshot owner check allowed", user, null, null, null);
AccessChecker.logResult(result);
} else {
accessChecker.requirePermission(user, "deleteSnapshot " + snapshot.getName(), null,
Action.ADMIN);
}
}
/**
* For the given snapshot, find all files which this {@code snapshotName} references. After a file
* is found to be referenced by the snapshot, it is removed from {@code filesToUpdate} and
* {@code snapshotSizeChanges} is updated in concert.
*
* @param snapshotName The snapshot to check
* @param filesToUpdate A mapping of archived files to their size
* @param snapshotSizeChanges A mapping of snapshots and their change in size
*/
void bucketFilesToSnapshot(
String snapshotName, Map<String,Long> filesToUpdate, Map<String,Long> snapshotSizeChanges)
throws IOException {
// A quick check to avoid doing work if the caller unnecessarily invoked this method.
if (filesToUpdate.isEmpty()) {
return;
}
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
snapshotName, CommonFSUtils.getRootDir(conf));
SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
// For each region referenced by the snapshot
for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
// For each column family in this region
for (FamilyFiles ff : rm.getFamilyFilesList()) {
// And each store file in that family
for (StoreFile sf : ff.getStoreFilesList()) {
Long valueOrNull = filesToUpdate.remove(sf.getName());
if (valueOrNull != null) {
// This storefile was recently archived, we should update this snapshot with its size
snapshotSizeChanges.merge(snapshotName, valueOrNull, Long::sum);
}
// Short-circuit, if we have no more files that were archived, we don't need to iterate
// over the rest of the snapshot.
if (filesToUpdate.isEmpty()) {
return;
}
}
}
}
}
/**
* Action before any real action of restoring from snapshot.
* @param env MasterProcedureEnv
* @throws IOException
*/
private void prepareRestore(final MasterProcedureEnv env) throws IOException {
final TableName tableName = getTableName();
// Checks whether the table exists
if (!MetaTableAccessor.tableExists(env.getMasterServices().getConnection(), tableName)) {
throw new TableNotFoundException(tableName);
}
// Check whether table is disabled.
env.getMasterServices().checkTableModifiable(tableName);
// Check that we have at least 1 CF
if (modifiedTableDescriptor.getColumnFamilyCount() == 0) {
throw new DoNotRetryIOException("Table " + getTableName().toString() +
" should have at least one column family.");
}
if (!getTableName().isSystemTable()) {
// Table already exist. Check and update the region quota for this table namespace.
final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
SnapshotManifest manifest = SnapshotManifest.open(
env.getMasterConfiguration(),
mfs.getFileSystem(),
SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, mfs.getRootDir()),
snapshot);
int snapshotRegionCount = manifest.getRegionManifestsMap().size();
int tableRegionCount =
ProcedureSyncWait.getMasterQuotaManager(env).getRegionCountOfTable(tableName);
if (snapshotRegionCount > 0 && tableRegionCount != snapshotRegionCount) {
ProcedureSyncWait.getMasterQuotaManager(env).checkAndUpdateNamespaceRegionQuota(
tableName, snapshotRegionCount);
}
}
}
/**
* Execute the on-disk Restore
* @param env MasterProcedureEnv
* @throws IOException
**/
private void restoreSnapshot(final MasterProcedureEnv env) throws IOException {
MasterFileSystem fileSystemManager = env.getMasterServices().getMasterFileSystem();
FileSystem fs = fileSystemManager.getFileSystem();
Path rootDir = fileSystemManager.getRootDir();
final ForeignExceptionDispatcher monitorException = new ForeignExceptionDispatcher();
LOG.info("Starting restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot));
try {
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
SnapshotManifest manifest = SnapshotManifest.open(
env.getMasterServices().getConfiguration(), fs, snapshotDir, snapshot);
RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(
env.getMasterServices().getConfiguration(),
fs,
manifest,
modifiedTableDescriptor,
rootDir,
monitorException,
getMonitorStatus());
RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();
regionsToRestore = metaChanges.getRegionsToRestore();
regionsToRemove = metaChanges.getRegionsToRemove();
regionsToAdd = metaChanges.getRegionsToAdd();
parentsToChildrenPairMap = metaChanges.getParentToChildrenPairMap();
} catch (IOException e) {
String msg = "restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot)
+ " failed in on-disk restore. Try re-running the restore command.";
LOG.error(msg, e);
monitorException.receive(
new ForeignException(env.getMasterServices().getServerName().toString(), e));
throw new IOException(msg, e);
}
}
private void restoreSnapshotAcl(final MasterProcedureEnv env) throws IOException {
if (restoreAcl && snapshot.hasUsersAndPermissions() && snapshot.getUsersAndPermissions() != null
&& SnapshotDescriptionUtils
.isSecurityAvailable(env.getMasterServices().getConfiguration())) {
// restore acl of snapshot to table.
RestoreSnapshotHelper.restoreSnapshotAcl(snapshot, TableName.valueOf(snapshot.getTable()),
env.getMasterServices().getConfiguration());
}
}
private void restoreSnapshotAcl(MasterProcedureEnv env) throws IOException {
Configuration conf = env.getMasterServices().getConfiguration();
if (restoreAcl && snapshot.hasUsersAndPermissions() && snapshot.getUsersAndPermissions() != null
&& SnapshotDescriptionUtils.isSecurityAvailable(conf)) {
RestoreSnapshotHelper.restoreSnapshotAcl(snapshot, tableDescriptor.getTableName(), conf);
}
}
/**
* Create a snapshot file cache for all snapshots under the specified [root]/.snapshot on the
* filesystem
* @param fs {@link FileSystem} where the snapshots are stored
* @param rootDir hbase root directory
* @param workingFs {@link FileSystem} where ongoing snapshot mainifest files are stored
* @param workingDir Location to store ongoing snapshot manifest files
* @param cacheRefreshPeriod period (ms) with which the cache should be refreshed
* @param cacheRefreshDelay amount of time to wait for the cache to be refreshed
* @param refreshThreadName name of the cache refresh thread
* @param inspectSnapshotFiles Filter to apply to each snapshot to extract the files.
*/
public SnapshotFileCache(FileSystem fs, Path rootDir, FileSystem workingFs, Path workingDir,
long cacheRefreshPeriod, long cacheRefreshDelay, String refreshThreadName,
SnapshotFileInspector inspectSnapshotFiles) {
this.fs = fs;
this.workingFs = workingFs;
this.workingSnapshotDir = workingDir;
this.fileInspector = inspectSnapshotFiles;
this.snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
// periodically refresh the file cache to make sure we aren't superfluously saving files.
this.refreshTimer = new Timer(refreshThreadName, true);
this.refreshTimer.scheduleAtFixedRate(new RefreshCacheTask(), cacheRefreshDelay,
cacheRefreshPeriod);
}
/**
* Check that the snapshot description written in the filesystem matches the current snapshot
* @param snapshotDir snapshot directory to check
*/
private void verifySnapshotDescription(Path snapshotDir) throws CorruptedSnapshotException {
SnapshotDescription found = SnapshotDescriptionUtils.readSnapshotInfo(workingDirFs,
snapshotDir);
if (!this.snapshot.equals(found)) {
throw new CorruptedSnapshotException(
"Snapshot read (" + found + ") doesn't equal snapshot we ran (" + snapshot + ").",
ProtobufUtil.createSnapshotDesc(snapshot));
}
}
/**
* Cleans up any snapshots in the snapshot/.tmp directory that were left from failed
* snapshot attempts.
*
* @throws IOException if we can't reach the filesystem
*/
private void resetTempDir() throws IOException {
// cleanup any existing snapshots.
Path tmpdir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir,
master.getConfiguration());
FileSystem tmpFs = tmpdir.getFileSystem(master.getConfiguration());
if (!tmpFs.delete(tmpdir, true)) {
LOG.warn("Couldn't delete working snapshot directory: " + tmpdir);
}
}
/**
* Delete the specified snapshot
* @param snapshot
* @throws SnapshotDoesNotExistException If the specified snapshot does not exist.
* @throws IOException For filesystem IOExceptions
*/
public void deleteSnapshot(SnapshotDescription snapshot) throws IOException {
// check to see if it is completed
if (!isSnapshotCompleted(snapshot)) {
throw new SnapshotDoesNotExistException(ProtobufUtil.createSnapshotDesc(snapshot));
}
String snapshotName = snapshot.getName();
// first create the snapshot description and check to see if it exists
FileSystem fs = master.getMasterFileSystem().getFileSystem();
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
// Get snapshot info from file system. The one passed as parameter is a "fake" snapshotInfo with
// just the "name" and it does not contains the "real" snapshot information
snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
// call coproc pre hook
MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
org.apache.hadoop.hbase.client.SnapshotDescription snapshotPOJO = null;
if (cpHost != null) {
snapshotPOJO = ProtobufUtil.createSnapshotDesc(snapshot);
cpHost.preDeleteSnapshot(snapshotPOJO);
}
LOG.debug("Deleting snapshot: " + snapshotName);
// delete the existing snapshot
if (!fs.delete(snapshotDir, true)) {
throw new HBaseSnapshotException("Failed to delete snapshot directory: " + snapshotDir);
}
// call coproc post hook
if (cpHost != null) {
cpHost.postDeleteSnapshot(snapshotPOJO);
}
}
/**
* Check to see if the snapshot is one of the currently completed snapshots
* Returns true if the snapshot exists in the "completed snapshots folder".
*
* @param snapshot expected snapshot to check
* @return <tt>true</tt> if the snapshot is stored on the {@link FileSystem}, <tt>false</tt> if is
* not stored
* @throws IOException if the filesystem throws an unexpected exception,
* @throws IllegalArgumentException if snapshot name is invalid.
*/
private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException {
try {
final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
FileSystem fs = master.getMasterFileSystem().getFileSystem();
// check to see if the snapshot already exists
return fs.exists(snapshotDir);
} catch (IllegalArgumentException iae) {
throw new UnknownSnapshotException("Unexpected exception thrown", iae);
}
}
@Override
public void initialize(MasterServices master, MetricsMaster metricsMaster) throws KeeperException,
IOException, UnsupportedOperationException {
this.master = master;
this.rootDir = master.getMasterFileSystem().getRootDir();
checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
// get the configuration for the coordinator
Configuration conf = master.getConfiguration();
long wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT);
long timeoutMillis = Math.max(
conf.getLong(SnapshotDescriptionUtils.MASTER_SNAPSHOT_TIMEOUT_MILLIS,
SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME),
conf.getLong(SnapshotDescriptionUtils.MASTER_SNAPSHOT_TIMEOUT_MILLIS,
SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME));
int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT);
// setup the default procedure coordinator
String name = master.getServerName().toString();
ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, opThreads);
ProcedureCoordinatorRpcs comms = new ZKProcedureCoordinator(
master.getZooKeeper(), SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, name);
this.coordinator = new ProcedureCoordinator(comms, tpool, timeoutMillis, wakeFrequency);
this.executorService = master.getExecutorService();
resetTempDir();
snapshotHandlerChoreCleanerTask =
scheduleThreadPool.scheduleAtFixedRate(this::cleanupSentinels, 10, 10, TimeUnit.SECONDS);
}
protected static void initCommon() throws Exception {
UTIL.startMiniDFSCluster(1);
fs = UTIL.getDFSCluster().getFileSystem();
rootDir = UTIL.getDefaultRootDirPath();
snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
conf = UTIL.getConfiguration();
}
@Test
public void testFindsSnapshotFilesWhenCleaning() throws IOException {
CommonFSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir());
Path rootDir = CommonFSUtils.getRootDir(conf);
Path archivedHfileDir = new Path(TEST_UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
FileSystem fs = FileSystem.get(conf);
SnapshotHFileCleaner cleaner = new SnapshotHFileCleaner();
cleaner.setConf(conf);
// write an hfile to the snapshot directory
String snapshotName = "snapshot";
final TableName tableName = TableName.valueOf(name.getMethodName());
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
RegionInfo mockRegion = RegionInfoBuilder.newBuilder(tableName).build();
Path regionSnapshotDir = new Path(snapshotDir, mockRegion.getEncodedName());
Path familyDir = new Path(regionSnapshotDir, "family");
// create a reference to a supposedly valid hfile
String hfile = "fd1e73e8a96c486090c5cec07b4894c4";
Path refFile = new Path(familyDir, hfile);
// make sure the reference file exists
fs.create(refFile);
// create the hfile in the archive
fs.mkdirs(archivedHfileDir);
fs.createNewFile(new Path(archivedHfileDir, hfile));
// make sure that the file isn't deletable
assertFalse(cleaner.isFileDeletable(fs.getFileStatus(refFile)));
}