下面列出了org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils#getCompletedSnapshotDir ( ) 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
/**
* get region infos
*
* @param zookeeperQuorum zookeeper quorum
* @param zookeeperClientPort zookeeper client port
* @param hBaseRootDir HBase root dir
* @param snapshotName snapshot name
* @return region info list
* @throws IOException IOException
*/
public static List<HRegionInfo> getRegionInfos(String zookeeperQuorum, String zookeeperClientPort,
String hBaseRootDir, String snapshotName) throws IOException {
try {
Configuration conf = Utils.getHadoopConf(zookeeperQuorum, zookeeperClientPort);
Path root = new Path(hBaseRootDir);
FileSystem fs = FileSystem.get(conf);
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, root);
HBaseProtos.SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
return Utils.getRegionInfosFromManifest(manifest);
} catch (IOException ex) {
logger.error("get region info error: " + ex.getMessage(), ex);
throw ex;
}
}
private void openWithoutRestoringSnapshot() throws IOException {
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
SnapshotProtos.SnapshotDescription snapshotDesc =
SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
if (regionManifests == null) {
throw new IllegalArgumentException("Snapshot seems empty, snapshotName: " + snapshotName);
}
regions = new ArrayList<>(regionManifests.size());
regionManifests.stream().map(r -> ProtobufUtil.toRegionInfo(r.getRegionInfo()))
.filter(this::isValidRegion).sorted().forEach(r -> regions.add(r));
htd = manifest.getTableDescriptor();
}
private Set<String> getFilesReferencedBySnapshot(String snapshotName) throws IOException {
HashSet<String> files = new HashSet<>();
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
snapshotName, CommonFSUtils.getRootDir(conf));
SnapshotProtos.SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(
fs, snapshotDir);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
// For each region referenced by the snapshot
for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
// For each column family in this region
for (FamilyFiles ff : rm.getFamilyFilesList()) {
// And each store file in that family
for (StoreFile sf : ff.getStoreFilesList()) {
files.add(sf.getName());
}
}
}
return files;
}
@Override
public List<HRegionLocation> getRegionBoundaries(StatementContext context, byte[] tableName) throws SQLException {
String snapshotName;
Configuration conf = context.getConnection().getQueryServices().getConfiguration();
if((snapshotName = getSnapshotName(conf)) != null) {
try {
Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
FileSystem fs = rootDir.getFileSystem(conf);
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
SnapshotDescription snapshotDescription = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDescription);
return getRegionLocationsFromManifest(manifest);
}
catch (Exception e) {
throw new RuntimeException(e);
}
}
else {
return context.getConnection().getQueryServices().getAllTableRegions(tableName);
}
}
/**
* For the given snapshot, find all files which this {@code snapshotName} references. After a file
* is found to be referenced by the snapshot, it is removed from {@code filesToUpdate} and
* {@code snapshotSizeChanges} is updated in concert.
*
* @param snapshotName The snapshot to check
* @param filesToUpdate A mapping of archived files to their size
* @param snapshotSizeChanges A mapping of snapshots and their change in size
*/
void bucketFilesToSnapshot(
String snapshotName, Map<String,Long> filesToUpdate, Map<String,Long> snapshotSizeChanges)
throws IOException {
// A quick check to avoid doing work if the caller unnecessarily invoked this method.
if (filesToUpdate.isEmpty()) {
return;
}
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
snapshotName, CommonFSUtils.getRootDir(conf));
SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
// For each region referenced by the snapshot
for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
// For each column family in this region
for (FamilyFiles ff : rm.getFamilyFilesList()) {
// And each store file in that family
for (StoreFile sf : ff.getStoreFilesList()) {
Long valueOrNull = filesToUpdate.remove(sf.getName());
if (valueOrNull != null) {
// This storefile was recently archived, we should update this snapshot with its size
snapshotSizeChanges.merge(snapshotName, valueOrNull, Long::sum);
}
// Short-circuit, if we have no more files that were archived, we don't need to iterate
// over the rest of the snapshot.
if (filesToUpdate.isEmpty()) {
return;
}
}
}
}
}
/**
* Execute the on-disk Restore
* @param env MasterProcedureEnv
* @throws IOException
**/
private void restoreSnapshot(final MasterProcedureEnv env) throws IOException {
MasterFileSystem fileSystemManager = env.getMasterServices().getMasterFileSystem();
FileSystem fs = fileSystemManager.getFileSystem();
Path rootDir = fileSystemManager.getRootDir();
final ForeignExceptionDispatcher monitorException = new ForeignExceptionDispatcher();
LOG.info("Starting restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot));
try {
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
SnapshotManifest manifest = SnapshotManifest.open(
env.getMasterServices().getConfiguration(), fs, snapshotDir, snapshot);
RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(
env.getMasterServices().getConfiguration(),
fs,
manifest,
modifiedTableDescriptor,
rootDir,
monitorException,
getMonitorStatus());
RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();
regionsToRestore = metaChanges.getRegionsToRestore();
regionsToRemove = metaChanges.getRegionsToRemove();
regionsToAdd = metaChanges.getRegionsToAdd();
parentsToChildrenPairMap = metaChanges.getParentToChildrenPairMap();
} catch (IOException e) {
String msg = "restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot)
+ " failed in on-disk restore. Try re-running the restore command.";
LOG.error(msg, e);
monitorException.receive(
new ForeignException(env.getMasterServices().getServerName().toString(), e));
throw new IOException(msg, e);
}
}
/**
* Delete the specified snapshot
* @param snapshot
* @throws SnapshotDoesNotExistException If the specified snapshot does not exist.
* @throws IOException For filesystem IOExceptions
*/
public void deleteSnapshot(SnapshotDescription snapshot) throws IOException {
// check to see if it is completed
if (!isSnapshotCompleted(snapshot)) {
throw new SnapshotDoesNotExistException(ProtobufUtil.createSnapshotDesc(snapshot));
}
String snapshotName = snapshot.getName();
// first create the snapshot description and check to see if it exists
FileSystem fs = master.getMasterFileSystem().getFileSystem();
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
// Get snapshot info from file system. The one passed as parameter is a "fake" snapshotInfo with
// just the "name" and it does not contains the "real" snapshot information
snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
// call coproc pre hook
MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
org.apache.hadoop.hbase.client.SnapshotDescription snapshotPOJO = null;
if (cpHost != null) {
snapshotPOJO = ProtobufUtil.createSnapshotDesc(snapshot);
cpHost.preDeleteSnapshot(snapshotPOJO);
}
LOG.debug("Deleting snapshot: " + snapshotName);
// delete the existing snapshot
if (!fs.delete(snapshotDir, true)) {
throw new HBaseSnapshotException("Failed to delete snapshot directory: " + snapshotDir);
}
// call coproc post hook
if (cpHost != null) {
cpHost.postDeleteSnapshot(snapshotPOJO);
}
}
/**
* Check to see if the snapshot is one of the currently completed snapshots
* Returns true if the snapshot exists in the "completed snapshots folder".
*
* @param snapshot expected snapshot to check
* @return <tt>true</tt> if the snapshot is stored on the {@link FileSystem}, <tt>false</tt> if is
* not stored
* @throws IOException if the filesystem throws an unexpected exception,
* @throws IllegalArgumentException if snapshot name is invalid.
*/
private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException {
try {
final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
FileSystem fs = master.getMasterFileSystem().getFileSystem();
// check to see if the snapshot already exists
return fs.exists(snapshotDir);
} catch (IllegalArgumentException iae) {
throw new UnknownSnapshotException("Unexpected exception thrown", iae);
}
}
@Test
public void testFindsSnapshotFilesWhenCleaning() throws IOException {
CommonFSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir());
Path rootDir = CommonFSUtils.getRootDir(conf);
Path archivedHfileDir = new Path(TEST_UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
FileSystem fs = FileSystem.get(conf);
SnapshotHFileCleaner cleaner = new SnapshotHFileCleaner();
cleaner.setConf(conf);
// write an hfile to the snapshot directory
String snapshotName = "snapshot";
final TableName tableName = TableName.valueOf(name.getMethodName());
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
RegionInfo mockRegion = RegionInfoBuilder.newBuilder(tableName).build();
Path regionSnapshotDir = new Path(snapshotDir, mockRegion.getEncodedName());
Path familyDir = new Path(regionSnapshotDir, "family");
// create a reference to a supposedly valid hfile
String hfile = "fd1e73e8a96c486090c5cec07b4894c4";
Path refFile = new Path(familyDir, hfile);
// make sure the reference file exists
fs.create(refFile);
// create the hfile in the archive
fs.mkdirs(archivedHfileDir);
fs.createNewFile(new Path(archivedHfileDir, hfile));
// make sure that the file isn't deletable
assertFalse(cleaner.isFileDeletable(fs.getFileStatus(refFile)));
}
/**
* Restore or Clone the specified snapshot
* @param reqSnapshot
* @param nonceKey unique identifier to prevent duplicated RPC
* @throws IOException
*/
public long restoreOrCloneSnapshot(final SnapshotDescription reqSnapshot, final NonceKey nonceKey,
final boolean restoreAcl) throws IOException {
FileSystem fs = master.getMasterFileSystem().getFileSystem();
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);
// check if the snapshot exists
if (!fs.exists(snapshotDir)) {
LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist.");
throw new SnapshotDoesNotExistException(
ProtobufUtil.createSnapshotDesc(reqSnapshot));
}
// Get snapshot info from file system. The reqSnapshot is a "fake" snapshotInfo with
// just the snapshot "name" and table name to restore. It does not contains the "real" snapshot
// information.
SnapshotDescription snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
SnapshotManifest manifest = SnapshotManifest.open(master.getConfiguration(), fs,
snapshotDir, snapshot);
TableDescriptor snapshotTableDesc = manifest.getTableDescriptor();
TableName tableName = TableName.valueOf(reqSnapshot.getTable());
// sanity check the new table descriptor
TableDescriptorChecker.sanityCheck(master.getConfiguration(), snapshotTableDesc);
// stop tracking "abandoned" handlers
cleanupSentinels();
// Verify snapshot validity
SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, manifest);
// Execute the restore/clone operation
long procId;
if (MetaTableAccessor.tableExists(master.getConnection(), tableName)) {
procId = restoreSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey,
restoreAcl);
} else {
procId =
cloneSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey, restoreAcl);
}
return procId;
}
public static SnapshotManifest getSnapshotManifest(Configuration conf, String snapshotName,
Path rootDir, FileSystem fs) throws IOException {
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
return SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
}