下面列出了org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils#readSnapshotInfo ( ) 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
/**
* get region infos
*
* @param zookeeperQuorum zookeeper quorum
* @param zookeeperClientPort zookeeper client port
* @param hBaseRootDir HBase root dir
* @param snapshotName snapshot name
* @return region info list
* @throws IOException IOException
*/
public static List<HRegionInfo> getRegionInfos(String zookeeperQuorum, String zookeeperClientPort,
String hBaseRootDir, String snapshotName) throws IOException {
try {
Configuration conf = Utils.getHadoopConf(zookeeperQuorum, zookeeperClientPort);
Path root = new Path(hBaseRootDir);
FileSystem fs = FileSystem.get(conf);
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, root);
HBaseProtos.SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
return Utils.getRegionInfosFromManifest(manifest);
} catch (IOException ex) {
logger.error("get region info error: " + ex.getMessage(), ex);
throw ex;
}
}
/**
* Get table descriptor
* @param tableName is the table backed up
* @return {@link TableDescriptor} saved in backup image of the table
*/
TableDescriptor getTableDesc(TableName tableName) throws IOException {
Path tableInfoPath = this.getTableInfoPath(tableName);
SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, tableInfoPath);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, tableInfoPath, desc);
TableDescriptor tableDescriptor = manifest.getTableDescriptor();
if (!tableDescriptor.getTableName().equals(tableName)) {
LOG.error("couldn't find Table Desc for table: " + tableName + " under tableInfoPath: "
+ tableInfoPath.toString());
LOG.error("tableDescriptor.getNameAsString() = "
+ tableDescriptor.getTableName().getNameAsString());
throw new FileNotFoundException("couldn't find Table Desc for table: " + tableName
+ " under tableInfoPath: " + tableInfoPath.toString());
}
return tableDescriptor;
}
private void openWithoutRestoringSnapshot() throws IOException {
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
SnapshotProtos.SnapshotDescription snapshotDesc =
SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
if (regionManifests == null) {
throw new IllegalArgumentException("Snapshot seems empty, snapshotName: " + snapshotName);
}
regions = new ArrayList<>(regionManifests.size());
regionManifests.stream().map(r -> ProtobufUtil.toRegionInfo(r.getRegionInfo()))
.filter(this::isValidRegion).sorted().forEach(r -> regions.add(r));
htd = manifest.getTableDescriptor();
}
private Set<String> getFilesReferencedBySnapshot(String snapshotName) throws IOException {
HashSet<String> files = new HashSet<>();
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
snapshotName, CommonFSUtils.getRootDir(conf));
SnapshotProtos.SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(
fs, snapshotDir);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
// For each region referenced by the snapshot
for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
// For each column family in this region
for (FamilyFiles ff : rm.getFamilyFilesList()) {
// And each store file in that family
for (StoreFile sf : ff.getStoreFilesList()) {
files.add(sf.getName());
}
}
}
return files;
}
@Override
public List<HRegionLocation> getRegionBoundaries(StatementContext context, byte[] tableName) throws SQLException {
String snapshotName;
Configuration conf = context.getConnection().getQueryServices().getConfiguration();
if((snapshotName = getSnapshotName(conf)) != null) {
try {
Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
FileSystem fs = rootDir.getFileSystem(conf);
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
SnapshotDescription snapshotDescription = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDescription);
return getRegionLocationsFromManifest(manifest);
}
catch (Exception e) {
throw new RuntimeException(e);
}
}
else {
return context.getConnection().getQueryServices().getAllTableRegions(tableName);
}
}
/**
* For the given snapshot, find all files which this {@code snapshotName} references. After a file
* is found to be referenced by the snapshot, it is removed from {@code filesToUpdate} and
* {@code snapshotSizeChanges} is updated in concert.
*
* @param snapshotName The snapshot to check
* @param filesToUpdate A mapping of archived files to their size
* @param snapshotSizeChanges A mapping of snapshots and their change in size
*/
void bucketFilesToSnapshot(
String snapshotName, Map<String,Long> filesToUpdate, Map<String,Long> snapshotSizeChanges)
throws IOException {
// A quick check to avoid doing work if the caller unnecessarily invoked this method.
if (filesToUpdate.isEmpty()) {
return;
}
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
snapshotName, CommonFSUtils.getRootDir(conf));
SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
// For each region referenced by the snapshot
for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
// For each column family in this region
for (FamilyFiles ff : rm.getFamilyFilesList()) {
// And each store file in that family
for (StoreFile sf : ff.getStoreFilesList()) {
Long valueOrNull = filesToUpdate.remove(sf.getName());
if (valueOrNull != null) {
// This storefile was recently archived, we should update this snapshot with its size
snapshotSizeChanges.merge(snapshotName, valueOrNull, Long::sum);
}
// Short-circuit, if we have no more files that were archived, we don't need to iterate
// over the rest of the snapshot.
if (filesToUpdate.isEmpty()) {
return;
}
}
}
}
}
/**
* Check that the snapshot description written in the filesystem matches the current snapshot
* @param snapshotDir snapshot directory to check
*/
private void verifySnapshotDescription(Path snapshotDir) throws CorruptedSnapshotException {
SnapshotDescription found = SnapshotDescriptionUtils.readSnapshotInfo(workingDirFs,
snapshotDir);
if (!this.snapshot.equals(found)) {
throw new CorruptedSnapshotException(
"Snapshot read (" + found + ") doesn't equal snapshot we ran (" + snapshot + ").",
ProtobufUtil.createSnapshotDesc(snapshot));
}
}
/**
* Delete the specified snapshot
* @param snapshot
* @throws SnapshotDoesNotExistException If the specified snapshot does not exist.
* @throws IOException For filesystem IOExceptions
*/
public void deleteSnapshot(SnapshotDescription snapshot) throws IOException {
// check to see if it is completed
if (!isSnapshotCompleted(snapshot)) {
throw new SnapshotDoesNotExistException(ProtobufUtil.createSnapshotDesc(snapshot));
}
String snapshotName = snapshot.getName();
// first create the snapshot description and check to see if it exists
FileSystem fs = master.getMasterFileSystem().getFileSystem();
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
// Get snapshot info from file system. The one passed as parameter is a "fake" snapshotInfo with
// just the "name" and it does not contains the "real" snapshot information
snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
// call coproc pre hook
MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
org.apache.hadoop.hbase.client.SnapshotDescription snapshotPOJO = null;
if (cpHost != null) {
snapshotPOJO = ProtobufUtil.createSnapshotDesc(snapshot);
cpHost.preDeleteSnapshot(snapshotPOJO);
}
LOG.debug("Deleting snapshot: " + snapshotName);
// delete the existing snapshot
if (!fs.delete(snapshotDir, true)) {
throw new HBaseSnapshotException("Failed to delete snapshot directory: " + snapshotDir);
}
// call coproc post hook
if (cpHost != null) {
cpHost.postDeleteSnapshot(snapshotPOJO);
}
}
private void createAndRestoreTable(Connection conn, TableName tableName, TableName newTableName,
Path tableBackupPath, boolean truncateIfExists, String lastIncrBackupId) throws IOException {
if (newTableName == null) {
newTableName = tableName;
}
FileSystem fileSys = tableBackupPath.getFileSystem(this.conf);
// get table descriptor first
TableDescriptor tableDescriptor = getTableDescriptor(fileSys, tableName, lastIncrBackupId);
if (tableDescriptor != null) {
LOG.debug("Retrieved descriptor: " + tableDescriptor + " thru " + lastIncrBackupId);
}
if (tableDescriptor == null) {
Path tableSnapshotPath = getTableSnapshotPath(backupRootPath, tableName, backupId);
if (fileSys.exists(tableSnapshotPath)) {
// snapshot path exist means the backup path is in HDFS
// check whether snapshot dir already recorded for target table
if (snapshotMap.get(tableName) != null) {
SnapshotDescription desc =
SnapshotDescriptionUtils.readSnapshotInfo(fileSys, tableSnapshotPath);
SnapshotManifest manifest = SnapshotManifest.open(conf, fileSys, tableSnapshotPath, desc);
tableDescriptor = manifest.getTableDescriptor();
} else {
tableDescriptor = getTableDesc(tableName);
snapshotMap.put(tableName, getTableInfoPath(tableName));
}
if (tableDescriptor == null) {
LOG.debug("Found no table descriptor in the snapshot dir, previous schema would be lost");
}
} else {
throw new IOException("Table snapshot directory: " +
tableSnapshotPath + " does not exist.");
}
}
Path tableArchivePath = getTableArchivePath(tableName);
if (tableArchivePath == null) {
if (tableDescriptor != null) {
// find table descriptor but no archive dir means the table is empty, create table and exit
if (LOG.isDebugEnabled()) {
LOG.debug("find table descriptor but no archive dir for table " + tableName
+ ", will only create table");
}
tableDescriptor = TableDescriptorBuilder.copy(newTableName, tableDescriptor);
checkAndCreateTable(conn, tableBackupPath, tableName, newTableName, null, tableDescriptor,
truncateIfExists);
return;
} else {
throw new IllegalStateException("Cannot restore hbase table because directory '"
+ " tableArchivePath is null.");
}
}
if (tableDescriptor == null) {
tableDescriptor = TableDescriptorBuilder.newBuilder(newTableName).build();
} else {
tableDescriptor = TableDescriptorBuilder.copy(newTableName, tableDescriptor);
}
// record all region dirs:
// load all files in dir
try {
ArrayList<Path> regionPathList = getRegionList(tableName);
// should only try to create the table with all region informations, so we could pre-split
// the regions in fine grain
checkAndCreateTable(conn, tableBackupPath, tableName, newTableName, regionPathList,
tableDescriptor, truncateIfExists);
RestoreJob restoreService = BackupRestoreFactory.getRestoreJob(conf);
Path[] paths = new Path[regionPathList.size()];
regionPathList.toArray(paths);
restoreService.run(paths, new TableName[]{tableName}, new TableName[] {newTableName}, true);
} catch (Exception e) {
LOG.error(e.toString(), e);
throw new IllegalStateException("Cannot restore hbase table", e);
}
}
/**
* Restore or Clone the specified snapshot
* @param reqSnapshot
* @param nonceKey unique identifier to prevent duplicated RPC
* @throws IOException
*/
public long restoreOrCloneSnapshot(final SnapshotDescription reqSnapshot, final NonceKey nonceKey,
final boolean restoreAcl) throws IOException {
FileSystem fs = master.getMasterFileSystem().getFileSystem();
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);
// check if the snapshot exists
if (!fs.exists(snapshotDir)) {
LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist.");
throw new SnapshotDoesNotExistException(
ProtobufUtil.createSnapshotDesc(reqSnapshot));
}
// Get snapshot info from file system. The reqSnapshot is a "fake" snapshotInfo with
// just the snapshot "name" and table name to restore. It does not contains the "real" snapshot
// information.
SnapshotDescription snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
SnapshotManifest manifest = SnapshotManifest.open(master.getConfiguration(), fs,
snapshotDir, snapshot);
TableDescriptor snapshotTableDesc = manifest.getTableDescriptor();
TableName tableName = TableName.valueOf(reqSnapshot.getTable());
// sanity check the new table descriptor
TableDescriptorChecker.sanityCheck(master.getConfiguration(), snapshotTableDesc);
// stop tracking "abandoned" handlers
cleanupSentinels();
// Verify snapshot validity
SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, manifest);
// Execute the restore/clone operation
long procId;
if (MetaTableAccessor.tableExists(master.getConnection(), tableName)) {
procId = restoreSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey,
restoreAcl);
} else {
procId =
cloneSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey, restoreAcl);
}
return procId;
}
public static SnapshotManifest getSnapshotManifest(Configuration conf, String snapshotName,
Path rootDir, FileSystem fs) throws IOException {
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
return SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
}