下面列出了org.apache.hadoop.fs.Path#SEPARATOR_CHAR 实例代码，或者点击链接到github查看源代码，也可以在右侧发表评论。
static private Map<String, Lease> findLeaseWithPrefixPath(
String prefix, SortedMap<String, Lease> path2lease) {
if (LOG.isDebugEnabled()) {
LOG.debug(LeaseManager.class.getSimpleName() + ".findLease: prefix=" + prefix);
}
final Map<String, Lease> entries = new HashMap<String, Lease>();
int srclen = prefix.length();
// prefix may ended with '/'
if (prefix.charAt(srclen - 1) == Path.SEPARATOR_CHAR) {
srclen -= 1;
}
for(Map.Entry<String, Lease> entry : path2lease.tailMap(prefix).entrySet()) {
final String p = entry.getKey();
if (!p.startsWith(prefix)) {
return entries;
}
if (p.length() == srclen || p.charAt(srclen) == Path.SEPARATOR_CHAR) {
entries.put(entry.getKey(), entry.getValue());
}
}
return entries;
}
private static int findLongestDirPrefix(String cwd, String path, boolean isDir) {
// add the path separator to dirs to simplify finding the longest match
if (!cwd.endsWith(Path.SEPARATOR)) {
cwd += Path.SEPARATOR;
}
if (isDir && !path.endsWith(Path.SEPARATOR)) {
path += Path.SEPARATOR;
}
// find longest directory prefix
int len = Math.min(cwd.length(), path.length());
int lastSep = -1;
for (int i=0; i < len; i++) {
if (cwd.charAt(i) != path.charAt(i)) break;
if (cwd.charAt(i) == Path.SEPARATOR_CHAR) lastSep = i;
}
return lastSep;
}
/**
* Get the relative path name with respect to the root of the volume.
* @param absolutePathName The absolute path name
* @param volume Root of the volume.
* @return null if the absolute path name is outside of the volume.
*/
private static String getRelativePathName(String absolutePathName,
String volume) {
absolutePathName = normalizePath(absolutePathName);
// Get the file names
if (!absolutePathName.startsWith(volume)) {
return null;
}
// Get rid of the volume prefix
String fileName = absolutePathName.substring(volume.length());
if (fileName.charAt(0) == Path.SEPARATOR_CHAR) {
fileName = fileName.substring(1);
}
return fileName;
}
static private List<Map.Entry<String, Lease>> findLeaseWithPrefixPath(
String prefix, SortedMap<String, Lease> path2lease) {
if (LOG.isDebugEnabled()) {
LOG.debug(LeaseManager.class.getSimpleName() + ".findLease: prefix=" + prefix);
}
List<Map.Entry<String, Lease>> entries = new ArrayList<Map.Entry<String, Lease>>();
final int srclen = prefix.length();
for(Map.Entry<String, Lease> entry : path2lease.tailMap(prefix).entrySet()) {
final String p = entry.getKey();
if (!p.startsWith(prefix)) {
return entries;
}
if (p.length() == srclen || p.charAt(srclen) == Path.SEPARATOR_CHAR) {
entries.add(entry);
}
}
return entries;
}
/**
* Get the relative path name with respect to the root of the volume.
* @param absolutePathName The absolute path name
* @param volume Root of the volume.
* @return null if the absolute path name is outside of the volume.
*/
private String getRelativePathName(String absolutePathName,
String volume) {
absolutePathName = normalizePath(absolutePathName);
// Get the file names
if (!absolutePathName.startsWith(volume)) {
return null;
}
// Get rid of the volume prefix
String fileName = absolutePathName.substring(volume.length());
if (fileName.charAt(0) == Path.SEPARATOR_CHAR) {
fileName = fileName.substring(1);
}
return fileName;
}
static private Map<String, Lease> findLeaseWithPrefixPath(
String prefix, SortedMap<String, Lease> path2lease) {
if (LOG.isDebugEnabled()) {
LOG.debug(LeaseManager.class.getSimpleName() + ".findLease: prefix=" + prefix);
}
final Map<String, Lease> entries = new HashMap<String, Lease>();
int srclen = prefix.length();
// prefix may ended with '/'
if (prefix.charAt(srclen - 1) == Path.SEPARATOR_CHAR) {
srclen -= 1;
}
for(Map.Entry<String, Lease> entry : path2lease.tailMap(prefix).entrySet()) {
final String p = entry.getKey();
if (!p.startsWith(prefix)) {
return entries;
}
if (p.length() == srclen || p.charAt(srclen) == Path.SEPARATOR_CHAR) {
entries.put(entry.getKey(), entry.getValue());
}
}
return entries;
}
private static void validateDestination(
String src, String dst, INode srcInode)
throws IOException {
String error;
if (srcInode.isSymlink() &&
dst.equals(srcInode.asSymlink().getSymlinkString())) {
throw new FileAlreadyExistsException("Cannot rename symlink " + src
+ " to its target " + dst);
}
// dst cannot be a directory or a file under src
if (dst.startsWith(src)
&& dst.charAt(src.length()) == Path.SEPARATOR_CHAR) {
error = "Rename destination " + dst
+ " is a directory or file under source " + src;
NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
+ error);
throw new IOException(error);
}
}
private static int findLongestDirPrefix(String cwd, String path, boolean isDir) {
// add the path separator to dirs to simplify finding the longest match
if (!cwd.endsWith(Path.SEPARATOR)) {
cwd += Path.SEPARATOR;
}
if (isDir && !path.endsWith(Path.SEPARATOR)) {
path += Path.SEPARATOR;
}
// find longest directory prefix
int len = Math.min(cwd.length(), path.length());
int lastSep = -1;
for (int i=0; i < len; i++) {
if (cwd.charAt(i) != path.charAt(i)) break;
if (cwd.charAt(i) == Path.SEPARATOR_CHAR) lastSep = i;
}
return lastSep;
}
/**
* This test creates some directories inside the toBeDeleted directory and
* then start the asyncDiskService.
* AsyncDiskService will create tasks to delete the content inside the
* toBeDeleted directories.
*/
@Test
public void testMRAsyncDiskServiceStartupCleaning() throws Throwable {
FileSystem localFileSystem = FileSystem.getLocal(new Configuration());
String[] vols = new String[]{TEST_ROOT_DIR + "/0",
TEST_ROOT_DIR + "/1"};
String a = "a";
String b = "b";
String c = "b/c";
String d = "d";
// Create directories inside SUBDIR
String suffix = Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED;
File fa = new File(vols[0] + suffix, a);
File fb = new File(vols[1] + suffix, b);
File fc = new File(vols[1] + suffix, c);
File fd = new File(vols[1] + suffix, d);
// Create the directories
fa.mkdirs();
fb.mkdirs();
fc.mkdirs();
fd.mkdirs();
assertTrue(fa.exists());
assertTrue(fb.exists());
assertTrue(fc.exists());
assertTrue(fd.exists());
// Create the asyncDiskService which will delete all contents inside SUBDIR
MRAsyncDiskService service = new MRAsyncDiskService(
localFileSystem, vols);
// Make sure everything is cleaned up
makeSureCleanedUp(vols, service);
}
/**
* This test creates some directories inside the toBeDeleted directory and
* then start the asyncDiskService.
* AsyncDiskService will create tasks to delete the content inside the
* toBeDeleted directories.
*/
public void testMRAsyncDiskServiceStartupCleaning() throws Throwable {
FileSystem localFileSystem = FileSystem.getLocal(new Configuration());
String[] vols = new String[]{TEST_ROOT_DIR + "/0",
TEST_ROOT_DIR + "/1"};
String a = "a";
String b = "b";
String c = "b/c";
String d = "d";
// Create directories inside SUBDIR
File fa = new File(vols[0] + Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED, a);
File fb = new File(vols[1] + Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED, b);
File fc = new File(vols[1] + Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED, c);
File fd = new File(vols[1] + Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED, d);
// Create the directories
fa.mkdirs();
fb.mkdirs();
fc.mkdirs();
fd.mkdirs();
assertTrue(fa.exists());
assertTrue(fb.exists());
assertTrue(fc.exists());
assertTrue(fd.exists());
// Create the asyncDiskService which will delete all contents inside SUBDIR
MRAsyncDiskService service = new MRAsyncDiskService(
localFileSystem, vols);
// Make sure everything is cleaned up
makeSureCleanedUp(vols, service);
}
public static String getSnapshotPath(String snapshottableDir,
String snapshotRelativePath) {
final StringBuilder b = new StringBuilder(snapshottableDir);
if (b.charAt(b.length() - 1) != Path.SEPARATOR_CHAR) {
b.append(Path.SEPARATOR);
}
return b.append(HdfsConstants.DOT_SNAPSHOT_DIR)
.append(Path.SEPARATOR)
.append(snapshotRelativePath)
.toString();
}
@VisibleForTesting
String[] getPathElements(String path) {
path = path.trim();
if (path.charAt(0) != Path.SEPARATOR_CHAR) {
throw new IllegalArgumentException("It must be an absolute path: " +
path);
}
int numOfElements = StringUtils.countMatches(path, Path.SEPARATOR);
if (path.length() > 1 && path.endsWith(Path.SEPARATOR)) {
numOfElements--;
}
String[] pathElements = new String[numOfElements];
int elementIdx = 0;
int idx = 0;
int found = path.indexOf(Path.SEPARATOR_CHAR, idx);
while (found > -1) {
if (found > idx) {
pathElements[elementIdx++] = path.substring(idx, found);
}
idx = found + 1;
found = path.indexOf(Path.SEPARATOR_CHAR, idx);
}
if (idx < path.length()) {
pathElements[elementIdx] = path.substring(idx);
}
return pathElements;
}
public static String getSnapshotPath(String snapshottableDir,
String snapshotRelativePath) {
final StringBuilder b = new StringBuilder(snapshottableDir);
if (b.charAt(b.length() - 1) != Path.SEPARATOR_CHAR) {
b.append(Path.SEPARATOR);
}
return b.append(HdfsConstants.DOT_SNAPSHOT_DIR)
.append(Path.SEPARATOR)
.append(snapshotRelativePath)
.toString();
}
private String getCacheEntryFilePath(String checksum, String filename) {
return SharedCacheUtil.getCacheEntryPath(this.cacheDepth,
this.cacheRoot, checksum) + Path.SEPARATOR_CHAR + filename;
}
/**
* Move the path name on one volume to a temporary location and then
* delete them.
*
* This functions returns when the moves are done, but not necessarily all
* deletions are done. This is usually good enough because applications
* won't see the path name under the old name anyway after the move.
*
* @param volume The disk volume
* @param pathName The path name relative to volume root.
* @throws IOException If the move failed
* @return false if the file is not found
*/
public boolean moveAndDeleteRelativePath(String volume, String pathName)
throws IOException {
volume = normalizePath(volume);
// Move the file right now, so that it can be deleted later
String newPathName =
format.format(new Date()) + "_" + uniqueId.getAndIncrement();
newPathName = TOBEDELETED + Path.SEPARATOR_CHAR + newPathName;
Path source = new Path(volume, pathName);
Path target = new Path(volume, newPathName);
try {
if (!localFileSystem.rename(source, target)) {
// If the source does not exists, return false.
// This is necessary because rename can return false if the source
// does not exists.
if (!localFileSystem.exists(source)) {
return false;
}
// Try to recreate the parent directory just in case it gets deleted.
if (!localFileSystem.mkdirs(new Path(volume, TOBEDELETED))) {
throw new IOException("Cannot create " + TOBEDELETED + " under "
+ volume);
}
// Try rename again. If it fails, return false.
if (!localFileSystem.rename(source, target)) {
throw new IOException("Cannot rename " + source + " to "
+ target);
}
}
} catch (FileNotFoundException e) {
// Return false in case that the file is not found.
return false;
}
DeleteTask task = new DeleteTask(volume, pathName, newPathName);
execute(volume, task);
return true;
}
public static byte[][] splitAndGetPathComponents(String str) {
try {
final int len;
if (str == null
|| ((len = str.length()) == 0)
|| str.charAt(0) != Path.SEPARATOR_CHAR) {
return null;
}
char[] charArray = UTF8.getCharArray(len);
str.getChars(0, len, charArray, 0);
// allocate list for components
List<byte[]> componentByteList = new ArrayList<byte[]>(20);
// for each component to know if it has only ascii chars
boolean canFastConvert = true;
int startIndex = 0;
for (int i = 0; i < len; i++) {
if (charArray[i] == Path.SEPARATOR_CHAR) {
componentByteList.add(extractBytes(str, startIndex, i, charArray,
canFastConvert));
startIndex = i + 1;
// assume only ASCII chars
canFastConvert = true;
} else if (charArray[i] > UTF8.MAX_ASCII_CODE) {
// found non-ASCII char, we will do
// full conversion for this component
canFastConvert = false;
}
}
// the case when the last component is a filename ("/" not at the end)
if (charArray[len - 1] != Path.SEPARATOR_CHAR) {
componentByteList.add(extractBytes(str, startIndex, len, charArray,
canFastConvert));
}
int last = componentByteList.size(); // last split
while (--last>=1 && componentByteList.get(last).length == 0) {
componentByteList.remove(last);
}
return componentByteList.toArray(new byte[last+1][]);
} catch (UnsupportedEncodingException e) {
return null;
}
}
private static Path getDirLockFile(Path dir) {
return new Path(dir.toString() + Path.SEPARATOR_CHAR + DIR_LOCK_FILE );
}
/**
* Move the path name on one volume to a temporary location and then
* delete them.
*
* This functions returns when the moves are done, but not necessarily all
* deletions are done. This is usually good enough because applications
* won't see the path name under the old name anyway after the move.
*
* @param volume The disk volume
* @param pathName The path name relative to volume root.
* @throws IOException If the move failed
* @return false if the file is not found
*/
public boolean moveAndDeleteRelativePath(String volume, String pathName)
throws IOException {
volume = normalizePath(volume);
// Move the file right now, so that it can be deleted later
String newPathName =
format.format(new Date()) + "_" + uniqueId.getAndIncrement();
newPathName = TOBEDELETED + Path.SEPARATOR_CHAR + newPathName;
Path source = new Path(volume, pathName);
Path target = new Path(volume, newPathName);
try {
if (!localFileSystem.rename(source, target)) {
// If the source does not exists, return false.
// This is necessary because rename can return false if the source
// does not exists.
if (!localFileSystem.exists(source)) {
return false;
}
// Try to recreate the parent directory just in case it gets deleted.
if (!localFileSystem.mkdirs(new Path(volume, TOBEDELETED))) {
throw new IOException("Cannot create " + TOBEDELETED + " under "
+ volume);
}
// Try rename again. If it fails, return false.
if (!localFileSystem.rename(source, target)) {
throw new IOException("Cannot rename " + source + " to "
+ target);
}
}
} catch (FileNotFoundException e) {
// Return false in case that the file is not found.
return false;
}
DeleteTask task = new DeleteTask(volume, pathName, newPathName);
execute(volume, task);
return true;
}
private AuthzStatus isAccessAllowedForHierarchy(INode inode, INodeAttributes inodeAttribs, String path, FsAction access, String user, Set<String> groups, RangerHdfsPlugin plugin) {
AuthzStatus ret = null;
String pathOwner = inodeAttribs != null ? inodeAttribs.getUserName() : null;
if (pathOwner == null && inode != null) {
pathOwner = inode.getUserName();
}
if (RangerHadoopConstants.HDFS_ROOT_FOLDER_PATH_ALT.equals(path)) {
path = HDFS_ROOT_FOLDER_PATH;
}
if (LOG.isDebugEnabled()) {
LOG.debug("==> RangerAccessControlEnforcer.isAccessAllowedForHierarchy(" + path + ", " + access + ", " + user + ")");
}
if (path != null) {
Set<String> accessTypes = access2ActionListMapper.get(access);
if (accessTypes == null) {
LOG.warn("RangerAccessControlEnforcer.isAccessAllowedForHierarchy(" + path + ", " + access + ", " + user + "): no Ranger accessType found for " + access);
accessTypes = access2ActionListMapper.get(FsAction.NONE);
}
String subDirPath = path;
if (subDirPath.charAt(subDirPath.length() - 1) != Path.SEPARATOR_CHAR) {
subDirPath = subDirPath + Character.toString(Path.SEPARATOR_CHAR);
}
subDirPath = subDirPath + rangerPlugin.getRandomizedWildcardPathName();
for (String accessType : accessTypes) {
RangerHdfsAccessRequest request = new RangerHdfsAccessRequest(null, subDirPath, pathOwner, access, accessType, user, groups);
RangerAccessResult result = plugin.isAccessAllowed(request, null);
if (result == null || !result.getIsAccessDetermined()) {
ret = AuthzStatus.NOT_DETERMINED;
// don't break yet; subsequent accessType could be denied
} else if(! result.getIsAllowed()) { // explicit deny
ret = AuthzStatus.DENY;
break;
} else { // allowed
if(!AuthzStatus.NOT_DETERMINED.equals(ret)) { // set to ALLOW only if there was no NOT_DETERMINED earlier
ret = AuthzStatus.ALLOW;
}
}
}
}
if(ret == null) {
ret = AuthzStatus.NOT_DETERMINED;
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== RangerAccessControlEnforcer.isAccessAllowedForHierarchy(" + path + ", " + access + ", " + user + "): " + ret);
}
return ret;
}
/**
* Move the path name on one volume to a temporary location and then
* delete them.
*
* This functions returns when the moves are done, but not necessarily all
* deletions are done. This is usually good enough because applications
* won't see the path name under the old name anyway after the move.
*
* @param volume The disk volume
* @param pathName The path name relative to volume root.
* @throws IOException If the move failed
* @return false if the file is not found
*/
public boolean moveAndDeleteRelativePath(String volume, String pathName)
throws IOException {
volume = normalizePath(volume);
// Move the file right now, so that it can be deleted later
String newPathName =
format.format(new Date()) + "_" + uniqueId.getAndIncrement();
newPathName = TOBEDELETED + Path.SEPARATOR_CHAR + newPathName;
Path source = new Path(volume, pathName);
Path target = new Path(volume, newPathName);
try {
if (!localFileSystem.rename(source, target)) {
// If the source does not exists, return false.
// This is necessary because rename can return false if the source
// does not exists.
if (!localFileSystem.exists(source)) {
return false;
}
// Try to recreate the parent directory just in case it gets deleted.
if (!localFileSystem.mkdirs(new Path(volume, TOBEDELETED))) {
throw new IOException("Cannot create " + TOBEDELETED + " under "
+ volume);
}
// Try rename again. If it fails, return false.
if (!localFileSystem.rename(source, target)) {
throw new IOException("Cannot rename " + source + " to "
+ target);
}
}
} catch (FileNotFoundException e) {
// Return false in case that the file is not found.
return false;
}
DeleteTask task = new DeleteTask(volume, pathName, newPathName);
execute(volume, task);
return true;
}