下面列出了怎么用org.apache.hadoop.fs.FsConstants的API类实例代码及写法,或者点击链接到github查看源代码。
@BeforeClass
public static void clusterSetupAtBegining() throws IOException,
LoginException, URISyntaxException {
cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(2).build();
cluster.waitClusterUp();
fHdfs = cluster.getFileSystem();
defaultWorkingDirectory = fHdfs.makeQualified( new Path("/user/" +
UserGroupInformation.getCurrentUser().getShortUserName()));
fHdfs.mkdirs(defaultWorkingDirectory);
// Setup the ViewFS to be used for all tests.
Configuration conf = ViewFileSystemTestSetup.createConfig();
ConfigUtil.addLink(conf, "/vfstmp", new URI(fHdfs.getUri() + "/hdfstmp"));
ConfigUtil.addLink(conf, "/tmp", new URI(fHdfs.getUri() + "/tmp"));
vfs = FileSystem.get(FsConstants.VIEWFS_URI, conf);
assertEquals(ViewFileSystem.class, vfs.getClass());
}
@Before
public void setUp() throws Exception {
fcTarget = fc;
fcTarget2 = fc2;
targetTestRoot = fileContextTestHelper.getAbsoluteTestRootPath(fc);
targetTestRoot2 = fileContextTestHelper.getAbsoluteTestRootPath(fc2);
fcTarget.delete(targetTestRoot, true);
fcTarget2.delete(targetTestRoot2, true);
fcTarget.mkdir(targetTestRoot, new FsPermission((short)0750), true);
fcTarget2.mkdir(targetTestRoot2, new FsPermission((short)0750), true);
fsViewConf = ViewFileSystemTestSetup.createConfig();
setupMountPoints();
fcView = FileContext.getFileContext(FsConstants.VIEWFS_URI, fsViewConf);
}
@Before
public void setUp() throws Exception {
fcTarget = fc;
fcTarget2 = fc2;
targetTestRoot = fileContextTestHelper.getAbsoluteTestRootPath(fc);
targetTestRoot2 = fileContextTestHelper.getAbsoluteTestRootPath(fc2);
fcTarget.delete(targetTestRoot, true);
fcTarget2.delete(targetTestRoot2, true);
fcTarget.mkdir(targetTestRoot, new FsPermission((short) 0750), true);
fcTarget2.mkdir(targetTestRoot2, new FsPermission((short) 0750), true);
fsViewConf = ViewFileSystemTestSetup.createConfig();
setupMountPoints();
fcView = FileContext.getFileContext(FsConstants.VIEWFS_URI, fsViewConf);
}
@Before
public void setUp() throws Exception {
fsTarget = fHdfs;
fsTarget2 = fHdfs2;
targetTestRoot = fileSystemTestHelper.getAbsoluteTestRootPath(fsTarget);
targetTestRoot2 = fileSystemTestHelper.getAbsoluteTestRootPath(fsTarget2);
fsTarget.delete(targetTestRoot, true);
fsTarget2.delete(targetTestRoot2, true);
fsTarget.mkdirs(targetTestRoot);
fsTarget2.mkdirs(targetTestRoot2);
fsViewConf = ViewFileSystemTestSetup.createConfig();
setupMountPoints();
fsView = FileSystem.get(FsConstants.VIEWFS_URI, fsViewConf);
}
@Before
public void setUp() throws Exception {
fsTarget = fHdfs;
fsTarget2 = fHdfs2;
targetTestRoot = fileSystemTestHelper.getAbsoluteTestRootPath(fsTarget);
targetTestRoot2 = fileSystemTestHelper.getAbsoluteTestRootPath(fsTarget2);
fsTarget.delete(targetTestRoot, true);
fsTarget2.delete(targetTestRoot2, true);
fsTarget.mkdirs(targetTestRoot);
fsTarget2.mkdirs(targetTestRoot2);
fsViewConf = ViewFileSystemTestSetup.createConfig();
setupMountPoints();
fsView = FileSystem.get(FsConstants.VIEWFS_URI, fsViewConf);
}
@BeforeClass
public static void setup() throws URISyntaxException{
try {
Path fswd = FileSystem.get(getConf()).getWorkingDirectory();
Configuration vConf = ViewFileSystemTestSetup.createConfig(false);
ConfigUtil.addLink(vConf, "/usr", new URI(fswd.toString()));
fs = FileSystem.get(FsConstants.VIEWFS_URI, vConf);
fs.setWorkingDirectory(new Path("/usr"));
listFile = new Path("target/tmp/listing").makeQualified(fs.getUri(),
fs.getWorkingDirectory());
target = new Path("target/tmp/target").makeQualified(fs.getUri(),
fs.getWorkingDirectory());
root = new Path("target/tmp").makeQualified(fs.getUri(),
fs.getWorkingDirectory()).toString();
TestDistCpUtils.delete(fs, root);
} catch (IOException e) {
LOG.error("Exception encountered ", e);
}
}
@Before
public void setUp() throws Exception {
initializeTargetTestRoot();
// Make user and data dirs - we creates links to them in the mount table
fsTarget.mkdirs(new Path(targetTestRoot,"user"));
fsTarget.mkdirs(new Path(targetTestRoot,"data"));
fsTarget.mkdirs(new Path(targetTestRoot,"dir2"));
fsTarget.mkdirs(new Path(targetTestRoot,"dir3"));
FileSystemTestHelper.createFile(fsTarget, new Path(targetTestRoot,"aFile"));
// Now we use the mount fs to set links to user and dir
// in the test root
// Set up the defaultMT in the config with our mount point links
conf = ViewFileSystemTestSetup.createConfig();
setupMountPoints();
fsView = FileSystem.get(FsConstants.VIEWFS_URI, conf);
}
@BeforeClass
public static void clusterSetupAtBegining() throws IOException,
LoginException, URISyntaxException {
cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(2).build();
cluster.waitClusterUp();
fHdfs = cluster.getFileSystem();
defaultWorkingDirectory = fHdfs.makeQualified( new Path("/user/" +
UserGroupInformation.getCurrentUser().getShortUserName()));
fHdfs.mkdirs(defaultWorkingDirectory);
// Setup the ViewFS to be used for all tests.
Configuration conf = ViewFileSystemTestSetup.createConfig();
ConfigUtil.addLink(conf, "/vfstmp", new URI(fHdfs.getUri() + "/hdfstmp"));
ConfigUtil.addLink(conf, "/tmp", new URI(fHdfs.getUri() + "/tmp"));
vfs = FileSystem.get(FsConstants.VIEWFS_URI, conf);
assertEquals(ViewFileSystem.class, vfs.getClass());
}
@Before
public void setUp() throws Exception {
fcTarget = fc;
fcTarget2 = fc2;
targetTestRoot = fileContextTestHelper.getAbsoluteTestRootPath(fc);
targetTestRoot2 = fileContextTestHelper.getAbsoluteTestRootPath(fc2);
fcTarget.delete(targetTestRoot, true);
fcTarget2.delete(targetTestRoot2, true);
fcTarget.mkdir(targetTestRoot, new FsPermission((short)0750), true);
fcTarget2.mkdir(targetTestRoot2, new FsPermission((short)0750), true);
fsViewConf = ViewFileSystemTestSetup.createConfig();
setupMountPoints();
fcView = FileContext.getFileContext(FsConstants.VIEWFS_URI, fsViewConf);
}
@Before
public void setUp() throws Exception {
fcTarget = fc;
fcTarget2 = fc2;
targetTestRoot = fileContextTestHelper.getAbsoluteTestRootPath(fc);
targetTestRoot2 = fileContextTestHelper.getAbsoluteTestRootPath(fc2);
fcTarget.delete(targetTestRoot, true);
fcTarget2.delete(targetTestRoot2, true);
fcTarget.mkdir(targetTestRoot, new FsPermission((short) 0750), true);
fcTarget2.mkdir(targetTestRoot2, new FsPermission((short) 0750), true);
fsViewConf = ViewFileSystemTestSetup.createConfig();
setupMountPoints();
fcView = FileContext.getFileContext(FsConstants.VIEWFS_URI, fsViewConf);
}
@Before
public void setUp() throws Exception {
fsTarget = fHdfs;
fsTarget2 = fHdfs2;
targetTestRoot = fileSystemTestHelper.getAbsoluteTestRootPath(fsTarget);
targetTestRoot2 = fileSystemTestHelper.getAbsoluteTestRootPath(fsTarget2);
fsTarget.delete(targetTestRoot, true);
fsTarget2.delete(targetTestRoot2, true);
fsTarget.mkdirs(targetTestRoot);
fsTarget2.mkdirs(targetTestRoot2);
fsViewConf = ViewFileSystemTestSetup.createConfig();
setupMountPoints();
fsView = FileSystem.get(FsConstants.VIEWFS_URI, fsViewConf);
}
@Before
public void setUp() throws Exception {
fsTarget = fHdfs;
fsTarget2 = fHdfs2;
targetTestRoot = fileSystemTestHelper.getAbsoluteTestRootPath(fsTarget);
targetTestRoot2 = fileSystemTestHelper.getAbsoluteTestRootPath(fsTarget2);
fsTarget.delete(targetTestRoot, true);
fsTarget2.delete(targetTestRoot2, true);
fsTarget.mkdirs(targetTestRoot);
fsTarget2.mkdirs(targetTestRoot2);
fsViewConf = ViewFileSystemTestSetup.createConfig();
setupMountPoints();
fsView = FileSystem.get(FsConstants.VIEWFS_URI, fsViewConf);
}
@BeforeClass
public static void setup() throws URISyntaxException{
try {
Path fswd = FileSystem.get(getConf()).getWorkingDirectory();
Configuration vConf = ViewFileSystemTestSetup.createConfig(false);
ConfigUtil.addLink(vConf, "/usr", new URI(fswd.toString()));
fs = FileSystem.get(FsConstants.VIEWFS_URI, vConf);
fs.setWorkingDirectory(new Path("/usr"));
listFile = new Path("target/tmp/listing").makeQualified(fs.getUri(),
fs.getWorkingDirectory());
target = new Path("target/tmp/target").makeQualified(fs.getUri(),
fs.getWorkingDirectory());
root = new Path("target/tmp").makeQualified(fs.getUri(),
fs.getWorkingDirectory()).toString();
TestDistCpUtils.delete(fs, root);
} catch (IOException e) {
LOG.error("Exception encountered ", e);
}
}
@Before
public void setUp() throws Exception {
initializeTargetTestRoot();
// Make user and data dirs - we creates links to them in the mount table
fsTarget.mkdirs(new Path(targetTestRoot,"user"));
fsTarget.mkdirs(new Path(targetTestRoot,"data"));
fsTarget.mkdirs(new Path(targetTestRoot,"dir2"));
fsTarget.mkdirs(new Path(targetTestRoot,"dir3"));
FileSystemTestHelper.createFile(fsTarget, new Path(targetTestRoot,"aFile"));
// Now we use the mount fs to set links to user and dir
// in the test root
// Set up the defaultMT in the config with our mount point links
conf = ViewFileSystemTestSetup.createConfig();
setupMountPoints();
fsView = FileSystem.get(FsConstants.VIEWFS_URI, conf);
}
protected ElevatedFileSystem() throws IOException, URISyntaxException {
super(FsConstants.LOCAL_FS_URI,
new ElevatedRawLocalFilesystem(),
new Configuration(),
FsConstants.LOCAL_FS_URI.getScheme(),
false);
}
@BeforeClass
public static void clusterSetupAtBegining() throws IOException,
LoginException, URISyntaxException {
SupportsBlocks = true;
CONF.setBoolean(
DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
cluster =
new MiniDFSCluster.Builder(CONF).nnTopology(
MiniDFSNNTopology.simpleFederatedTopology(2))
.numDataNodes(2)
.build();
cluster.waitClusterUp();
fHdfs = cluster.getFileSystem(0);
fHdfs2 = cluster.getFileSystem(1);
fHdfs.getConf().set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY,
FsConstants.VIEWFS_URI.toString());
fHdfs2.getConf().set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY,
FsConstants.VIEWFS_URI.toString());
defaultWorkingDirectory = fHdfs.makeQualified( new Path("/user/" +
UserGroupInformation.getCurrentUser().getShortUserName()));
defaultWorkingDirectory2 = fHdfs2.makeQualified( new Path("/user/" +
UserGroupInformation.getCurrentUser().getShortUserName()));
fHdfs.mkdirs(defaultWorkingDirectory);
fHdfs2.mkdirs(defaultWorkingDirectory2);
}
/**
* This constructor has the signature needed by
* {@link AbstractFileSystem#createFileSystem(URI, Configuration)}.
*
* @param theUri which must be that of ViewFs
* @param conf
* @throws IOException
* @throws URISyntaxException
*/
ViewFs(final URI theUri, final Configuration conf) throws IOException,
URISyntaxException {
super(theUri, FsConstants.VIEWFS_SCHEME, false, -1);
creationTime = Time.now();
ugi = UserGroupInformation.getCurrentUser();
config = conf;
// Now build client side view (i.e. client side mount table) from config.
String authority = theUri.getAuthority();
fsState = new InodeTree<AbstractFileSystem>(conf, authority) {
@Override
protected
AbstractFileSystem getTargetFileSystem(final URI uri)
throws URISyntaxException, UnsupportedFileSystemException {
String pathString = uri.getPath();
if (pathString.isEmpty()) {
pathString = "/";
}
return new ChRootedFs(
AbstractFileSystem.createFileSystem(uri, config),
new Path(pathString));
}
@Override
protected
AbstractFileSystem getTargetFileSystem(
final INodeDir<AbstractFileSystem> dir) throws URISyntaxException {
return new InternalDirOfViewFs(dir, creationTime, ugi, getUri());
}
@Override
protected
AbstractFileSystem getTargetFileSystem(URI[] mergeFsURIList)
throws URISyntaxException, UnsupportedFileSystemException {
throw new UnsupportedFileSystemException("mergefs not implemented yet");
// return MergeFs.createMergeFs(mergeFsURIList, config);
}
};
}
public InternalDirOfViewFs(final InodeTree.INodeDir<AbstractFileSystem> dir,
final long cTime, final UserGroupInformation ugi, final URI uri)
throws URISyntaxException {
super(FsConstants.VIEWFS_URI, FsConstants.VIEWFS_SCHEME, false, -1);
theInternalDir = dir;
creationTime = cTime;
this.ugi = ugi;
myUri = uri;
}
@Test
public void testBasicPaths() {
Assert.assertEquals(FsConstants.VIEWFS_URI,
fsView.getUri());
Assert.assertEquals(fsView.makeQualified(
new Path("/user/" + System.getProperty("user.name"))),
fsView.getWorkingDirectory());
Assert.assertEquals(fsView.makeQualified(
new Path("/user/" + System.getProperty("user.name"))),
fsView.getHomeDirectory());
Assert.assertEquals(
new Path("/foo/bar").makeQualified(FsConstants.VIEWFS_URI, null),
fsView.makeQualified(new Path("/foo/bar")));
}
/**
*
* @param fsTarget - the target fs of the view fs.
* @return return the ViewFS File context to be used for tests
* @throws Exception
*/
static public FileSystem setupForViewFileSystem(Configuration conf, FileSystemTestHelper fileSystemTestHelper, FileSystem fsTarget) throws Exception {
/**
* create the test root on local_fs - the mount table will point here
*/
Path targetOfTests = fileSystemTestHelper.getTestRootPath(fsTarget);
// In case previous test was killed before cleanup
fsTarget.delete(targetOfTests, true);
fsTarget.mkdirs(targetOfTests);
// Set up viewfs link for test dir as described above
String testDir = fileSystemTestHelper.getTestRootPath(fsTarget).toUri()
.getPath();
linkUpFirstComponents(conf, testDir, fsTarget, "test dir");
// Set up viewfs link for home dir as described above
setUpHomeDir(conf, fsTarget);
// the test path may be relative to working dir - we need to make that work:
// Set up viewfs link for wd as described above
String wdDir = fsTarget.getWorkingDirectory().toUri().getPath();
linkUpFirstComponents(conf, wdDir, fsTarget, "working dir");
FileSystem fsView = FileSystem.get(FsConstants.VIEWFS_URI, conf);
fsView.setWorkingDirectory(new Path(wdDir)); // in case testdir relative to wd.
Log.info("Working dir is: " + fsView.getWorkingDirectory());
return fsView;
}
@Before
public void setUp() throws Exception {
initializeTargetTestRoot();
// Make user and data dirs - we creates links to them in the mount table
fcTarget.mkdir(new Path(targetTestRoot,"user"),
FileContext.DEFAULT_PERM, true);
fcTarget.mkdir(new Path(targetTestRoot,"data"),
FileContext.DEFAULT_PERM, true);
fcTarget.mkdir(new Path(targetTestRoot,"dir2"),
FileContext.DEFAULT_PERM, true);
fcTarget.mkdir(new Path(targetTestRoot,"dir3"),
FileContext.DEFAULT_PERM, true);
FileContextTestHelper.createFile(fcTarget, new Path(targetTestRoot,"aFile"));
// Now we use the mount fs to set links to user and dir
// in the test root
// Set up the defaultMT in the config with our mount point links
conf = new Configuration();
ConfigUtil.addLink(conf, "/targetRoot", targetTestRoot.toUri());
ConfigUtil.addLink(conf, "/user",
new Path(targetTestRoot,"user").toUri());
ConfigUtil.addLink(conf, "/user2",
new Path(targetTestRoot,"user").toUri());
ConfigUtil.addLink(conf, "/data",
new Path(targetTestRoot,"data").toUri());
ConfigUtil.addLink(conf, "/internalDir/linkToDir2",
new Path(targetTestRoot,"dir2").toUri());
ConfigUtil.addLink(conf, "/internalDir/internalDir2/linkToDir3",
new Path(targetTestRoot,"dir3").toUri());
ConfigUtil.addLink(conf, "/danglingLink",
new Path(targetTestRoot,"missingTarget").toUri());
ConfigUtil.addLink(conf, "/linkToAFile",
new Path(targetTestRoot,"aFile").toUri());
fcView = FileContext.getFileContext(FsConstants.VIEWFS_URI, conf);
// Also try viewfs://default/ - note authority is name of mount table
}
@Test
public void testBasicPaths() {
Assert.assertEquals(FsConstants.VIEWFS_URI,
fcView.getDefaultFileSystem().getUri());
Assert.assertEquals(fcView.makeQualified(
new Path("/user/" + System.getProperty("user.name"))),
fcView.getWorkingDirectory());
Assert.assertEquals(fcView.makeQualified(
new Path("/user/" + System.getProperty("user.name"))),
fcView.getHomeDirectory());
Assert.assertEquals(
new Path("/foo/bar").makeQualified(FsConstants.VIEWFS_URI, null),
fcView.makeQualified(new Path("/foo/bar")));
}
@Test
public void testURIEmptyPath() throws Exception {
Configuration conf = new Configuration();
ConfigUtil.addLink(conf, "/user", new URI("file://foo"));
FileContext.getFileContext(FsConstants.VIEWFS_URI, conf);
}
@Test
public void testBasicPaths() {
URI uri = fSys.getUri();
Assert.assertEquals(chrootedTo.toUri(), uri);
Assert.assertEquals(fSys.makeQualified(
new Path(System.getProperty("user.home"))),
fSys.getWorkingDirectory());
Assert.assertEquals(fSys.makeQualified(
new Path(System.getProperty("user.home"))),
fSys.getHomeDirectory());
/*
* ChRootedFs as its uri like file:///chrootRoot.
* This is questionable since path.makequalified(uri, path) ignores
* the pathPart of a uri. So our notion of chrooted URI is questionable.
* But if we were to fix Path#makeQualified() then the next test should
* have been:
Assert.assertEquals(
new Path(chrootedTo + "/foo/bar").makeQualified(
FsConstants.LOCAL_FS_URI, null),
fSys.makeQualified(new Path( "/foo/bar")));
*/
Assert.assertEquals(
new Path("/foo/bar").makeQualified(FsConstants.LOCAL_FS_URI, null),
fSys.makeQualified(new Path("/foo/bar")));
}
static public FileContext setupForViewFsLocalFs(FileContextTestHelper helper) throws Exception {
/**
* create the test root on local_fs - the mount table will point here
*/
FileContext fsTarget = FileContext.getLocalFSFileContext();
Path targetOfTests = helper.getTestRootPath(fsTarget);
// In case previous test was killed before cleanup
fsTarget.delete(targetOfTests, true);
fsTarget.mkdir(targetOfTests, FileContext.DEFAULT_PERM, true);
Configuration conf = new Configuration();
// Set up viewfs link for test dir as described above
String testDir = helper.getTestRootPath(fsTarget).toUri()
.getPath();
linkUpFirstComponents(conf, testDir, fsTarget, "test dir");
// Set up viewfs link for home dir as described above
setUpHomeDir(conf, fsTarget);
// the test path may be relative to working dir - we need to make that work:
// Set up viewfs link for wd as described above
String wdDir = fsTarget.getWorkingDirectory().toUri().getPath();
linkUpFirstComponents(conf, wdDir, fsTarget, "working dir");
FileContext fc = FileContext.getFileContext(FsConstants.VIEWFS_URI, conf);
fc.setWorkingDirectory(new Path(wdDir)); // in case testdir relative to wd.
Log.info("Working dir is: " + fc.getWorkingDirectory());
//System.out.println("SRCOfTests = "+ getTestRootPath(fc, "test"));
//System.out.println("TargetOfTests = "+ targetOfTests.toUri());
return fc;
}
@Test
public void testBasicPaths() {
URI uri = fc.getDefaultFileSystem().getUri();
Assert.assertEquals(chrootedTo.toUri(), uri);
Assert.assertEquals(fc.makeQualified(
new Path(System.getProperty("user.home"))),
fc.getWorkingDirectory());
Assert.assertEquals(fc.makeQualified(
new Path(System.getProperty("user.home"))),
fc.getHomeDirectory());
/*
* ChRootedFs as its uri like file:///chrootRoot.
* This is questionable since path.makequalified(uri, path) ignores
* the pathPart of a uri. So our notion of chrooted URI is questionable.
* But if we were to fix Path#makeQualified() then the next test should
* have been:
Assert.assertEquals(
new Path(chrootedTo + "/foo/bar").makeQualified(
FsConstants.LOCAL_FS_URI, null),
fc.makeQualified(new Path( "/foo/bar")));
*/
Assert.assertEquals(
new Path("/foo/bar").makeQualified(FsConstants.LOCAL_FS_URI, null),
fc.makeQualified(new Path("/foo/bar")));
}
@BeforeClass
public static void setup() throws Exception {
conf = ViewFileSystemTestSetup.createConfig();
fs1 = setupFileSystem(new URI("fs1:/"), FakeFileSystem.class);
fs2 = setupFileSystem(new URI("fs2:/"), FakeFileSystem.class);
viewFs = FileSystem.get(FsConstants.VIEWFS_URI, conf);
}
@Override
@Before
public void setUp() throws Exception {
// create the test root on local_fs
fsTarget = FileSystem.getLocal(new Configuration());
super.setUp(); // this sets up conf (and fcView which we replace)
// Now create a viewfs using a mount table called "default"
// hence viewfs://default/
schemeWithAuthority =
new URI(FsConstants.VIEWFS_SCHEME, "default", "/", null, null);
fsView = FileSystem.get(schemeWithAuthority, conf);
}
@Override
@Before
public void setUp() throws Exception {
// create the test root on local_fs
fcTarget = FileContext.getLocalFSFileContext();
super.setUp(); // this sets up conf (and fcView which we replace)
// Now create a viewfs using a mount table called "default"
// hence viewfs://default/
schemeWithAuthority =
new URI(FsConstants.VIEWFS_SCHEME, "default", "/", null, null);
fcView = FileContext.getFileContext(schemeWithAuthority, conf);
}
@Before
public void setUp() throws Exception {
fsTarget = FileSystem.getLocal(new Configuration());
fsTarget.mkdirs(new Path(fileSystemTestHelper.
getTestRootPath(fsTarget), "dir1"));
conf = ViewFileSystemTestSetup.createConfig();
fsView = ViewFileSystemTestSetup.setupForViewFileSystem(conf, fileSystemTestHelper, fsTarget);
conf.set("fs.defaultFS", FsConstants.VIEWFS_URI.toString());
}