类org.apache.hadoop.fs.AbstractFileSystem源码实例Demo

下面列出了怎么用org.apache.hadoop.fs.AbstractFileSystem的API类实例代码及写法,或者点击链接到github查看源代码。

源代码1 项目: hadoop   文件: ChRootedFs.java
public ChRootedFs(final AbstractFileSystem fs, final Path theRoot)
  throws URISyntaxException {
  super(fs.getUri(), fs.getUri().getScheme(),
      fs.getUri().getAuthority() != null, fs.getUriDefaultPort());
  myFs = fs;
  myFs.checkPath(theRoot);
  chRootPathPart = new Path(myFs.getUriPath(theRoot));
  chRootPathPartString = chRootPathPart.toUri().getPath();
  /*
   * We are making URI include the chrootedPath: e.g. file:///chrootedPath.
   * This is questionable since Path#makeQualified(uri, path) ignores
   * the pathPart of a uri. Since this class is internal we can ignore
   * this issue but if we were to make it external then this needs
   * to be resolved.
   */
  // Handle the two cases:
  //              scheme:/// and scheme://authority/
  myUri = new URI(myFs.getUri().toString() + 
      (myFs.getUri().getAuthority() == null ? "" :  Path.SEPARATOR) +
        chRootPathPart.toUri().getPath().substring(1));
  super.checkPath(theRoot);
}
 
源代码2 项目: hadoop   文件: ViewFs.java
@Override
public FSDataOutputStream createInternal(final Path f,
    final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
    final int bufferSize, final short replication, final long blockSize,
    final Progressable progress, final ChecksumOpt checksumOpt,
    final boolean createParent) throws AccessControlException,
    FileAlreadyExistsException, FileNotFoundException,
    ParentNotDirectoryException, UnsupportedFileSystemException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res;
  try {
    res = fsState.resolve(getUriPath(f), false);
  } catch (FileNotFoundException e) {
    if (createParent) {
      throw readOnlyMountTable("create", f);
    } else {
      throw e;
    }
  }
  assert(res.remainingPath != null);
  return res.targetFileSystem.createInternal(res.remainingPath, flag,
      absolutePermission, bufferSize, replication,
      blockSize, progress, checksumOpt,
      createParent);
}
 
源代码3 项目: big-c   文件: ViewFsBaseTest.java
@Test
public void testGetFileChecksum() throws AccessControlException
  , UnresolvedLinkException, IOException {
  AbstractFileSystem mockAFS = Mockito.mock(AbstractFileSystem.class);
  InodeTree.ResolveResult<AbstractFileSystem> res =
    new InodeTree.ResolveResult<AbstractFileSystem>(null, mockAFS , null,
      new Path("someFile"));
  @SuppressWarnings("unchecked")
  InodeTree<AbstractFileSystem> fsState = Mockito.mock(InodeTree.class);
  Mockito.when(fsState.resolve(Mockito.anyString()
    , Mockito.anyBoolean())).thenReturn(res);
  ViewFs vfs = Mockito.mock(ViewFs.class);
  vfs.fsState = fsState;

  Mockito.when(vfs.getFileChecksum(new Path("/tmp/someFile")))
    .thenCallRealMethod();
  vfs.getFileChecksum(new Path("/tmp/someFile"));

  Mockito.verify(mockAFS).getFileChecksum(new Path("someFile"));
}
 
源代码4 项目: hadoop   文件: ViewFs.java
@Override
public FileStatus getFileStatus(final Path f) throws AccessControlException,
    FileNotFoundException, UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res = 
    fsState.resolve(getUriPath(f), true);

  //  FileStatus#getPath is a fully qualified path relative to the root of 
  // target file system.
  // We need to change it to viewfs URI - relative to root of mount table.
  
  // The implementors of RawLocalFileSystem were trying to be very smart.
  // They implement FileStatus#getOwener lazily -- the object
  // returned is really a RawLocalFileSystem that expect the
  // FileStatus#getPath to be unchanged so that it can get owner when needed.
  // Hence we need to interpose a new ViewFsFileStatus that works around.
  
  
  FileStatus status =  res.targetFileSystem.getFileStatus(res.remainingPath);
  return new ViewFsFileStatus(status, this.makeQualified(f));
}
 
源代码5 项目: hadoop   文件: ViewFs.java
@Override
public FileStatus[] listStatus(final Path f) throws AccessControlException,
    FileNotFoundException, UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res =
    fsState.resolve(getUriPath(f), true);
  
  FileStatus[] statusLst = res.targetFileSystem.listStatus(res.remainingPath);
  if (!res.isInternalDir()) {
    // We need to change the name in the FileStatus as described in
    // {@link #getFileStatus }
    ChRootedFs targetFs;
    targetFs = (ChRootedFs) res.targetFileSystem;
    int i = 0;
    for (FileStatus status : statusLst) {
        String suffix = targetFs.stripOutRoot(status.getPath());
        statusLst[i++] = new ViewFsFileStatus(status, this.makeQualified(
            suffix.length() == 0 ? f : new Path(res.resolvedPath, suffix)));
    }
  }
  return statusLst;
}
 
源代码6 项目: hadoop   文件: ViewFs.java
@Override
public void createSymlink(final Path target, final Path link,
    final boolean createParent) throws IOException, UnresolvedLinkException {
  InodeTree.ResolveResult<AbstractFileSystem> res;
  try {
    res = fsState.resolve(getUriPath(link), false);
  } catch (FileNotFoundException e) {
    if (createParent) {
      throw readOnlyMountTable("createSymlink", link);
    } else {
      throw e;
    }
  }
  assert(res.remainingPath != null);
  res.targetFileSystem.createSymlink(target, res.remainingPath,
      createParent);  
}
 
源代码7 项目: hadoop   文件: ViewFs.java
@Override
public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
  List<InodeTree.MountPoint<AbstractFileSystem>> mountPoints = 
              fsState.getMountPoints();
  int initialListSize  = 0;
  for (InodeTree.MountPoint<AbstractFileSystem> im : mountPoints) {
    initialListSize += im.target.targetDirLinkList.length; 
  }
  List<Token<?>> result = new ArrayList<Token<?>>(initialListSize);
  for ( int i = 0; i < mountPoints.size(); ++i ) {
    List<Token<?>> tokens = 
      mountPoints.get(i).target.targetFileSystem.getDelegationTokens(renewer);
    if (tokens != null) {
      result.addAll(tokens);
    }
  }
  return result;
}
 
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
    primaryFsUri = new URI(primaryFileSystemUriPath());

    primaryFsCfg = new Configuration();

    primaryFsCfg.addResource(U.resolveIgniteUrl(primaryFileSystemConfigPath()));

    UserGroupInformation ugi = UserGroupInformation.getBestUGI(null, getClientFsUser());

    // Create Fs on behalf of the client user:
    ugi.doAs(new PrivilegedExceptionAction<Object>() {
        @Override public Object run() throws Exception {
            fs = AbstractFileSystem.get(primaryFsUri, primaryFsCfg);

            return null;
        }
    });

    barrier = new CyclicBarrier(THREAD_CNT);
}
 
源代码9 项目: hadoop   文件: ViewFsBaseTest.java
@Test
public void testGetFileChecksum() throws AccessControlException
  , UnresolvedLinkException, IOException {
  AbstractFileSystem mockAFS = Mockito.mock(AbstractFileSystem.class);
  InodeTree.ResolveResult<AbstractFileSystem> res =
    new InodeTree.ResolveResult<AbstractFileSystem>(null, mockAFS , null,
      new Path("someFile"));
  @SuppressWarnings("unchecked")
  InodeTree<AbstractFileSystem> fsState = Mockito.mock(InodeTree.class);
  Mockito.when(fsState.resolve(Mockito.anyString()
    , Mockito.anyBoolean())).thenReturn(res);
  ViewFs vfs = Mockito.mock(ViewFs.class);
  vfs.fsState = fsState;

  Mockito.when(vfs.getFileChecksum(new Path("/tmp/someFile")))
    .thenCallRealMethod();
  vfs.getFileChecksum(new Path("/tmp/someFile"));

  Mockito.verify(mockAFS).getFileChecksum(new Path("someFile"));
}
 
源代码10 项目: big-c   文件: ChRootedFs.java
public ChRootedFs(final AbstractFileSystem fs, final Path theRoot)
  throws URISyntaxException {
  super(fs.getUri(), fs.getUri().getScheme(),
      fs.getUri().getAuthority() != null, fs.getUriDefaultPort());
  myFs = fs;
  myFs.checkPath(theRoot);
  chRootPathPart = new Path(myFs.getUriPath(theRoot));
  chRootPathPartString = chRootPathPart.toUri().getPath();
  /*
   * We are making URI include the chrootedPath: e.g. file:///chrootedPath.
   * This is questionable since Path#makeQualified(uri, path) ignores
   * the pathPart of a uri. Since this class is internal we can ignore
   * this issue but if we were to make it external then this needs
   * to be resolved.
   */
  // Handle the two cases:
  //              scheme:/// and scheme://authority/
  myUri = new URI(myFs.getUri().toString() + 
      (myFs.getUri().getAuthority() == null ? "" :  Path.SEPARATOR) +
        chRootPathPart.toUri().getPath().substring(1));
  super.checkPath(theRoot);
}
 
源代码11 项目: big-c   文件: ViewFs.java
@Override
public FSDataOutputStream createInternal(final Path f,
    final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
    final int bufferSize, final short replication, final long blockSize,
    final Progressable progress, final ChecksumOpt checksumOpt,
    final boolean createParent) throws AccessControlException,
    FileAlreadyExistsException, FileNotFoundException,
    ParentNotDirectoryException, UnsupportedFileSystemException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res;
  try {
    res = fsState.resolve(getUriPath(f), false);
  } catch (FileNotFoundException e) {
    if (createParent) {
      throw readOnlyMountTable("create", f);
    } else {
      throw e;
    }
  }
  assert(res.remainingPath != null);
  return res.targetFileSystem.createInternal(res.remainingPath, flag,
      absolutePermission, bufferSize, replication,
      blockSize, progress, checksumOpt,
      createParent);
}
 
源代码12 项目: big-c   文件: ViewFs.java
@Override
public FileStatus[] listStatus(final Path f) throws AccessControlException,
    FileNotFoundException, UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res =
    fsState.resolve(getUriPath(f), true);
  
  FileStatus[] statusLst = res.targetFileSystem.listStatus(res.remainingPath);
  if (!res.isInternalDir()) {
    // We need to change the name in the FileStatus as described in
    // {@link #getFileStatus }
    ChRootedFs targetFs;
    targetFs = (ChRootedFs) res.targetFileSystem;
    int i = 0;
    for (FileStatus status : statusLst) {
        String suffix = targetFs.stripOutRoot(status.getPath());
        statusLst[i++] = new ViewFsFileStatus(status, this.makeQualified(
            suffix.length() == 0 ? f : new Path(res.resolvedPath, suffix)));
    }
  }
  return statusLst;
}
 
源代码13 项目: big-c   文件: ViewFs.java
@Override
public void createSymlink(final Path target, final Path link,
    final boolean createParent) throws IOException, UnresolvedLinkException {
  InodeTree.ResolveResult<AbstractFileSystem> res;
  try {
    res = fsState.resolve(getUriPath(link), false);
  } catch (FileNotFoundException e) {
    if (createParent) {
      throw readOnlyMountTable("createSymlink", link);
    } else {
      throw e;
    }
  }
  assert(res.remainingPath != null);
  res.targetFileSystem.createSymlink(target, res.remainingPath,
      createParent);  
}
 
源代码14 项目: big-c   文件: ViewFs.java
@Override
public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
  List<InodeTree.MountPoint<AbstractFileSystem>> mountPoints = 
              fsState.getMountPoints();
  int initialListSize  = 0;
  for (InodeTree.MountPoint<AbstractFileSystem> im : mountPoints) {
    initialListSize += im.target.targetDirLinkList.length; 
  }
  List<Token<?>> result = new ArrayList<Token<?>>(initialListSize);
  for ( int i = 0; i < mountPoints.size(); ++i ) {
    List<Token<?>> tokens = 
      mountPoints.get(i).target.targetFileSystem.getDelegationTokens(renewer);
    if (tokens != null) {
      result.addAll(tokens);
    }
  }
  return result;
}
 
源代码15 项目: big-c   文件: ViewFs.java
@Override
public void setXAttr(Path path, String name, byte[] value,
                     EnumSet<XAttrSetFlag> flag) throws IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res =
      fsState.resolve(getUriPath(path), true);
  res.targetFileSystem.setXAttr(res.remainingPath, name, value, flag);
}
 
源代码16 项目: hadoop   文件: TestDelegationTokensWithHA.java
@Test(timeout = 300000)
public void testHdfsGetCanonicalServiceName() throws Exception {
  Configuration conf = dfs.getConf();
  URI haUri = HATestUtil.getLogicalUri(cluster);
  AbstractFileSystem afs =  AbstractFileSystem.createFileSystem(haUri, conf);    
  String haService = HAUtil.buildTokenServiceForLogicalUri(haUri,
      HdfsConstants.HDFS_URI_SCHEME).toString();
  assertEquals(haService, afs.getCanonicalServiceName());
  Token<?> token = afs.getDelegationTokens(
      UserGroupInformation.getCurrentUser().getShortUserName()).get(0);
  assertEquals(haService, token.getService().toString());
  // make sure the logical uri is handled correctly
  token.renew(conf);
  token.cancel(conf);
}
 
源代码17 项目: hadoop   文件: ViewFs.java
/**
 * This constructor has the signature needed by
 * {@link AbstractFileSystem#createFileSystem(URI, Configuration)}.
 * 
 * @param theUri which must be that of ViewFs
 * @param conf
 * @throws IOException
 * @throws URISyntaxException 
 */
ViewFs(final URI theUri, final Configuration conf) throws IOException,
    URISyntaxException {
  super(theUri, FsConstants.VIEWFS_SCHEME, false, -1);
  creationTime = Time.now();
  ugi = UserGroupInformation.getCurrentUser();
  config = conf;
  // Now build  client side view (i.e. client side mount table) from config.
  String authority = theUri.getAuthority();
  fsState = new InodeTree<AbstractFileSystem>(conf, authority) {

    @Override
    protected
    AbstractFileSystem getTargetFileSystem(final URI uri)
      throws URISyntaxException, UnsupportedFileSystemException {
        String pathString = uri.getPath();
        if (pathString.isEmpty()) {
          pathString = "/";
        }
        return new ChRootedFs(
            AbstractFileSystem.createFileSystem(uri, config),
            new Path(pathString));
    }

    @Override
    protected
    AbstractFileSystem getTargetFileSystem(
        final INodeDir<AbstractFileSystem> dir) throws URISyntaxException {
      return new InternalDirOfViewFs(dir, creationTime, ugi, getUri());
    }

    @Override
    protected
    AbstractFileSystem getTargetFileSystem(URI[] mergeFsURIList)
        throws URISyntaxException, UnsupportedFileSystemException {
      throw new UnsupportedFileSystemException("mergefs not implemented yet");
      // return MergeFs.createMergeFs(mergeFsURIList, config);
    }
  };
}
 
源代码18 项目: hadoop   文件: ViewFs.java
@Override
public Path resolvePath(final Path f) throws FileNotFoundException,
        AccessControlException, UnresolvedLinkException, IOException {
  final InodeTree.ResolveResult<AbstractFileSystem> res;
    res = fsState.resolve(getUriPath(f), true);
  if (res.isInternalDir()) {
    return f;
  }
  return res.targetFileSystem.resolvePath(res.remainingPath);

}
 
源代码19 项目: hadoop   文件: ViewFs.java
@Override
public FileChecksum getFileChecksum(final Path f)
    throws AccessControlException, FileNotFoundException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res = 
    fsState.resolve(getUriPath(f), true);
  return res.targetFileSystem.getFileChecksum(res.remainingPath);
}
 
源代码20 项目: hadoop   文件: ViewFs.java
@Override
public void access(Path path, FsAction mode) throws AccessControlException,
    FileNotFoundException, UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res =
    fsState.resolve(getUriPath(path), true);
  res.targetFileSystem.access(res.remainingPath, mode);
}
 
源代码21 项目: hadoop   文件: ViewFs.java
@Override
public FileStatus getFileLinkStatus(final Path f)
   throws AccessControlException, FileNotFoundException,
   UnsupportedFileSystemException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res = 
    fsState.resolve(getUriPath(f), false); // do not follow mount link
  return res.targetFileSystem.getFileLinkStatus(res.remainingPath);
}
 
源代码22 项目: hadoop   文件: ViewFs.java
@Override
public RemoteIterator<FileStatus> listStatusIterator(final Path f)
  throws AccessControlException, FileNotFoundException,
  UnresolvedLinkException, IOException {
  final InodeTree.ResolveResult<AbstractFileSystem> res =
    fsState.resolve(getUriPath(f), true);
  final RemoteIterator<FileStatus> fsIter =
    res.targetFileSystem.listStatusIterator(res.remainingPath);
  if (res.isInternalDir()) {
    return fsIter;
  }
  
  return new RemoteIterator<FileStatus>() {
    final RemoteIterator<FileStatus> myIter;
    final ChRootedFs targetFs;
    { // Init
        myIter = fsIter;
        targetFs = (ChRootedFs) res.targetFileSystem;
    }
    
    @Override
    public boolean hasNext() throws IOException {
      return myIter.hasNext();
    }
    
    @Override
    public FileStatus next() throws IOException {
      FileStatus status =  myIter.next();
      String suffix = targetFs.stripOutRoot(status.getPath());
      return new ViewFsFileStatus(status, makeQualified(
          suffix.length() == 0 ? f : new Path(res.resolvedPath, suffix)));
    }
  };
}
 
源代码23 项目: big-c   文件: TestChRootedFs.java
@Test
public void testIsValidNameInvalidInBaseFs() throws Exception {
  AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem());
  ChRootedFs chRootedFs = new ChRootedFs(baseFs, new Path("/chroot"));
  Mockito.doReturn(false).when(baseFs).isValidName(Mockito.anyString());
  Assert.assertFalse(chRootedFs.isValidName("/test"));
  Mockito.verify(baseFs).isValidName("/chroot/test");
}
 
源代码24 项目: hadoop   文件: ViewFs.java
@Override
public FSDataInputStream open(final Path f, final int bufferSize)
    throws AccessControlException, FileNotFoundException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res = 
      fsState.resolve(getUriPath(f), true);
  return res.targetFileSystem.open(res.remainingPath, bufferSize);
}
 
源代码25 项目: hadoop   文件: ViewFs.java
@Override
public boolean truncate(final Path f, final long newLength)
    throws AccessControlException, FileNotFoundException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res =
      fsState.resolve(getUriPath(f), true);
  return res.targetFileSystem.truncate(res.remainingPath, newLength);
}
 
源代码26 项目: big-c   文件: ViewFs.java
@Override
public Map<String, byte[]> getXAttrs(Path path, List<String> names)
    throws IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res =
      fsState.resolve(getUriPath(path), true);
  return res.targetFileSystem.getXAttrs(res.remainingPath, names);
}
 
源代码27 项目: hadoop   文件: ViewFs.java
@Override
public void setPermission(final Path f, final FsPermission permission)
    throws AccessControlException, FileNotFoundException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res = 
    fsState.resolve(getUriPath(f), true);
  res.targetFileSystem.setPermission(res.remainingPath, permission); 
  
}
 
源代码28 项目: hadoop   文件: ViewFs.java
@Override
public boolean setReplication(final Path f, final short replication)
    throws AccessControlException, FileNotFoundException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res = 
    fsState.resolve(getUriPath(f), true);
  return res.targetFileSystem.setReplication(res.remainingPath, replication);
}
 
源代码29 项目: hadoop   文件: ViewFs.java
@Override
public void setTimes(final Path f, final long mtime, final long atime)
    throws AccessControlException, FileNotFoundException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res = 
    fsState.resolve(getUriPath(f), true);
  res.targetFileSystem.setTimes(res.remainingPath, mtime, atime); 
}
 
源代码30 项目: big-c   文件: ViewFs.java
@Override
public void removeAcl(Path path)
    throws IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res =
      fsState.resolve(getUriPath(path), true);
  res.targetFileSystem.removeAcl(res.remainingPath);
}
 
 类所在包
 类方法
 同包方法