private void doAppLogAggregationPostCleanUp() {
  // Remove the local app-log-dirs
  List<Path> localAppLogDirs = new ArrayList<Path>();
  for (String rootLogDir : dirsHandler.getLogDirsForCleanup()) {
    Path logPath = new Path(rootLogDir, applicationId);
    try {
      // check if log dir exists
      lfs.getFileStatus(logPath);
      localAppLogDirs.add(logPath);
    } catch (UnsupportedFileSystemException ue) {
      LOG.warn("Log dir " + rootLogDir + "is an unsupported file system", ue);
      continue;
    } catch (IOException fe) {
      continue;
    }
  }

  if (localAppLogDirs.size() > 0) {
    this.delService.delete(this.userUgi.getShortUserName(), null,
      localAppLogDirs.toArray(new Path[localAppLogDirs.size()]));
  }
}
 

@Override
public FSDataOutputStream createInternal(final Path f,
    final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
    final int bufferSize, final short replication, final long blockSize,
    final Progressable progress, final ChecksumOpt checksumOpt,
    final boolean createParent) throws AccessControlException,
    FileAlreadyExistsException, FileNotFoundException,
    ParentNotDirectoryException, UnsupportedFileSystemException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res;
  try {
    res = fsState.resolve(getUriPath(f), false);
  } catch (FileNotFoundException e) {
    if (createParent) {
      throw readOnlyMountTable("create", f);
    } else {
      throw e;
    }
  }
  assert(res.remainingPath != null);
  return res.targetFileSystem.createInternal(res.remainingPath, flag,
      absolutePermission, bufferSize, replication,
      blockSize, progress, checksumOpt,
      createParent);
}
 

protected static void printResults(PrintStream out) throws UnsupportedFileSystemException {
  out.println("Result of running LoadGenerator against fileSystem: " + 
  FileContext.getFileContext().getDefaultFileSystem().getUri());
  if (numOfOps[OPEN] != 0) {
    out.println("Average open execution time: " + 
        (double)executionTime[OPEN]/numOfOps[OPEN] + "ms");
  }
  if (numOfOps[LIST] != 0) {
    out.println("Average list execution time: " + 
        (double)executionTime[LIST]/numOfOps[LIST] + "ms");
  }
  if (numOfOps[DELETE] != 0) {
    out.println("Average deletion execution time: " + 
        (double)executionTime[DELETE]/numOfOps[DELETE] + "ms");
    out.println("Average create execution time: " + 
        (double)executionTime[CREATE]/numOfOps[CREATE] + "ms");
    out.println("Average write_close execution time: " + 
        (double)executionTime[WRITE_CLOSE]/numOfOps[WRITE_CLOSE] + "ms");
  }
  if (totalTime != 0) { 
    out.println("Average operations per second: " + 
        (double)totalOps/totalTime +"ops/s");
  }
  out.println();
}
 

/**
 * Creates a new instance.
 *
 * @param configuration the hadoop configuration
 * @param pathBase base path for all non-absolute location created through this {@link LocationFactory}.
 */
public FileContextLocationFactory(final Configuration configuration, String pathBase) {
  this.configuration = configuration;
  this.pathBase = new Path(pathBase.startsWith("/") ? pathBase : "/" + pathBase);

  int maxCacheSize = configuration.getInt(Configs.Keys.FILE_CONTEXT_CACHE_MAX_SIZE,
                                          Configs.Defaults.FILE_CONTEXT_CACHE_MAX_SIZE);
  this.fileContextCache = CacheBuilder
    .newBuilder()
    .weakKeys()
    .weakValues()
    .maximumSize(maxCacheSize)
    .build(new CacheLoader<UserGroupInformation, FileContext>() {
      @Override
      public FileContext load(UserGroupInformation ugi) throws Exception {
        return ugi.doAs(new PrivilegedExceptionAction<FileContext>() {
          @Override
          public FileContext run() throws UnsupportedFileSystemException {
            return FileContext.getFileContext(configuration);
          }
        });
      }
    });
}
 

private void doAppLogAggregationPostCleanUp() {
  // Remove the local app-log-dirs
  List<Path> localAppLogDirs = new ArrayList<Path>();
  for (String rootLogDir : dirsHandler.getLogDirsForCleanup()) {
    Path logPath = new Path(rootLogDir, applicationId);
    try {
      // check if log dir exists
      lfs.getFileStatus(logPath);
      localAppLogDirs.add(logPath);
    } catch (UnsupportedFileSystemException ue) {
      LOG.warn("Log dir " + rootLogDir + "is an unsupported file system", ue);
      continue;
    } catch (IOException fe) {
      continue;
    }
  }

  if (localAppLogDirs.size() > 0) {
    this.delService.delete(this.userUgi.getShortUserName(), null,
      localAppLogDirs.toArray(new Path[localAppLogDirs.size()]));
  }
}
 

@Override
public FSDataOutputStream createInternal(final Path f,
    final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
    final int bufferSize, final short replication, final long blockSize,
    final Progressable progress, final ChecksumOpt checksumOpt,
    final boolean createParent) throws AccessControlException,
    FileAlreadyExistsException, FileNotFoundException,
    ParentNotDirectoryException, UnsupportedFileSystemException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res;
  try {
    res = fsState.resolve(getUriPath(f), false);
  } catch (FileNotFoundException e) {
    if (createParent) {
      throw readOnlyMountTable("create", f);
    } else {
      throw e;
    }
  }
  assert(res.remainingPath != null);
  return res.targetFileSystem.createInternal(res.remainingPath, flag,
      absolutePermission, bufferSize, replication,
      blockSize, progress, checksumOpt,
      createParent);
}
 

protected static void printResults(PrintStream out) throws UnsupportedFileSystemException {
  out.println("Result of running LoadGenerator against fileSystem: " + 
  FileContext.getFileContext().getDefaultFileSystem().getUri());
  if (numOfOps[OPEN] != 0) {
    out.println("Average open execution time: " + 
        (double)executionTime[OPEN]/numOfOps[OPEN] + "ms");
  }
  if (numOfOps[LIST] != 0) {
    out.println("Average list execution time: " + 
        (double)executionTime[LIST]/numOfOps[LIST] + "ms");
  }
  if (numOfOps[DELETE] != 0) {
    out.println("Average deletion execution time: " + 
        (double)executionTime[DELETE]/numOfOps[DELETE] + "ms");
    out.println("Average create execution time: " + 
        (double)executionTime[CREATE]/numOfOps[CREATE] + "ms");
    out.println("Average write_close execution time: " + 
        (double)executionTime[WRITE_CLOSE]/numOfOps[WRITE_CLOSE] + "ms");
  }
  if (totalTime != 0) { 
    out.println("Average operations per second: " + 
        (double)totalOps/totalTime +"ops/s");
  }
  out.println();
}
 

@BeforeClass
public static void setup() throws AccessControlException,
    FileNotFoundException, UnsupportedFileSystemException, IOException {
  localFS = FileContext.getLocalFSFileContext();
  localFS.delete(new Path(localFSDirBase.getAbsolutePath()), true);
  localFSDirBase.mkdirs();
  // Do not start cluster here
}
 

private void submitDirForDeletion(String userName, Path dir) {
  try {
    lfs.getFileStatus(dir);
    delService.delete(userName, dir, new Path[] {});
  } catch (UnsupportedFileSystemException ue) {
    LOG.warn("Local dir " + dir + " is an unsupported filesystem", ue);
  } catch (IOException ie) {
    // ignore
    return;
  }
}
 

@Override
protected void serviceInit(Configuration conf) throws Exception {
  try {
    //TODO Is this required?
    FileContext.getLocalFSFileContext(conf);
  } catch (UnsupportedFileSystemException e) {
    throw new YarnRuntimeException("Failed to start ContainersLauncher", e);
  }
  super.serviceInit(conf);
}
 

@Override
@SuppressWarnings("unchecked")
public void run() {
  List<Path> localAppLogDirs = new ArrayList<Path>();
  FileContext lfs = getLocalFileContext(getConfig());
  for (String rootLogDir : dirsHandler.getLogDirsForCleanup()) {
    Path logDir = new Path(rootLogDir, applicationId.toString());
    try {
      lfs.getFileStatus(logDir);
      localAppLogDirs.add(logDir);
    } catch (UnsupportedFileSystemException ue) {
      LOG.warn("Unsupported file system used for log dir " + logDir, ue);
      continue;
    } catch (IOException ie) {
      continue;
    }
  }

  // Inform the application before the actual delete itself, so that links
  // to logs will no longer be there on NM web-UI.
  NonAggregatingLogHandler.this.dispatcher.getEventHandler().handle(
    new ApplicationEvent(this.applicationId,
      ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED));
  if (localAppLogDirs.size() > 0) {
    NonAggregatingLogHandler.this.delService.delete(user, null,
      (Path[]) localAppLogDirs.toArray(new Path[localAppLogDirs.size()]));
  }
  try {
    NonAggregatingLogHandler.this.stateStore.removeLogDeleter(
        this.applicationId);
  } catch (IOException e) {
    LOG.error("Error removing log deletion state", e);
  }
}
 

public DockerContainerExecutor() {
  try {
    this.lfs = FileContext.getLocalFSFileContext();
    this.dockerImagePattern = Pattern.compile(DOCKER_IMAGE_PATTERN);
  } catch (UnsupportedFileSystemException e) {
    throw new RuntimeException(e);
  }
}
 

static final FileContext getLfs() {
  try {
    return FileContext.getLocalFSFileContext();
  } catch (UnsupportedFileSystemException e) {
    throw new RuntimeException(e);
  }
}
 

public DefaultContainerExecutor() {
  try {
    this.lfs = FileContext.getLocalFSFileContext();
  } catch (UnsupportedFileSystemException e) {
    throw new RuntimeException(e);
  }
}
 

public BaseContainerManagerTest() throws UnsupportedFileSystemException {
  localFS = FileContext.getLocalFSFileContext();
  localDir =
      new File("target", this.getClass().getSimpleName() + "-localDir")
          .getAbsoluteFile();
  localLogDir =
      new File("target", this.getClass().getSimpleName() + "-localLogDir")
          .getAbsoluteFile();
  remoteLogDir =
    new File("target", this.getClass().getSimpleName() + "-remoteLogDir")
        .getAbsoluteFile();
  tmpDir = new File("target", this.getClass().getSimpleName() + "-tmpDir");
}
 

private static final FileContext getLfs() {
  try {
    return FileContext.getLocalFSFileContext();
  } catch (UnsupportedFileSystemException e) {
    throw new RuntimeException(e);
  }
}
 

@Before
public void setup() throws UnsupportedFileSystemException {
  localFS = FileContext.getLocalFSFileContext();
  tmpDir.mkdirs();
  logsDir.mkdirs();
  remoteLogsDir.mkdirs();
  nmLocalDir.mkdirs();

  // Construct the Container-id
  cId = createContainerId();
}
 

@Before
public void setup() throws UnsupportedFileSystemException {
  localFS = FileContext.getLocalFSFileContext();
  tmpDir.mkdirs();
  logsDir.mkdirs();
  remoteLogsDir.mkdirs();
  nmLocalDir.mkdirs();
  syncBarrier = new CyclicBarrier(2);
}
 

public LocalContainerLauncher(AppContext context,
                              TaskUmbilicalProtocol umbilical) {
  super(LocalContainerLauncher.class.getName());
  this.context = context;
  this.umbilical = umbilical;
      // umbilical:  MRAppMaster creates (taskAttemptListener), passes to us
      // (TODO/FIXME:  pointless to use RPC to talk to self; should create
      // LocalTaskAttemptListener or similar:  implement umbilical protocol
      // but skip RPC stuff)

  try {
    curFC = FileContext.getFileContext(curDir.toURI());
  } catch (UnsupportedFileSystemException ufse) {
    LOG.error("Local filesystem " + curDir.toURI().toString()
              + " is unsupported?? (should never happen)");
  }

  // Save list of files/dirs that are supposed to be present so can delete
  // any extras created by one task before starting subsequent task.  Note
  // that there's no protection against deleted or renamed localization;
  // users who do that get what they deserve (and will have to disable
  // uberization in order to run correctly).
  File[] curLocalFiles = curDir.listFiles();
  localizedFiles = new HashSet<File>(curLocalFiles.length);
  for (int j = 0; j < curLocalFiles.length; ++j) {
    localizedFiles.add(curLocalFiles[j]);
  }

  // Relocalization note/future FIXME (per chrisdo, 20110315):  At moment,
  // full localization info is in AppSubmissionContext passed from client to
  // RM and then to NM for AM-container launch:  no difference between AM-
  // localization and MapTask- or ReduceTask-localization, so can assume all
  // OK.  Longer-term, will need to override uber-AM container-localization
  // request ("needed resources") with union of regular-AM-resources + task-
  // resources (and, if maps and reduces ever differ, then union of all three
  // types), OR will need localizer service/API that uber-AM can request
  // after running (e.g., "localizeForTask()" or "localizeForMapTask()").
}
 

/**
 * Similar to {@link YARNRunner#YARNRunner(Configuration, ResourceMgrDelegate)}
 * but allowing injecting {@link ClientCache}. Enable mocking and testing.
 * @param conf the configuration object
 * @param resMgrDelegate the resource manager delegate
 * @param clientCache the client cache object.
 */
public YARNRunner(Configuration conf, ResourceMgrDelegate resMgrDelegate,
    ClientCache clientCache) {
  this.conf = conf;
  try {
    this.resMgrDelegate = resMgrDelegate;
    this.clientCache = clientCache;
    this.defaultFileContext = FileContext.getFileContext(this.conf);
  } catch (UnsupportedFileSystemException ufe) {
    throw new RuntimeException("Error in instantiating YarnClient", ufe);
  }
}
 

/**
 * Get default file system URI for the cluster (used to ensure consistency
 * of history done/staging locations) over different context
 *
 * @return Default file context
 */
private static FileContext getDefaultFileContext() {
  // If FS_DEFAULT_NAME_KEY was set solely by core-default.xml then we ignore
  // ignore it. This prevents defaulting history paths to file system specified
  // by core-default.xml which would not make sense in any case. For a test
  // case to exploit this functionality it should create core-site.xml
  FileContext fc = null;
  Configuration defaultConf = new Configuration();
  String[] sources;
  sources = defaultConf.getPropertySources(
      CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY);
  if (sources != null &&
      (!Arrays.asList(sources).contains("core-default.xml") ||
      sources.length > 1)) {
    try {
      fc = FileContext.getFileContext(defaultConf);
      LOG.info("Default file system [" +
                fc.getDefaultFileSystem().getUri() + "]");
    } catch (UnsupportedFileSystemException e) {
      LOG.error("Unable to create default file context [" +
          defaultConf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY) +
          "]",
          e);
    }
  }
  else {
    LOG.info("Default file system is set solely " +
        "by core-default.xml therefore -  ignoring");
  }

  return fc;
}
 

/**
 * This constructor has the signature needed by
 * {@link AbstractFileSystem#createFileSystem(URI, Configuration)}.
 * 
 * @param theUri which must be that of ViewFs
 * @param conf
 * @throws IOException
 * @throws URISyntaxException 
 */
ViewFs(final URI theUri, final Configuration conf) throws IOException,
    URISyntaxException {
  super(theUri, FsConstants.VIEWFS_SCHEME, false, -1);
  creationTime = Time.now();
  ugi = UserGroupInformation.getCurrentUser();
  config = conf;
  // Now build  client side view (i.e. client side mount table) from config.
  String authority = theUri.getAuthority();
  fsState = new InodeTree<AbstractFileSystem>(conf, authority) {

    @Override
    protected
    AbstractFileSystem getTargetFileSystem(final URI uri)
      throws URISyntaxException, UnsupportedFileSystemException {
        String pathString = uri.getPath();
        if (pathString.isEmpty()) {
          pathString = "/";
        }
        return new ChRootedFs(
            AbstractFileSystem.createFileSystem(uri, config),
            new Path(pathString));
    }

    @Override
    protected
    AbstractFileSystem getTargetFileSystem(
        final INodeDir<AbstractFileSystem> dir) throws URISyntaxException {
      return new InternalDirOfViewFs(dir, creationTime, ugi, getUri());
    }

    @Override
    protected
    AbstractFileSystem getTargetFileSystem(URI[] mergeFsURIList)
        throws URISyntaxException, UnsupportedFileSystemException {
      throw new UnsupportedFileSystemException("mergefs not implemented yet");
      // return MergeFs.createMergeFs(mergeFsURIList, config);
    }
  };
}
 

@Override
public FileStatus getFileLinkStatus(final Path f)
   throws AccessControlException, FileNotFoundException,
   UnsupportedFileSystemException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res = 
    fsState.resolve(getUriPath(f), false); // do not follow mount link
  return res.targetFileSystem.getFileLinkStatus(res.remainingPath);
}
 

@Override
public FSDataOutputStream createInternal(final Path f,
    final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
    final int bufferSize, final short replication, final long blockSize,
    final Progressable progress, final ChecksumOpt checksumOpt,
    final boolean createParent) throws AccessControlException,
    FileAlreadyExistsException, FileNotFoundException,
    ParentNotDirectoryException, UnsupportedFileSystemException,
    UnresolvedLinkException, IOException {
  throw readOnlyMountTable("create", f);
}
 

@Test(expected=FileAlreadyExistsException.class)
public void testInvalidConfig() throws IOException, URISyntaxException {
  Configuration conf = new Configuration();
  ConfigUtil.addLink(conf, "/internalDir/linkToDir2",
      new Path("file:///dir2").toUri());
  ConfigUtil.addLink(conf, "/internalDir/linkToDir2/linkToDir3",
      new Path("file:///dir3").toUri());
  
  class Foo { };
  
   new InodeTree<Foo>(conf, null) {

    @Override
    protected
    Foo getTargetFileSystem(final URI uri)
      throws URISyntaxException, UnsupportedFileSystemException {
        return null;
    }

    @Override
    protected
    Foo getTargetFileSystem(
        org.apache.hadoop.fs.viewfs.InodeTree.INodeDir<Foo>
                                        dir)
      throws URISyntaxException {
      return null;
    }

    @Override
    protected
    Foo getTargetFileSystem(URI[] mergeFsURIList)
        throws URISyntaxException, UnsupportedFileSystemException {
      return null;
    }
  };
}
 

@Override
public void createSymlink(Path target, Path link, boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, IOException {
  try (WaitRecorder recorder = OperatorStats.getWaitRecorder(operatorStats)) {
    underlyingFs.createSymlink(target, link, createParent);
  } catch(FSError e) {
    throw propagateFSError(e);
  }
}
 

@Override
public FileStatus getFileLinkStatus(Path f) throws AccessControlException, FileNotFoundException,
    UnsupportedFileSystemException, IOException {
  try (WaitRecorder recorder = OperatorStats.getWaitRecorder(operatorStats)) {
    return underlyingFs.getFileLinkStatus(f);
  } catch(FSError e) {
    throw propagateFSError(e);
  }
}
 

@BeforeClass
public static void setup() throws AccessControlException,
    FileNotFoundException, UnsupportedFileSystemException, IOException {
  localFS = FileContext.getLocalFSFileContext();
  localFS.delete(new Path(localFSDirBase.getAbsolutePath()), true);
  localFSDirBase.mkdirs();
  // Do not start cluster here
}
 

private void submitDirForDeletion(String userName, Path dir) {
  try {
    lfs.getFileStatus(dir);
    delService.delete(userName, dir, new Path[] {});
  } catch (UnsupportedFileSystemException ue) {
    LOG.warn("Local dir " + dir + " is an unsupported filesystem", ue);
  } catch (IOException ie) {
    // ignore
    return;
  }
}
 

@Override
protected void serviceInit(Configuration conf) throws Exception {
  try {
    //TODO Is this required?
    FileContext.getLocalFSFileContext(conf);
  } catch (UnsupportedFileSystemException e) {
    throw new YarnRuntimeException("Failed to start ContainersLauncher", e);
  }
  super.serviceInit(conf);
}