org.apache.hadoop.fs.FileUtil#listFiles ( )源码实例Demo

/**
 * @return true if the storage directory should prompt the user prior
 * to formatting (i.e if the directory appears to contain some data)
 * @throws IOException if the SD cannot be accessed due to an IO error
 */
@Override
public boolean hasSomeData() throws IOException {
  // Its alright for a dir not to exist, or to exist (properly accessible)
  // and be completely empty.
  if (!root.exists()) return false;
  
  if (!root.isDirectory()) {
    // a file where you expect a directory should not cause silent
    // formatting
    return true;
  }
  
  if (FileUtil.listFiles(root).length == 0) {
    // Empty dir can format without prompt.
    return false;
  }
  
  return true;
}
 

/**
 * Purge files in the given directory which match any of the set of patterns.
 * The patterns must have a single numeric capture group which determines
 * the associated transaction ID of the file. Only those files for which
 * the transaction ID is less than the <code>minTxIdToKeep</code> parameter
 * are removed.
 */
private static void purgeMatching(File dir, List<Pattern> patterns,
    long minTxIdToKeep) throws IOException {

  for (File f : FileUtil.listFiles(dir)) {
    if (!f.isFile()) continue;
    
    for (Pattern p : patterns) {
      Matcher matcher = p.matcher(f.getName());
      if (matcher.matches()) {
        // This parsing will always succeed since the group(1) is
        // /\d+/ in the regex itself.
        long txid = Long.parseLong(matcher.group(1));
        if (txid < minTxIdToKeep) {
          LOG.info("Purging no-longer needed file " + txid);
          if (!f.delete()) {
            LOG.warn("Unable to delete no-longer-needed data " +
                f);
          }
          break;
        }
      }
    }
  }
}
 

/**
 * List all of the files in 'dir' that match the regex 'pattern'.
 * Then check that this list is identical to 'expectedMatches'.
 * @throws IOException if the dir is inaccessible
 */
public static void assertGlobEquals(File dir, String pattern,
    String ... expectedMatches) throws IOException {
  
  Set<String> found = Sets.newTreeSet();
  for (File f : FileUtil.listFiles(dir)) {
    if (f.getName().matches(pattern)) {
      found.add(f.getName());
    }
  }
  Set<String> expectedSet = Sets.newTreeSet(
      Arrays.asList(expectedMatches));
  Assert.assertEquals("Bad files matching " + pattern + " in " + dir,
      Joiner.on(",").join(expectedSet),
      Joiner.on(",").join(found));
}
 

/**
 * @return true if the storage directory should prompt the user prior
 * to formatting (i.e if the directory appears to contain some data)
 * @throws IOException if the SD cannot be accessed due to an IO error
 */
@Override
public boolean hasSomeData() throws IOException {
  // Its alright for a dir not to exist, or to exist (properly accessible)
  // and be completely empty.
  if (!root.exists()) return false;
  
  if (!root.isDirectory()) {
    // a file where you expect a directory should not cause silent
    // formatting
    return true;
  }
  
  if (FileUtil.listFiles(root).length == 0) {
    // Empty dir can format without prompt.
    return false;
  }
  
  return true;
}
 

/**
 * Purge files in the given directory which match any of the set of patterns.
 * The patterns must have a single numeric capture group which determines
 * the associated transaction ID of the file. Only those files for which
 * the transaction ID is less than the <code>minTxIdToKeep</code> parameter
 * are removed.
 */
private static void purgeMatching(File dir, List<Pattern> patterns,
    long minTxIdToKeep) throws IOException {

  for (File f : FileUtil.listFiles(dir)) {
    if (!f.isFile()) continue;
    
    for (Pattern p : patterns) {
      Matcher matcher = p.matcher(f.getName());
      if (matcher.matches()) {
        // This parsing will always succeed since the group(1) is
        // /\d+/ in the regex itself.
        long txid = Long.parseLong(matcher.group(1));
        if (txid < minTxIdToKeep) {
          LOG.info("Purging no-longer needed file " + txid);
          if (!f.delete()) {
            LOG.warn("Unable to delete no-longer-needed data " +
                f);
          }
          break;
        }
      }
    }
  }
}
 

/**
 * List all of the files in 'dir' that match the regex 'pattern'.
 * Then check that this list is identical to 'expectedMatches'.
 * @throws IOException if the dir is inaccessible
 */
public static void assertGlobEquals(File dir, String pattern,
    String ... expectedMatches) throws IOException {
  
  Set<String> found = Sets.newTreeSet();
  for (File f : FileUtil.listFiles(dir)) {
    if (f.getName().matches(pattern)) {
      found.add(f.getName());
    }
  }
  Set<String> expectedSet = Sets.newTreeSet(
      Arrays.asList(expectedMatches));
  Assert.assertEquals("Bad files matching " + pattern + " in " + dir,
      Joiner.on(",").join(expectedSet),
      Joiner.on(",").join(found));
}
 

@Override
public void purgeLogsOlderThan(long minTxIdToKeep)
    throws IOException {
  LOG.info("Purging logs older than " + minTxIdToKeep);
  File[] files = FileUtil.listFiles(sd.getCurrentDir());
  List<EditLogFile> editLogs = matchEditLogs(files, true);
  for (EditLogFile log : editLogs) {
    if (log.getFirstTxId() < minTxIdToKeep &&
        log.getLastTxId() < minTxIdToKeep) {
      purger.purgeLog(log);
    }
  }
}
 

@Override
public void purgeLogsOlderThan(long minTxIdToKeep)
    throws IOException {
  LOG.info("Purging logs older than " + minTxIdToKeep);
  File[] files = FileUtil.listFiles(sd.getCurrentDir());
  List<EditLogFile> editLogs = matchEditLogs(files, true);
  for (EditLogFile log : editLogs) {
    if (log.getFirstTxId() < minTxIdToKeep &&
        log.getLastTxId() < minTxIdToKeep) {
      purger.purgeLog(log);
    }
  }
}
 

@Override
public boolean delete(Path p, boolean recursive) throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug(String.format("EFS:delete: %s %b", p, recursive));
  }
  
  // The super delete uses the FileUtil.fullyDelete, 
  // but we cannot rely on that because we need to use the elevated 
  // operations to remove the files
  //
  File f = pathToFile(p);
  if (!f.exists()) {
    //no path, return false "nothing to delete"
    return false;
  }
  else if (f.isFile()) {
    return Native.Elevated.deleteFile(p);
  } 
  else if (f.isDirectory()) {
    
    // This is a best-effort attempt. There are race conditions in that
    // child files can be created/deleted after we snapped the list. 
    // No need to protect against that case.
    File[] files = FileUtil.listFiles(f);
    int childCount = files.length;
    
    if (recursive) {
      for(File child:files) {
        if (delete(new Path(child.getPath()), recursive)) {
          --childCount;
        }
      }
    }
    if (childCount == 0) {
      return Native.Elevated.deleteDirectory(p);
    } 
    else {
      throw new IOException("Directory " + f.toString() + " is not empty");
    }
  }
  else {
    // This can happen under race conditions if an external agent 
    // is messing with the file type between IFs
    throw new IOException("Path " + f.toString() + 
        " exists, but is neither a file nor a directory");
  }
}
 

@Override
public boolean delete(Path p, boolean recursive) throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug(String.format("EFS:delete: %s %b", p, recursive));
  }
  
  // The super delete uses the FileUtil.fullyDelete, 
  // but we cannot rely on that because we need to use the elevated 
  // operations to remove the files
  //
  File f = pathToFile(p);
  if (!f.exists()) {
    //no path, return false "nothing to delete"
    return false;
  }
  else if (f.isFile()) {
    return Native.Elevated.deleteFile(p);
  } 
  else if (f.isDirectory()) {
    
    // This is a best-effort attempt. There are race conditions in that
    // child files can be created/deleted after we snapped the list. 
    // No need to protect against that case.
    File[] files = FileUtil.listFiles(f);
    int childCount = files.length;
    
    if (recursive) {
      for(File child:files) {
        if (delete(new Path(child.getPath()), recursive)) {
          --childCount;
        }
      }
    }
    if (childCount == 0) {
      return Native.Elevated.deleteDirectory(p);
    } 
    else {
      throw new IOException("Directory " + f.toString() + " is not empty");
    }
  }
  else {
    // This can happen under race conditions if an external agent 
    // is messing with the file type between IFs
    throw new IOException("Path " + f.toString() + 
        " exists, but is neither a file nor a directory");
  }
}