/**
 * Determine the number of unprocessed flag files in the flag directory
 * 
 * @param fc
 * @return the flag found for this ingest pool
 */
private int countFlagFileBacklog(final FlagDataTypeConfig fc) {
    final MutableInt fileCounter = new MutableInt(0);
    final FileFilter fileFilter = new WildcardFileFilter("*_" + fc.getIngestPool() + "_" + fc.getDataName() + "_*.flag");
    final FileVisitor<java.nio.file.Path> visitor = new SimpleFileVisitor<java.nio.file.Path>() {
        
        @Override
        public FileVisitResult visitFile(java.nio.file.Path path, BasicFileAttributes attrs) throws IOException {
            if (fileFilter.accept(path.toFile())) {
                fileCounter.increment();
            }
            return super.visitFile(path, attrs);
        }
    };
    try {
        Files.walkFileTree(Paths.get(fmc.getFlagFileDirectory()), visitor);
    } catch (IOException e) {
        // unable to get a flag count....
        log.error("Unable to get flag file count", e);
        return -1;
    }
    return fileCounter.intValue();
}
 

public void delete() {
    FileVisitor<Path> fileVisitor = new SimpleFileVisitor<>() {
        @Override
        public FileVisitResult visitFile(Path file, BasicFileAttributes attributes) throws IOException {
            Files.delete(file);
            return FileVisitResult.CONTINUE;
        }

        @Override
        public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
            Files.delete(dir);
            return FileVisitResult.CONTINUE;
        }
    };

    try {
        Files.walkFileTree(dir, fileVisitor);
    } catch (IOException e) {
        throw new ResourceException("failed to delete dir, path={}", dir, e);
    }
}
 

/**
 * Very simplified stand-in for {@link Files#walkFileTree(Path, FileVisitor)} where we control the traversal order.
 *
 * @param path the start path.
 * @param visitor the visitor processing the files.
 * @throws IOException forwarded from nested I/O operations.
 */
private static void walkFileTreeOurWay(Path path, FileVisitor<? super Path> visitor) throws IOException {
    if (!path.toFile().isDirectory()) {
        throw new IOException("Directory expected.");
    }

    try (Stream<Path> children = Files.list(path)) {
        children.sorted(Bucket::compareUtf8Binary)
                .filter(p -> p.toFile().isFile())
                .forEach(p -> {
            try {
                BasicFileAttributes attrs = Files.readAttributes(p, BasicFileAttributes.class);
                visitor.visitFile(p, attrs);
            } catch (IOException e) {
                Exceptions.handle(e);
            }
        });
    }
}
 

@Test
public void testWalkRelativeFileTreeWhenPathIsAFile() throws IOException {
  FakeProjectFilesystem filesystem = new FakeProjectFilesystem();
  filesystem.touch(Paths.get("A.txt"));

  List<Path> filesVisited = new ArrayList<>();

  FileVisitor<Path> fileVisitor =
      new SimpleFileVisitor<Path>() {
        @Override
        public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) {
          filesVisited.add(path);
          return FileVisitResult.CONTINUE;
        }
      };

  filesystem.walkRelativeFileTree(Paths.get("A.txt"), fileVisitor);

  // Despite the awkward name, "contains" implies an exact match.
  assertThat(filesVisited, contains(Paths.get("A.txt")));
}
 

@Test
public void testWalkFileTreeWhenPathIsAFile() throws IOException {
  FakeProjectFilesystem filesystem = new FakeProjectFilesystem();
  filesystem.touch(Paths.get("dir/dir2/A.txt"));
  filesystem.touch(Paths.get("dir/B.txt"));

  List<Path> filesVisited = new ArrayList<>();

  FileVisitor<Path> fileVisitor =
      new SimpleFileVisitor<Path>() {
        @Override
        public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) {
          filesVisited.add(filesystem.relativize(path).getPath());
          return FileVisitResult.CONTINUE;
        }
      };

  filesystem.walkFileTree(
      Paths.get("dir"), EnumSet.of(FileVisitOption.FOLLOW_LINKS), fileVisitor);

  // Despite the awkward name, "contains" implies an exact match.
  assertThat(
      filesVisited, containsInAnyOrder(Paths.get("dir/dir2/A.txt"), Paths.get("dir/B.txt")));
}
 

@Override
public boolean execute(final FileVisitor<Path> visitor) throws IOException {
    final List<PathWithAttributes> sortedPaths = getSortedPaths();
    trace("Sorted paths:", sortedPaths);

    for (final PathWithAttributes element : sortedPaths) {
        try {
            visitor.visitFile(element.getPath(), element.getAttributes());
        } catch (final IOException ioex) {
            LOGGER.error("Error in post-rollover Delete when visiting {}", element.getPath(), ioex);
            visitor.visitFileFailed(element.getPath(), ioex);
        }
    }
    // TODO return (visitor.success || ignoreProcessingFailure)
    return true; // do not abort rollover even if processing failed
}
 

@Override
protected FileVisitor<Path> createFileVisitor(final Path basePath,
        final List<PathCondition> conditions) {
    return new SimpleFileVisitor<Path>() {
        @Override
        public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
            for (final PathCondition pathFilter : conditions) {
                final Path relative = basePath.relativize(file);
                if (!pathFilter.accept(basePath, relative, attrs)) {
                    LOGGER.trace("Not defining posix attribute base={}, relative={}", basePath, relative);
                    return FileVisitResult.CONTINUE;
                }
            }
            FileUtils.defineFilePosixAttributeView(file, filePermissions, fileOwner, fileGroup);
            return FileVisitResult.CONTINUE;
        }
    };
}
 

public static Collection<String> resolveUserPropertiesLocations() {
    final String conf = resolveUserPropertiesLocation();
    final Set<String> locations = new LinkedHashSet<>();

    // Additional locations
    if (ObjectHelper.isNotEmpty(conf)) {
        Path root = Paths.get(conf);
        FileVisitor<Path> visitor = new SimpleFileVisitor<>() {
            @Override
            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                Objects.requireNonNull(file);
                Objects.requireNonNull(attrs);

                final String path = file.toFile().getAbsolutePath();
                if (path.endsWith(".properties")) {
                    locations.add(path);
                }

                return FileVisitResult.CONTINUE;
            }
        };

        if (Files.exists(root)) {
            try {
                Files.walkFileTree(root, visitor);
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }
    }

    return locations;
}
 

private static void deleteRecursive ( final File updates ) throws IOException
{
    System.out.println ( "Deleting: " + updates );

    final FileVisitor<? super Path> visitor = new RecursiveDeleteVisitior ();

    Files.walkFileTree ( updates.toPath (), visitor );
}
 

public void walkResources(FileVisitor<? super Path> visitor) throws URISyntaxException, IOException {
  try (FileSystem zipFs =
               FileSystems.newFileSystem(artifactURL.toURI(), new HashMap<>())) {
    Path resourcesPath = zipFs.getPath("/resources");
    if(Files.isDirectory(resourcesPath)) {
      Files.walkFileTree(resourcesPath, visitor);
    }
  }
}
 

/**
 * Populate the parameters
 *
 * @return the parameters. Basically all the errors with lookuped paths
 */
@Parameters(name = "{index}: {0}")
public static Iterable<Object[]> getTracePaths() {
    final List<Object[]> dirs = new LinkedList<>();

    FileVisitor<Path> visitor = new SimpleFileVisitor<Path>() {
        @Override
        public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
            final Path fileName = dir.getFileName();
            String res = ERRORS.get(fileName.toString());
            if (res != null) {
                dirs.add(new Object[] { dir.toFile(), res });
                return FileVisitResult.SKIP_SUBTREE;
            }
            return FileVisitResult.CONTINUE;
        }

    };
    /* TODO: add way of handling an error in a trace during a run */
    /* when that is done, we can add regression/stream/fail */

    Path badMetadata = BASE_PATH.resolve(Paths.get("regression", "metadata", "fail"));
    try {
        Files.walkFileTree(badMetadata, visitor);
    } catch (IOException e) {
        e.printStackTrace();
    }
    return dirs;
}
 

@Nonnull
public static Path walkFileTree (@Nonnull final Path aStart,
                                 @Nonnegative final int nMaxDepth,
                                 @Nonnull final FileVisitor <? super Path> aVisitor)
{
  return walkFileTree (aStart, EnumSet.noneOf (FileVisitOption.class), nMaxDepth, aVisitor);
}
 

private void includeFiles(Jar jar, String destinationRoot, String sourceRoot)
        throws IOException {
    Path sourceRootPath = Paths.get(sourceRoot);
    // iterate through sources
    // put each source on the jar
    FileVisitor<Path> visitor = new SimpleFileVisitor<Path>() {
        @Override
        public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
            Path relativePath = sourceRootPath.relativize(file);
            String destination = destinationRoot != null ?
                    destinationRoot + "/" + relativePath.toString() : //TODO
                    relativePath.toString();

            addFileToJar(jar, destination, file.toAbsolutePath().toString());
            return FileVisitResult.CONTINUE;
        }
    };
    File dir = new File(sourceRoot);
    if (dir.isFile()) {
        addFileToJar(jar, destinationRoot, dir.getAbsolutePath());
    } else if (dir.isDirectory()) {
        walkFileTree(sourceRootPath, visitor);
    } else {
        warn("Skipping resource in bundle %s: %s (File Not Found)\n",
                bundleSymbolicName, sourceRoot);
    }
}
 

public void check() throws IOException{
    FileVisitor<Path> fileProcessor = new ProcessFile();
    Files.walkFileTree(Paths.get(BASE_DIR), fileProcessor);
    double success = (double)mentionsArticlesWeGotRight/(double)articlesWithLocations; 
    double focusSuccess = (double)focusArticlesWeGotRight/(double)articlesWithLocations; 
    logger.info("Checked "+articlesWithLocations+" Articles - Base success rate: "+success);
    logger.info("Checked "+articlesWithLocations+" Articles - Aboutness success rate: "+focusSuccess);
}
 

public void check() throws IOException {
    FileVisitor<Path> fileProcessor = new ProcessFile();
    Files.walkFileTree(Paths.get(NYT_BASE_DIR), fileProcessor);
    double success = (double)articlesWeGotRight/(double)articlesWithLocations; 
    double focusSuccess = (double)focusArticlesWeGotRight/(double)articlesWithLocations; 
    logger.info("Checked "+articlesWithLocations+" Articles - Base success rate: "+success);
    logger.info("Checked "+articlesWithLocations+" Articles - Aboutness success rate: "+focusSuccess);        
}
 

private void walkNodeTree(FileVisitor<Map.Entry<String, WorkspaceNode>> visitor)
    throws IOException {
  Stack<Iterator<Map.Entry<String, WorkspaceNode>>> iterators = new Stack<>();
  Stack<Map.Entry<String, WorkspaceNode>> groups = new Stack<>();
  iterators.push(this.children.entrySet().iterator());
  while (!iterators.isEmpty()) {
    if (!iterators.peek().hasNext()) {
      if (groups.isEmpty()) {
        break;
      }
      visitor.postVisitDirectory(groups.pop(), null);
      iterators.pop();
      continue;
    }
    Map.Entry<String, WorkspaceNode> nextEntry = iterators.peek().next();
    WorkspaceNode nextNode = nextEntry.getValue();
    if (nextNode instanceof WorkspaceGroup) {
      visitor.preVisitDirectory(nextEntry, null);
      WorkspaceGroup nextGroup = (WorkspaceGroup) nextNode;
      groups.push(nextEntry);
      iterators.push(nextGroup.getChildren().entrySet().iterator());
    } else if (nextNode instanceof WorkspaceFileRef) {
      visitor.visitFile(nextEntry, null);
    } else {
      // Unreachable
      throw new HumanReadableException(
          "Expected a workspace to only contain groups and file references");
    }
  }
}
 

private void walkNodeTree(FileVisitor<Map.Entry<String, WorkspaceNode>> visitor)
    throws IOException {
  Stack<Iterator<Map.Entry<String, WorkspaceNode>>> iterators = new Stack<>();
  Stack<Map.Entry<String, WorkspaceNode>> groups = new Stack<>();
  iterators.push(this.children.entrySet().iterator());
  while (!iterators.isEmpty()) {
    if (!iterators.peek().hasNext()) {
      if (groups.isEmpty()) {
        break;
      }
      visitor.postVisitDirectory(groups.pop(), null);
      iterators.pop();
      continue;
    }
    Map.Entry<String, WorkspaceNode> nextEntry = iterators.peek().next();
    WorkspaceNode nextNode = nextEntry.getValue();
    if (nextNode instanceof WorkspaceGroup) {
      visitor.preVisitDirectory(nextEntry, null);
      WorkspaceGroup nextGroup = (WorkspaceGroup) nextNode;
      groups.push(nextEntry);
      iterators.push(nextGroup.getChildren().entrySet().iterator());
    } else if (nextNode instanceof WorkspaceFileRef) {
      visitor.visitFile(nextEntry, null);
    } else {
      // Unreachable
      throw new HumanReadableException(
          "Expected a workspace to only contain groups and file references");
    }
  }
}
 

@Override
public void walkRelativeFileTree(
    Path pathRelativeToProjectRoot, FileVisitor<Path> fileVisitor, boolean skipIgnored)
    throws IOException {
  walkRelativeFileTree(
      pathRelativeToProjectRoot,
      EnumSet.of(FileVisitOption.FOLLOW_LINKS),
      fileVisitor,
      skipIgnored);
}
 

private void walkRelativeFileTree(
    Path pathRelativeToProjectRoot,
    EnumSet<FileVisitOption> visitOptions,
    FileVisitor<Path> fileVisitor)
    throws IOException {
  walkRelativeFileTree(pathRelativeToProjectRoot, visitOptions, fileVisitor, true);
}
 

/** Walks a project-root relative file tree with a visitor and visit options. */
@Override
public void walkRelativeFileTree(
    Path pathRelativeToProjectRoot,
    EnumSet<FileVisitOption> visitOptions,
    FileVisitor<Path> fileVisitor,
    boolean skipIgnored)
    throws IOException {
  walkRelativeFileTree(
      pathRelativeToProjectRoot,
      visitOptions,
      fileVisitor,
      skipIgnored ? input -> !isIgnored(relativize(input)) : input -> true);
}
 

void walkRelativeFileTree(
    Path pathRelativeToProjectRoot,
    EnumSet<FileVisitOption> visitOptions,
    FileVisitor<Path> fileVisitor,
    DirectoryStream.Filter<? super Path> ignoreFilter)
    throws IOException {
  Path rootPath = getPathForRelativePath(pathRelativeToProjectRoot);
  walkFileTreeWithPathMapping(
      rootPath, visitOptions, fileVisitor, ignoreFilter, path -> relativize(path).getPath());
}
 

private void walkFileTree(
    Path root, Set<FileVisitOption> options, FileVisitor<Path> fileVisitor, boolean skipIgnored)
    throws IOException {
  walkFileTree(
      root,
      options,
      fileVisitor,
      skipIgnored ? input -> !isIgnored(relativize(input)) : input -> true);
}
 

void walkFileTree(
    Path root,
    Set<FileVisitOption> options,
    FileVisitor<Path> fileVisitor,
    DirectoryStream.Filter<? super Path> ignoreFilter)
    throws IOException {
  root = getPathForRelativePath(root);
  new FileTreeWalker(root, options, fileVisitor, ignoreFilter).walk();
}
 

FileTreeWalker(
    Path root,
    Set<FileVisitOption> options,
    FileVisitor<Path> pathFileVisitor,
    DirectoryStream.Filter<? super Path> ignoreFilter) {
  this.followLinks = options.contains(FileVisitOption.FOLLOW_LINKS);
  this.visitor = pathFileVisitor;
  this.root = root;
  this.state = new ArrayDeque<>();
  this.ignoreFilter = ignoreFilter;
}
 

@Override
public void walkRelativeFileTree(
    Path pathRelativeToProjectRoot,
    EnumSet<FileVisitOption> visitOptions,
    FileVisitor<Path> fileVisitor)
    throws IOException {
  filesystemParent.walkFileTreeWithPathMapping(
      projectRoot.resolve(pathRelativeToProjectRoot),
      visitOptions,
      fileVisitor,
      this::shouldExplorePaths,
      this::relativize);
}
 

@Override
public void walkFileTree(
    Path pathRelativeToProjectRoot, Set<FileVisitOption> options, FileVisitor<Path> fileVisitor)
    throws IOException {
  filesystemParent.walkFileTree(
      projectRoot.resolve(pathRelativeToProjectRoot),
      options,
      fileVisitor,
      this::shouldExplorePaths);
}
 

/**
 * Walks a project-root relative file tree with a visitor and visit options.
 *
 * <p>This is deprecated. Please use {@link ProjectFilesystemView#walkRelativeFileTree(Path,
 * EnumSet, FileVisitor)} instead.
 */
@Deprecated
void walkRelativeFileTree(
    Path pathRelativeToProjectRoot,
    EnumSet<FileVisitOption> visitOptions,
    FileVisitor<Path> fileVisitor,
    boolean skipIgnored)
    throws IOException;
 

@Test
public void testWalkRelativeFileTree() throws IOException {
  FakeProjectFilesystem filesystem = new FakeProjectFilesystem();
  filesystem.touch(Paths.get("root/A.txt"));
  filesystem.touch(Paths.get("root/A/B/C.txt"));
  filesystem.touch(Paths.get("root/A/B.txt"));

  List<Path> filesVisited = new ArrayList<>();

  FileVisitor<Path> fileVisitor =
      new SimpleFileVisitor<Path>() {
        @Override
        public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) {
          filesVisited.add(path);
          return FileVisitResult.CONTINUE;
        }
      };

  filesystem.walkRelativeFileTree(Paths.get("root"), fileVisitor);
  assertThat(
      filesVisited,
      containsInAnyOrder(
          Paths.get("root/A.txt"), Paths.get("root/A/B/C.txt"), Paths.get("root/A/B.txt")));

  filesVisited.clear();
  filesystem.walkRelativeFileTree(Paths.get("root/A"), fileVisitor);
  assertThat(
      filesVisited, containsInAnyOrder(Paths.get("root/A/B/C.txt"), Paths.get("root/A/B.txt")));
}
 

public boolean execute(final FileVisitor<Path> visitor) throws IOException {
    final long start = System.nanoTime();
    LOGGER.debug("Starting {}", this);

    Files.walkFileTree(getBasePath(), options, maxDepth, visitor);

    final double duration = System.nanoTime() - start;
    LOGGER.debug("{} complete in {} seconds", getClass().getSimpleName(), duration / TimeUnit.SECONDS.toNanos(1));

    // TODO return (visitor.success || ignoreProcessingFailure)
    return true; // do not abort rollover even if processing failed
}
 

@Test
public void testCreateFileVisitorTestModeIsActionTestMode() {
    final DeleteAction delete = createAnyFilter("any", true, 0, false);
    assertFalse(delete.isTestMode());
    final FileVisitor<Path> visitor = delete.createFileVisitor(delete.getBasePath(), delete.getPathConditions());
    assertTrue(visitor instanceof DeletingVisitor);
    assertFalse(((DeletingVisitor) visitor).isTestMode());

    final DeleteAction deleteTestMode = createAnyFilter("any", true, 0, true);
    assertTrue(deleteTestMode.isTestMode());
    final FileVisitor<Path> testVisitor = deleteTestMode.createFileVisitor(delete.getBasePath(),
            delete.getPathConditions());
    assertTrue(testVisitor instanceof DeletingVisitor);
    assertTrue(((DeletingVisitor) testVisitor).isTestMode());
}