/**
 * @param client recommender to load
 * @param dataDirectory a directory containing data files from which user and item IDs should be read
 * @param steps number of load steps to run
 */
public LoadRunner(MyrrixRecommender client, File dataDirectory, int steps) throws IOException {
  Preconditions.checkNotNull(client);
  Preconditions.checkNotNull(dataDirectory);
  Preconditions.checkArgument(steps > 0);  
  
  log.info("Reading IDs...");    
  FastIDSet userIDsSet = new FastIDSet();
  FastIDSet itemIDsSet = new FastIDSet();
  Splitter comma = Splitter.on(',');
  for (File f : dataDirectory.listFiles(new PatternFilenameFilter(".+\\.csv(\\.(zip|gz))?"))) {
    for (CharSequence line : new FileLineIterable(f)) {
      Iterator<String> it = comma.split(line).iterator();
      userIDsSet.add(Long.parseLong(it.next()));
      itemIDsSet.add(Long.parseLong(it.next()));
    }
  }
  
  this.client = client;    
  this.uniqueUserIDs = userIDsSet.toArray();
  this.uniqueItemIDs = itemIDsSet.toArray();
  this.steps = steps;
}
 

protected static Collection<Object[]> data(String first) {
  // inUrl = "file:/home/fred/calcite/core/target/test-classes/sql/agg.iq"
  final URL inUrl = JdbcTest.class.getResource("/" + n2u(first));
  final File firstFile = Sources.of(inUrl).file();
  final int commonPrefixLength = firstFile.getAbsolutePath().length() - first.length();
  final File dir = firstFile.getParentFile();
  final List<String> paths = new ArrayList<>();
  final FilenameFilter filter = new PatternFilenameFilter(".*\\.iq$");
  for (File f : Util.first(dir.listFiles(filter), new File[0])) {
    paths.add(f.getAbsolutePath().substring(commonPrefixLength));
  }
  return Lists.transform(paths, path -> new Object[] {path});
}
 

@TestFactory
public Stream<DynamicTest> parse() {
  File[] testFiles = testcaseRoot.listFiles(new PatternFilenameFilter("^struct\\d+\\.json$"));

  return Arrays.stream(testFiles)
      .map(f -> dynamicTest(f.getName(), () -> {
        Struct struct = ObjectMapperFactory.INSTANCE.readValue(f, Struct.class);
        assertNotNull(struct);

      }));
}
 

public AbstractSourceConnectorConfig(ConfigDef definition, Map<?, ?> originals, boolean bufferedInputStream) {
  super(definition, originals);
  this.bufferedInputStream = bufferedInputStream;
  this.inputPath = ConfigUtils.getAbsoluteFile(this, INPUT_PATH_CONFIG);
  this.cleanupPolicy = ConfigUtils.getEnum(CleanupPolicy.class, this, CLEANUP_POLICY_CONF);

  if (finishedPathRequired()) {
    this.finishedPath = ConfigUtils.getAbsoluteFile(this, FINISHED_PATH_CONFIG);
  } else {
    this.finishedPath = null;
  }

  this.errorPath = ConfigUtils.getAbsoluteFile(this, ERROR_PATH_CONFIG);
  this.haltOnError = this.getBoolean(HALT_ON_ERROR_CONF);
  this.minimumFileAgeMS = this.getLong(FILE_MINIMUM_AGE_MS_CONF);
  this.batchSize = this.getInt(BATCH_SIZE_CONF);
  this.topic = this.getString(TOPIC_CONF);
  this.emptyPollWaitMs = this.getLong(EMPTY_POLL_WAIT_MS_CONF);
  this.processingFileExtension = this.getString(PROCESSING_FILE_EXTENSION_CONF);
  this.timestampMode = ConfigUtils.getEnum(TimestampMode.class, this, TIMESTAMP_MODE_CONF);
  final String inputPatternText = this.getString(INPUT_FILE_PATTERN_CONF);
  final Pattern inputPattern = Pattern.compile(inputPatternText);
  this.inputFilenameFilter = new PatternFilenameFilter(inputPattern);
  this.fileSortAttributes = ConfigUtils.getEnums(FileAttribute.class, this, FILE_SORT_ATTRIBUTES_CONF);
  this.taskIndex = getInt(TASK_INDEX_CONF);
  this.taskCount = getInt(TASK_COUNT_CONF);
  this.taskPartitioner = ConfigUtils.getEnum(TaskPartitioner.class, this, TASK_PARTITIONER_CONF);

  if (bufferedInputStream) {
    this.fileBufferSizeBytes = getInt(FILE_BUFFER_SIZE_CONF);
  } else {
    this.fileBufferSizeBytes = 0;
  }
  this.metadataLocation = ConfigUtils.getEnum(MetadataLocation.class, this, METADATA_LOCATION_CONF);
  this.metadataField = this.getString(METADATA_FIELD_CONF);
}
 

@Test
public void recordingTestThatShouldBeRecordedAndRetained() {
    File target = vncRecordingDirectory.getRoot();
    try (
        // recordAll {
        // To do this, simply add extra parameters to the rule constructor:
        BrowserWebDriverContainer chrome = new BrowserWebDriverContainer()
            .withCapabilities(new ChromeOptions())
            .withRecordingMode(RECORD_ALL, target)
            // }
            .withRecordingFileFactory(new DefaultRecordingFileFactory())
    ) {
        chrome.start();

        doSimpleExplore(chrome);
        chrome.afterTest(new TestDescription() {
            @Override
            public String getTestId() {
                return getFilesystemFriendlyName();
            }

            @Override
            public String getFilesystemFriendlyName() {
                return "ChromeThatRecordsAllTests-recordingTestThatShouldBeRecordedAndRetained";
            }
        }, Optional.empty());

        String[] files = vncRecordingDirectory.getRoot().list(new PatternFilenameFilter("PASSED-.*\\.flv"));
        assertEquals("Recorded file not found", 1, files.length);
    }
}
 

@Test
public void recordingTestThatShouldBeRecordedAndRetained() {
    File target = vncRecordingDirectory.getRoot();
    try (
        // recordFailing {
        // or if you only want videos for test failures:
        BrowserWebDriverContainer chrome = new BrowserWebDriverContainer()
            .withCapabilities(new ChromeOptions())
            .withRecordingMode(RECORD_FAILING, target)
            // }
            .withRecordingFileFactory(new DefaultRecordingFileFactory())
    ) {
        chrome.start();

        doSimpleExplore(chrome);
        chrome.afterTest(new TestDescription() {
            @Override
            public String getTestId() {
                return getFilesystemFriendlyName();
            }

            @Override
            public String getFilesystemFriendlyName() {
                return "ChromeThatRecordsFailingTests-recordingTestThatShouldBeRecordedAndRetained";
            }
        }, Optional.of(new RuntimeException("Force writing of video file.")));

        String[] files = vncRecordingDirectory.getRoot().list(new PatternFilenameFilter("FAILED-.*\\.flv"));
        assertEquals("Recorded file not found", 1, files.length);
    }

}
 

protected static Collection<Object[]> data(String first) {
  // inUrl = "file:/home/fred/calcite/core/target/test-classes/sql/agg.iq"
  final URL inUrl = JdbcTest.class.getResource("/" + n2u(first));
  final File firstFile = Sources.of(inUrl).file();
  final int commonPrefixLength = firstFile.getAbsolutePath().length() - first.length();
  final File dir = firstFile.getParentFile();
  final List<String> paths = new ArrayList<>();
  final FilenameFilter filter = new PatternFilenameFilter(".*\\.iq$");
  for (File f : Util.first(dir.listFiles(filter), new File[0])) {
    paths.add(f.getAbsolutePath().substring(commonPrefixLength));
  }
  return Lists.transform(paths, path -> new Object[] {path});
}
 

private File getStructureFile(File dirWithDb) throws InvalidFileException {
  //Look for the STR file
  File[] dbfiles = dirWithDb.listFiles(new PatternFilenameFilter(Pattern.compile(".*\\.STR")));
  if (dbfiles == null || dbfiles.length != 1) {
    throw new InvalidFileException("I expected one .STR file in the zip");
  }
  return dbfiles[0];
}
 

private static Multimap<Long,RecommendedItem> readAndCopyDataFiles(File dataDir, File tempDir) throws IOException {
  Multimap<Long,RecommendedItem> data = ArrayListMultimap.create();
  for (File dataFile : dataDir.listFiles(new PatternFilenameFilter(".+\\.csv(\\.(zip|gz))?"))) {
    log.info("Reading {}", dataFile);
    int count = 0;
    for (CharSequence line : new FileLineIterable(dataFile)) {
      Iterator<String> parts = COMMA_TAB_SPLIT.split(line).iterator();
      long userID = Long.parseLong(parts.next());
      long itemID = Long.parseLong(parts.next());
      if (parts.hasNext()) {
        String token = parts.next().trim();
        if (!token.isEmpty()) {
          data.put(userID, new GenericRecommendedItem(itemID, LangUtils.parseFloat(token)));
        }
        // Ignore remove lines
      } else {
        data.put(userID, new GenericRecommendedItem(itemID, 1.0f));
      }
      if (++count % 1000000 == 0) {
        log.info("Finished {} lines", count);
      }
    }

    Files.copy(dataFile, new File(tempDir, dataFile.getName()));
  }
  return data;
}
 

public TaggerEmbeddings(final File modelFolder, final double beta, final int maxTagsPerWord,
		final CutoffsDictionaryInterface cutoffs) throws IOException {
	super(cutoffs, beta, loadCategories(new File(modelFolder, "categories")), maxTagsPerWord);
	try {
		final FilenameFilter embeddingsFileFilter = new PatternFilenameFilter("embeddings.*");

		// If we're using POS tags or lexical features, load l.
		this.posFeatures = loadSparseFeatures(new File(modelFolder + "/postags"));
		this.lexicalFeatures = loadSparseFeatures(new File(modelFolder + "/frequentwords"));

		// Load word embeddings.
		embeddingsFeatures = loadEmbeddings(true, modelFolder.listFiles(embeddingsFileFilter));

		// Load embeddings for capitalization and suffix features.
		discreteFeatures = new HashMap<>();
		discreteFeatures.putAll(loadEmbeddings(false, new File(modelFolder, "capitals")));
		discreteFeatures.putAll(loadEmbeddings(false, new File(modelFolder, "suffix")));
		totalFeatures = (embeddingsFeatures.get(unknownLower).length + discreteFeatures.get(unknownSuffix).length
				+ discreteFeatures.get(capsLower).length + posFeatures.size() + lexicalFeatures.size())
				* (2 * contextWindow + 1);

		// Load the list of categories used by the model.
		categoryToIndex = new HashMap<>();
		for (int i = 0; i < lexicalCategories.size(); i++) {
			categoryToIndex.put(lexicalCategories.get(i), i);
		}

		// Load the weight matrix used by the classifier.
		weightMatrix = new DenseMatrix(lexicalCategories.size(), totalFeatures);
		loadMatrix(weightMatrix, new File(modelFolder, "classifier"));

		weightMatrixRows = new ArrayList<>(lexicalCategories.size());
		for (int i = 0; i < lexicalCategories.size(); i++) {
			final Vector row = new DenseVector(totalFeatures);
			for (int j = 0; j < totalFeatures; j++) {
				row.set(j, weightMatrix.get(i, j));
			}
			weightMatrixRows.add(row);
		}

		bias = new DenseVector(lexicalCategories.size());

		loadVector(bias, new File(modelFolder, "bias"));

	} catch (final Exception e) {
		throw new RuntimeException(e);
	}
}
 

public TaggerEmbeddings(File modelFolder, int maxSentenceLength, double beta, int maxTagsPerWord) {
  try {
    FilenameFilter embeddingsFileFilter = new PatternFilenameFilter("embeddings.*");

    // If we're using POS tags or lexical features, load l.
    this.posFeatures = loadSparseFeatures(new File(modelFolder + "/postags"));     
    this.lexicalFeatures = loadSparseFeatures(new File(modelFolder + "/frequentwords"));     
    
    // Load word embeddings.
    embeddingsFeatures = loadEmbeddings(true, modelFolder.listFiles(embeddingsFileFilter));
    
    // Load embeddings for capitalization and suffix features.
    discreteFeatures = new HashMap<String, double[]>();
    discreteFeatures.putAll(loadEmbeddings(false, new File(modelFolder, "capitals")));
    discreteFeatures.putAll(loadEmbeddings(false, new File(modelFolder, "suffix")));
    totalFeatures = 
      (embeddingsFeatures.get(unknownLower).length + 
          discreteFeatures.get(unknownSuffix).length + 
          discreteFeatures.get(capsLower).length +
          posFeatures.size()  + lexicalFeatures.size())       * (2 * contextWindow + 1);
    
    // Load the list of categories used by the model.
    lexicalCategories = loadCategories(new File(modelFolder, "categories"));
    
    // Load the weight matrix used by the classifier.
    weightMatrix = new DenseMatrix(lexicalCategories.size(), totalFeatures);
    loadMatrix(weightMatrix, new File(modelFolder, "classifier"));

    weightMatrixRows = new ArrayList<Vector>(lexicalCategories.size());
    for (int i=0; i<lexicalCategories.size(); i++) {
      Vector row = new DenseVector(totalFeatures);
      for (int j=0; j<totalFeatures; j++) {
        row.set(j, weightMatrix.get(i, j));
      }
      weightMatrixRows.add(row);
    }

    bias = new DenseVector(lexicalCategories.size());
    this.beta = beta;
    this.maxTagsPerWord = maxTagsPerWord;

    int maxCategoryID = 0;
    for (Category c : lexicalCategories) {
      maxCategoryID = Math.max(maxCategoryID, c.getID());
    }
    
    this.tagDict = ImmutableMap.copyOf(loadTagDictionary(modelFolder));
    
    terminalFactory = new SyntaxTreeNodeFactory(maxSentenceLength, maxCategoryID);
    loadVector(bias, new File(modelFolder, "bias"));

  }
  catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

public String[] unprocessedImportItemPdfFiles(ImportItem importItem) {
    File folder = new File(bulkProductDataFolder);
    return folder.list(new PatternFilenameFilter("(?i)" + Pattern.quote(importItem.getIdentifier()) + "\\.(pdf)"));
}
 

public String[] unprocessedImportItemHtmlFiles(ImportItem importItem) {
    File folder = new File(bulkProductDataFolder);
    return folder.list(new PatternFilenameFilter("(?i)" + Pattern.quote(importItem.getIdentifier()) + "\\.(htm|html)"));
}
 

public String[] existingImportItemFiles(ImportItem importItem) {
    File folder = new File(existingPdfFileFolder);
    return folder.list(new PatternFilenameFilter("(?i)" + Pattern.quote(importItem.getIdentifier()) + "\\.(pdf)"));
}
 

public String[] unprocessedImportItemJacketImages(ImportItem importItem) {
    File folder = new File(bulkProductDataFolder);
    logger.debug("unprocessedImportItemJacketImages folder: " + folder.getAbsolutePath());
    return folder.list(new PatternFilenameFilter("(?i)" + Pattern.quote(importItem.getIdentifier()) + "\\.(jpg|png)"));
}
 

@Test public void testMerge() throws Exception {
  String dataDir =
      Paths.get(System.getenv("TEST_WORKSPACE"), System.getenv("TEST_BINARY"))
          .resolveSibling("testing/manifestmerge")
          .toString()
          .replace("\\", "/");

  final Path mergerManifest = rlocation(dataDir + "/merger/AndroidManifest.xml");
  final Path mergeeManifestOne = rlocation(dataDir + "/mergeeOne/AndroidManifest.xml");
  final Path mergeeManifestTwo = rlocation(dataDir + "/mergeeTwo/AndroidManifest.xml");
  assertThat(mergerManifest.toFile().exists()).isTrue();
  assertThat(mergeeManifestOne.toFile().exists()).isTrue();
  assertThat(mergeeManifestTwo.toFile().exists()).isTrue();

  // The following code retrieves the path of the only AndroidManifest.xml in the expected/
  // manifests directory. Unfortunately, this test runs internally and externally and the files
  // have different names.
  final File expectedManifestDirectory =
      mergerManifest.getParent().resolveSibling("expected").toFile();
  final String[] debug =
      expectedManifestDirectory.list(new PatternFilenameFilter(".*AndroidManifest\\.xml$"));
  assertThat(debug).isNotNull();
  final File[] expectedManifestDirectoryManifests =
      expectedManifestDirectory.listFiles((File dir, String name) -> true);
  assertThat(expectedManifestDirectoryManifests).isNotNull();
  assertThat(expectedManifestDirectoryManifests).hasLength(1);
  final Path expectedManifest = expectedManifestDirectoryManifests[0].toPath();

  Files.createDirectories(working.resolve("output"));
  final Path mergedManifest = working.resolve("output/mergedManifest.xml");

  List<String> args =
      generateArgs(
          mergerManifest,
          ImmutableMap.of(mergeeManifestOne, "mergeeOne", mergeeManifestTwo, "mergeeTwo"),
          false, /* isLibrary */
          ImmutableMap.of("applicationId", "com.google.android.apps.testapp"),
          "", /* custom_package */
          mergedManifest);
  ManifestMergerAction.main(args.toArray(new String[0]));

  assertThat(
      Joiner.on(" ")
          .join(Files.readAllLines(mergedManifest, UTF_8))
          .replaceAll("\\s+", " ")
          .trim())
      .isEqualTo(
          Joiner.on(" ")
              .join(Files.readAllLines(expectedManifest, UTF_8))
              .replaceAll("\\s+", " ")
              .trim());
}
 

@Override
@Before
public void setUp() throws Exception {
  super.setUp();

  // Clear some sys properties that may have been set per test
  for (Iterator<?> it = System.getProperties().keySet().iterator(); it.hasNext();) {
    if (it.next().toString().startsWith("model.")) {
      it.remove();
    }
  }

  File tempDir = getTestTempDir();
  File testDataDir = new File(getTestDataPath());
  Preconditions.checkState(testDataDir.exists() && testDataDir.isDirectory(),
                           "%s is not an existing directory", testDataDir.getAbsolutePath());

  log.info("Copying files to {}", tempDir);

  if (savedModelFile == null) {
    log.info("No saved model file, building model");
    File[] srcDataFiles = testDataDir.listFiles(new PatternFilenameFilter("[^.].*"));
    if (srcDataFiles != null) {
      for (File srcDataFile : srcDataFiles) {
        File destFile = new File(tempDir, srcDataFile.getName());
        Files.copy(srcDataFile, destFile);
      }
    }
  } else {
    log.info("Found saved model file {} (size {})", savedModelFile, savedModelFile.length());
    Files.copy(savedModelFile, new File(tempDir, "model.bin.gz"));
  }

  log.info("Configuring recommender...");

  RunnerConfiguration runnerConfig = new RunnerConfiguration();
  runnerConfig.setInstanceID("test");
  runnerConfig.setPort(8090); // Odd ports to avoid conflicts
  if (useSecurity()) {
    runnerConfig.setSecurePort(8453); // Odd ports to avoid conflicts
    runnerConfig.setKeystorePassword("changeit");
    runnerConfig.setKeystoreFile(new File("testdata/keystore"));
    runnerConfig.setUserName("foo");
    runnerConfig.setPassword("bar");
  }
  runnerConfig.setLocalInputDir(tempDir);

  runner = new Runner(runnerConfig);
  runner.call();

  boolean clientSecure = runnerConfig.getKeystoreFile() != null;
  int clientPort = clientSecure ? runnerConfig.getSecurePort() : runnerConfig.getPort();
  MyrrixClientConfiguration clientConfig = new MyrrixClientConfiguration();
  clientConfig.setHost("localhost");
  clientConfig.setPort(clientPort);
  clientConfig.setSecure(clientSecure);
  clientConfig.setKeystorePassword(runnerConfig.getKeystorePassword());
  clientConfig.setKeystoreFile(runnerConfig.getKeystoreFile());
  clientConfig.setUserName(runnerConfig.getUserName());
  clientConfig.setPassword(runnerConfig.getPassword());
  client = new ClientRecommender(clientConfig);

  if (callAwait()) {
    log.info("Waiting for client...");
    client.await();
    log.info("Client ready");
  }
}
 

@Test
public void find_files_by_extension_guava () {

	Pattern pattern = Pattern.compile("^.*.txt");
	
	FilenameFilter filterByExtension = new PatternFilenameFilter(pattern);
	
	File dir = new File(sourceFileURI);
	File[] files = dir.listFiles(filterByExtension);

	logger.info(Arrays.toString(files));
	
	assertTrue(files.length >= 4);
}