org.apache.lucene.index.RandomIndexWriter#commit ( )源码实例Demo

@Before
public void setupIndex() throws IOException {
	dirUnderTest = newDirectory();

	indexWriterUnderTest = new RandomIndexWriter(random(), dirUnderTest);
	String[] docs = new String[] { "how now brown cow", "woc",
			"nworb", "won woh nworb" };
	for (int i = 0; i < docs.length; i++) {
		Document doc = new Document();
		doc.add(newStringField("id", "" + i, Field.Store.YES));
		doc.add(newField("field", docs[i], Field.Store.NO));
		indexWriterUnderTest.addDocument(doc);
	}
	indexWriterUnderTest.commit();

	indexReaderUnderTest = indexWriterUnderTest.getReader();
	searcherUnderTest = newSearcher(indexReaderUnderTest);
}
 

@Test
public void testEmptyReader() throws IOException {
  Directory dir = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
  iwc.setMergePolicy(newLogMergePolicy());
  // Make sure the index is created?
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
  writer.commit();
  writer.close();
  IndexReader ir = DirectoryReader.open(dir);
  Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME);
  InputIterator inputIterator = dictionary.getEntryIterator();

  assertNull(inputIterator.next());
  assertEquals(inputIterator.weight(), 0);
  assertNull(inputIterator.payload());
  
  IOUtils.close(ir, analyzer, dir);
}
 

public void testEvilSearcherFactory() throws Exception {
  final Directory dir = newDirectory();
  final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
  w.commit();

  final IndexReader other = DirectoryReader.open(dir);

  final SearcherFactory theEvilOne = new SearcherFactory() {
    @Override
    public IndexSearcher newSearcher(IndexReader ignored, IndexReader previous) {
      return LuceneTestCase.newSearcher(other);
    }
    };

  expectThrows(IllegalStateException.class, () -> {
    new SearcherManager(w.w, false, false, theEvilOne);
  });

  w.close();
  other.close();
  dir.close();
}
 

@Test
public void testEmptyPrefixQuery() throws Exception {
  Analyzer analyzer = new MockAnalyzer(random());
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
  Document document = new Document();
  document.add(new SuggestField("suggest_field", "suggestion1", 1));
  iw.addDocument(document);

  if (rarely()) {
    iw.commit();
  }

  DirectoryReader reader = iw.getReader();
  SuggestIndexSearcher suggestIndexSearcher = new SuggestIndexSearcher(reader);
  PrefixCompletionQuery query = new PrefixCompletionQuery(analyzer, new Term("suggest_field", ""));

  TopSuggestDocs suggest = suggestIndexSearcher.suggest(query, 5, false);
  assertEquals(0, suggest.scoreDocs.length);

  reader.close();
  iw.close();
}
 

public void testApproximation() throws IOException {
  final int iters = atLeast(10);
  for (int iter = 0; iter < iters; ++iter) {
    Directory dir = newDirectory();
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
    final int numDocs = atLeast(100);
    for (int i = 0; i < numDocs; ++i) {
      Document doc = new Document();
      final boolean hasValue = random().nextBoolean();
      if (hasValue) {
        doc.add(new TextField("text1", "value", Store.NO));
        doc.add(new StringField("has_value", "yes", Store.NO));
      }
      doc.add(new StringField("f", random().nextBoolean() ? "yes" : "no", Store.NO));
      iw.addDocument(doc);
    }
    if (random().nextBoolean()) {
      iw.deleteDocuments(new TermQuery(new Term("f", "no")));
    }
    iw.commit();
    final IndexReader reader = iw.getReader();
    final IndexSearcher searcher = newSearcher(reader);
    iw.close();

    BooleanQuery.Builder ref = new BooleanQuery.Builder();
    ref.add(new TermQuery(new Term("f", "yes")), Occur.MUST);
    ref.add(new TermQuery(new Term("has_value", "yes")), Occur.FILTER);

    BooleanQuery.Builder bq1 = new BooleanQuery.Builder();
    bq1.add(new TermQuery(new Term("f", "yes")), Occur.MUST);
    bq1.add(new NormsFieldExistsQuery("text1"), Occur.FILTER);
    assertSameMatches(searcher, ref.build(), bq1.build(), true);

    reader.close();
    dir.close();
  }
}
 

public void testAllDocsHaveField() throws IOException {
  Directory dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(new TextField("f", "value", Store.NO));
  iw.addDocument(doc);
  iw.commit();
  final IndexReader reader = iw.getReader();
  final IndexSearcher searcher = newSearcher(reader);
  iw.close();
  assertEquals(1, searcher.count(new NormsFieldExistsQuery("f")));
  reader.close();
  dir.close();
}
 

@Override
public void setUp() throws Exception {
  super.setUp();
  dir = newDirectory();
  writer = new RandomIndexWriter(random(), dir);
  final int numDocs = atLeast(100);
  for (int i = 0; i < numDocs; i++) {
    writer.addDocument(new Document());
    if (rarely()) {
      writer.commit();
    }
  }
}
 

public void testRandom() throws IOException {
  final int iters = atLeast(10);
  for (int iter = 0; iter < iters; ++iter) {
    Directory dir = newDirectory();
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
    final int numDocs = atLeast(100);
    for (int i = 0; i < numDocs; ++i) {
      Document doc = new Document();
      final boolean hasValue = random().nextBoolean();
      if (hasValue) {
        doc.add(new NumericDocValuesField("dv1", 1));
        doc.add(new SortedNumericDocValuesField("dv2", 1));
        doc.add(new SortedNumericDocValuesField("dv2", 2));
        doc.add(new StringField("has_value", "yes", Store.NO));
      }
      doc.add(new StringField("f", random().nextBoolean() ? "yes" : "no", Store.NO));
      iw.addDocument(doc);
    }
    if (random().nextBoolean()) {
      iw.deleteDocuments(new TermQuery(new Term("f", "no")));
    }
    iw.commit();
    final IndexReader reader = iw.getReader();
    final IndexSearcher searcher = newSearcher(reader);
    iw.close();

    assertSameMatches(searcher, new TermQuery(new Term("has_value", "yes")), new DocValuesFieldExistsQuery("dv1"), false);
    assertSameMatches(searcher, new TermQuery(new Term("has_value", "yes")), new DocValuesFieldExistsQuery("dv2"), false);

    reader.close();
    dir.close();
  }
}
 

@Test @Slow
public void testDupSuggestFieldValues() throws Exception {
  Analyzer analyzer = new MockAnalyzer(random());
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
  final int num = Math.min(1000, atLeast(100));
  int[] weights = new int[num];
  for(int i = 0; i < num; i++) {
    Document document = new Document();
    weights[i] = random().nextInt(Integer.MAX_VALUE);
    document.add(new SuggestField("suggest_field", "abc", weights[i]));
    iw.addDocument(document);

    if (usually()) {
      iw.commit();
    }
  }

  DirectoryReader reader = iw.getReader();
  Entry[] expectedEntries = new Entry[num];
  Arrays.sort(weights);
  for (int i = 1; i <= num; i++) {
    expectedEntries[i - 1] = new Entry("abc", weights[num - i]);
  }

  SuggestIndexSearcher suggestIndexSearcher = new SuggestIndexSearcher(reader);
  PrefixCompletionQuery query = new PrefixCompletionQuery(analyzer, new Term("suggest_field", "abc"));
  TopSuggestDocs lookupDocs = suggestIndexSearcher.suggest(query, num, false);
  assertSuggestions(lookupDocs, expectedEntries);

  reader.close();
  iw.close();
}
 

public void testMissingField() throws IOException {
  Directory dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  iw.addDocument(new Document());
  iw.commit();
  final IndexReader reader = iw.getReader();
  final IndexSearcher searcher = newSearcher(reader);
  iw.close();
  assertEquals(0, searcher.count(new DocValuesFieldExistsQuery("f")));
  reader.close();
  dir.close();
}
 

public void testFeatureMissingFeatureNameInSegment() throws IOException {
  Directory dir = newDirectory();
  IndexWriterConfig config = newIndexWriterConfig().setMergePolicy(newLogMergePolicy(random().nextBoolean()));
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, config);
  Document doc = new Document();
  doc.add(new FeatureField("field", "different_name", 0.5F));
  writer.addDocument(doc);
  writer.commit();
  doc = new Document();
  doc.add(new FeatureField("field", "name", 1.3F));
  doc.add(newStringField("value", "1.3", Field.Store.YES));
  writer.addDocument(doc);
  doc = new Document();
  doc.add(new FeatureField("field", "name", 4.2F));
  doc.add(newStringField("value", "4.2", Field.Store.YES));
  writer.addDocument(doc);
  IndexReader ir = writer.getReader();
  writer.close();

  IndexSearcher searcher = newSearcher(ir);
  Sort sort = new Sort(FeatureField.newFeatureSort("field", "name"));

  TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
  assertEquals(3, td.totalHits.value);
  // null is treated as 0
  assertEquals("4.2", searcher.doc(td.scoreDocs[0].doc).get("value"));
  assertEquals("1.3", searcher.doc(td.scoreDocs[1].doc).get("value"));
  assertNull(searcher.doc(td.scoreDocs[2].doc).get("value"));

  ir.close();
  dir.close();
}
 

public void testScore() throws IOException {
  final int iters = atLeast(10);
  for (int iter = 0; iter < iters; ++iter) {
    Directory dir = newDirectory();
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
    final int numDocs = atLeast(100);
    for (int i = 0; i < numDocs; ++i) {
      Document doc = new Document();
      final boolean hasValue = random().nextBoolean();
      if (hasValue) {
        doc.add(new TextField("text1", "value", Store.NO));
        doc.add(new StringField("has_value", "yes", Store.NO));
      }
      doc.add(new StringField("f", random().nextBoolean() ? "yes" : "no", Store.NO));
      iw.addDocument(doc);
    }
    if (random().nextBoolean()) {
      iw.deleteDocuments(new TermQuery(new Term("f", "no")));
    }
    iw.commit();
    final IndexReader reader = iw.getReader();
    final IndexSearcher searcher = newSearcher(reader);
    iw.close();

    final float boost = random().nextFloat() * 10;
    final Query ref = new BoostQuery(new ConstantScoreQuery(new TermQuery(new Term("has_value", "yes"))), boost);

    final Query q1 = new BoostQuery(new NormsFieldExistsQuery("text1"), boost);
    assertSameMatches(searcher, ref, q1, true);

    reader.close();
    dir.close();
  }
}
 

@Test
public void testReturnedDocID() throws Exception {
  Analyzer analyzer = new MockAnalyzer(random());
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));

  int num = Math.min(1000, atLeast(10));
  for (int i = 0; i < num; i++) {
    Document document = new Document();
    document.add(new SuggestField("suggest_field", "abc_" + i, num));
    document.add(new StoredField("int_field", i));
    iw.addDocument(document);

    if (random().nextBoolean()) {
      iw.commit();
    }
  }

  DirectoryReader reader = iw.getReader();
  SuggestIndexSearcher indexSearcher = new SuggestIndexSearcher(reader);
  PrefixCompletionQuery query = new PrefixCompletionQuery(analyzer, new Term("suggest_field", "abc_"));
  TopSuggestDocs suggest = indexSearcher.suggest(query, num, false);
  assertEquals(num, suggest.totalHits.value);
  for (SuggestScoreDoc suggestScoreDoc : suggest.scoreLookupDocs()) {
    String key = suggestScoreDoc.key.toString();
    assertTrue(key.startsWith("abc_"));
    String substring = key.substring(4);
    int fieldValue = Integer.parseInt(substring);
    Document doc = reader.document(suggestScoreDoc.doc);
    assertEquals(doc.getField("int_field").numericValue().intValue(), fieldValue);
  }

  reader.close();
  iw.close();
}
 

public void testCacheScoresIfNecessary() throws IOException {
  Directory dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  iw.addDocument(new Document());
  iw.commit();
  DirectoryReader reader = iw.getReader();
  iw.close();
  
  final LeafReaderContext ctx = reader.leaves().get(0);

  expectThrows(AssertionError.class, () -> {
    collector(ScoreMode.COMPLETE_NO_SCORES, ScoreCachingWrappingScorer.class).getLeafCollector(ctx).setScorer(new ScoreAndDoc());
  });

  // no collector needs scores => no caching
  Collector c1 = collector(ScoreMode.COMPLETE_NO_SCORES, ScoreAndDoc.class);
  Collector c2 = collector(ScoreMode.COMPLETE_NO_SCORES, ScoreAndDoc.class);
  MultiCollector.wrap(c1, c2).getLeafCollector(ctx).setScorer(new ScoreAndDoc());

  // only one collector needs scores => no caching
  c1 = collector(ScoreMode.COMPLETE, ScoreAndDoc.class);
  c2 = collector(ScoreMode.COMPLETE_NO_SCORES, ScoreAndDoc.class);
  MultiCollector.wrap(c1, c2).getLeafCollector(ctx).setScorer(new ScoreAndDoc());

  // several collectors need scores => caching
  c1 = collector(ScoreMode.COMPLETE, ScoreCachingWrappingScorer.class);
  c2 = collector(ScoreMode.COMPLETE, ScoreCachingWrappingScorer.class);
  MultiCollector.wrap(c1, c2).getLeafCollector(ctx).setScorer(new ScoreAndDoc());

  reader.close();
  dir.close();
}
 

@Test
public void testContextPrecedenceBoost() throws Exception {
  Analyzer analyzer = new MockAnalyzer(random());
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
  Document document = new Document();

  document.add(new ContextSuggestField("suggest_field", "suggestion1", 4, "typetype"));
  document.add(new ContextSuggestField("suggest_field", "suggestion2", 3, "type"));
  iw.addDocument(document);

  if (rarely()) {
    iw.commit();
  }

  DirectoryReader reader = iw.getReader();
  SuggestIndexSearcher suggestIndexSearcher = new SuggestIndexSearcher(reader);
  ContextQuery query = new ContextQuery(new PrefixCompletionQuery(analyzer, new Term("suggest_field", "sugg")));
  query.addContext("type", 1);
  query.addContext("typetype", 2);
  TopSuggestDocs suggest = suggestIndexSearcher.suggest(query, 5, false);
  assertSuggestions(suggest,
      new Entry("suggestion1", "typetype", 4 * 2),
      new Entry("suggestion2", "type", 3 * 1)
  );

  reader.close();
  iw.close();
}
 

private void createIndex() throws IOException {
  indexDir = createTempDir("testIndex");

  Directory dir = newFSDirectory(indexDir);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, new StandardAnalyzer());

  for (int i = 0; i < 10; i++) {
    Document doc1 = new Document();
    doc1.add(newTextField("f1", "Apple Pie", Field.Store.YES));
    doc1.add(new SortedDocValuesField("f2", new BytesRef("a" + (i * 10 + 1))));
    doc1.add(new SortedSetDocValuesField("f3", new BytesRef("a" + (i * 10 + 1))));
    doc1.add(new NumericDocValuesField("f4", i * 10 + 1L));
    doc1.add(new FloatDocValuesField("f5", i * 10 + 1.0f));
    doc1.add(new DoubleDocValuesField("f6", i * 10 + 1.0));
    doc1.add(new SortedNumericDocValuesField("f7", i * 10 + 1L));
    doc1.add(new IntPoint("f8", i * 10 + 1));
    doc1.add(new LongPoint("f9", i * 10 + 1L));
    doc1.add(new FloatPoint("f10", i * 10 + 1.0f));
    doc1.add(new DoublePoint("f11", i * 10 + 1.0));
    writer.addDocument(doc1);

    Document doc2 = new Document();
    doc2.add(newTextField("f1", "Brownie", Field.Store.YES));
    doc2.add(new SortedDocValuesField("f2", new BytesRef("b" + (i * 10 + 2))));
    doc2.add(new SortedSetDocValuesField("f3", new BytesRef("b" + (i * 10 + 2))));
    doc2.add(new NumericDocValuesField("f4", i * 10 + 2L));
    doc2.add(new FloatDocValuesField("f5", i * 10 + 2.0f));
    doc2.add(new DoubleDocValuesField("f6", i * 10 + 2.0));
    doc2.add(new SortedNumericDocValuesField("f7", i * 10 + 2L));
    doc2.add(new IntPoint("f8", i * 10 + 2));
    doc2.add(new LongPoint("f9", i * 10 + 2L));
    doc2.add(new FloatPoint("f10", i * 10 + 2.0f));
    doc2.add(new DoublePoint("f11", i * 10 + 2.0));
    writer.addDocument(doc2);

    Document doc3 = new Document();
    doc3.add(newTextField("f1", "Chocolate Pie", Field.Store.YES));
    doc3.add(new SortedDocValuesField("f2", new BytesRef("c" + (i * 10 + 3))));
    doc3.add(new SortedSetDocValuesField("f3", new BytesRef("c" + (i * 10 + 3))));
    doc3.add(new NumericDocValuesField("f4", i * 10 + 3L));
    doc3.add(new FloatDocValuesField("f5", i * 10 + 3.0f));
    doc3.add(new DoubleDocValuesField("f6", i * 10 + 3.0));
    doc3.add(new SortedNumericDocValuesField("f7", i * 10 + 3L));
    doc3.add(new IntPoint("f8", i * 10 + 3));
    doc3.add(new LongPoint("f9", i * 10 + 3L));
    doc3.add(new FloatPoint("f10", i * 10 + 3.0f));
    doc3.add(new DoublePoint("f11", i * 10 + 3.0));
    writer.addDocument(doc3);

    Document doc4 = new Document();
    doc4.add(newTextField("f1", "Doughnut", Field.Store.YES));
    doc4.add(new SortedDocValuesField("f2", new BytesRef("d" + (i * 10 + 4))));
    doc4.add(new SortedSetDocValuesField("f3", new BytesRef("d" + (i * 10 + 4))));
    doc4.add(new NumericDocValuesField("f4", i * 10 + 4L));
    doc4.add(new FloatDocValuesField("f5", i * 10 + 4.0f));
    doc4.add(new DoubleDocValuesField("f6", i * 10 + 4.0));
    doc4.add(new SortedNumericDocValuesField("f7", i * 10 + 4L));
    doc4.add(new IntPoint("f8", i * 10 + 4));
    doc4.add(new LongPoint("f9", i * 10 + 4L));
    doc4.add(new FloatPoint("f10", i * 10 + 4.0f));
    doc4.add(new DoublePoint("f11", i * 10 + 4.0));
    writer.addDocument(doc4);

    Document doc5 = new Document();
    doc5.add(newTextField("f1", "Eclair", Field.Store.YES));
    doc5.add(new SortedDocValuesField("f2", new BytesRef("e" + (i * 10 + 5))));
    doc5.add(new SortedSetDocValuesField("f3", new BytesRef("e" + (i * 10 + 5))));
    doc5.add(new NumericDocValuesField("f4", i * 10 + 5L));
    doc5.add(new FloatDocValuesField("f5", i * 10 + 5.0f));
    doc5.add(new DoubleDocValuesField("f6", i * 10 + 5.0));
    doc5.add(new SortedNumericDocValuesField("f7", i * 10 + 5L));
    doc5.add(new IntPoint("f8", i * 10 + 5));
    doc5.add(new LongPoint("f9", i * 10 + 5L));
    doc5.add(new FloatPoint("f10", i * 10 + 5.0f));
    doc5.add(new DoublePoint("f11", i * 10 + 5.0));
    writer.addDocument(doc5);
  }
  writer.commit();
  writer.close();
  dir.close();
}
 

private void createRandomIndex(boolean singleSortedSegment) throws IOException {
  dir = newDirectory();
  numDocs = atLeast(150);
  final int numTerms = TestUtil.nextInt(random(), 1, numDocs / 5);
  Set<String> randomTerms = new HashSet<>();
  while (randomTerms.size() < numTerms) {
    randomTerms.add(TestUtil.randomSimpleString(random()));
  }
  terms = new ArrayList<>(randomTerms);
  final long seed = random().nextLong();
  final IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(new Random(seed)));
  if (iwc.getMergePolicy() instanceof MockRandomMergePolicy) {
    // MockRandomMP randomly wraps the leaf readers which makes merging angry
    iwc.setMergePolicy(newTieredMergePolicy());
  }
  iwc.setMergeScheduler(new SerialMergeScheduler()); // for reproducible tests
  iwc.setIndexSort(sort);
  iw = new RandomIndexWriter(new Random(seed), dir, iwc);
  iw.setDoRandomForceMerge(false); // don't do this, it may happen anyway with MockRandomMP
  for (int i = 0; i < numDocs; ++i) {
    final Document doc = randomDocument();
    iw.addDocument(doc);
    if (i == numDocs / 2 || (i != numDocs - 1 && random().nextInt(8) == 0)) {
      iw.commit();
    }
    if (random().nextInt(15) == 0) {
      final String term = RandomPicks.randomFrom(random(), terms);
      iw.deleteDocuments(new Term("s", term));
    }
  }
  if (singleSortedSegment) {
    iw.forceMerge(1);
  }
  else if (random().nextBoolean()) {
    iw.forceMerge(FORCE_MERGE_MAX_SEGMENT_COUNT);
  }
  reader = iw.getReader();
  if (reader.numDocs() == 0) {
    iw.addDocument(new Document());
    reader.close();
    reader = iw.getReader();
  }
}
 

protected void createIndex() throws IOException {
  indexDir = createTempDir();

  Directory dir = newFSDirectory(indexDir);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, new StandardAnalyzer());

  FieldType titleType = new FieldType();
  titleType.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
  titleType.setStored(true);
  titleType.setTokenized(true);
  titleType.setOmitNorms(true);

  FieldType authorType = new FieldType();
  authorType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
  authorType.setStored(true);
  authorType.setTokenized(true);
  authorType.setOmitNorms(false);

  FieldType textType = new FieldType();
  textType.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
  textType.setStored(false);
  textType.setTokenized(true);
  textType.setStoreTermVectors(true);
  textType.setOmitNorms(false);

  FieldType downloadsType = new FieldType();
  downloadsType.setDimensions(1, Integer.BYTES);
  downloadsType.setStored(true);

  Document doc1 = new Document();
  doc1.add(new Field("title", "Pride and Prejudice", titleType));
  doc1.add(new Field("author", "Jane Austen", authorType));
  doc1.add(new Field("text",
      "It is a truth universally acknowledged, that a single man in possession of a good fortune, must be in want of a wife.",
      textType));
  doc1.add(new SortedSetDocValuesField("subject", new BytesRef("Fiction")));
  doc1.add(new SortedSetDocValuesField("subject", new BytesRef("Love stories")));
  doc1.add(new Field("downloads", packInt(28533), downloadsType));
  writer.addDocument(doc1);

  Document doc2 = new Document();
  doc2.add(new Field("title", "Alice's Adventures in Wonderland", titleType));
  doc2.add(new Field("author", "Lewis Carroll", authorType));
  doc2.add(new Field("text", "Alice was beginning to get very tired of sitting by her sister on the bank, and of having nothing to do: once or twice she had peeped into the book her sister was reading, but it had no pictures or conversations in it, ‘and what is the use of a book,’ thought Alice ‘without pictures or conversations?’",
      textType));
  doc2.add(new SortedSetDocValuesField("subject", new BytesRef("Fantasy literature")));
  doc2.add(new Field("downloads", packInt(18712), downloadsType));
  writer.addDocument(doc2);

  Document doc3 = new Document();
  doc3.add(new Field("title", "Frankenstein; Or, The Modern Prometheus", titleType));
  doc3.add(new Field("author", "Mary Wollstonecraft Shelley", authorType));
  doc3.add(new Field("text", "You will rejoice to hear that no disaster has accompanied the commencement of an enterprise which you have regarded with such evil forebodings. I arrived here yesterday, and my first task is to assure my dear sister of my welfare and increasing confidence in the success of my undertaking.",
      textType));
  doc3.add(new SortedSetDocValuesField("subject", new BytesRef("Science fiction")));
  doc3.add(new SortedSetDocValuesField("subject", new BytesRef("Horror tales")));
  doc3.add(new SortedSetDocValuesField("subject", new BytesRef("Monsters")));
  doc3.add(new Field("downloads", packInt(14737), downloadsType));
  writer.addDocument(doc3);

  Document doc4 = new Document();
  doc4.add(new Field("title", "A Doll's House : a play", titleType));
  doc4.add(new Field("author", "Henrik Ibsen", authorType));
  doc4.add(new Field("text", "",
      textType));
  doc4.add(new SortedSetDocValuesField("subject", new BytesRef("Drama")));
  doc4.add(new Field("downloads", packInt(14629), downloadsType));
  writer.addDocument(doc4);

  Document doc5 = new Document();
  doc5.add(new Field("title", "The Adventures of Sherlock Holmes", titleType));
  doc5.add(new Field("author", "Arthur Conan Doyle", authorType));
  doc5.add(new Field("text", "To Sherlock Holmes she is always the woman. I have seldom heard him mention her under any other name. In his eyes she eclipses and predominates the whole of her sex.",
      textType));
  doc5.add(new SortedSetDocValuesField("subject", new BytesRef("Fiction")));
  doc5.add(new SortedSetDocValuesField("subject", new BytesRef("Detective and mystery stories")));
  doc5.add(new Field("downloads", packInt(12828), downloadsType));
  writer.addDocument(doc5);

  writer.commit();

  writer.close();
  dir.close();
}
 

public void testFilterClauseDoesNotImpactScore() throws IOException {
  Directory dir = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  Field f = newTextField("field", "a b c d", Field.Store.NO);
  doc.add(f);
  w.addDocument(doc);
  f.setStringValue("b d");
  w.addDocument(doc);
  f.setStringValue("a d");
  w.addDocument(doc);
  w.commit();

  DirectoryReader reader = w.getReader();
  final IndexSearcher searcher = new IndexSearcher(reader);

  BooleanQuery.Builder qBuilder = new BooleanQuery.Builder();
  BooleanQuery q = qBuilder.build();
  qBuilder.add(new TermQuery(new Term("field", "a")), Occur.FILTER);

  // With a single clause, we will rewrite to the underlying
  // query. Make sure that it returns null scores
  assertSameScoresWithoutFilters(searcher, qBuilder.build());

  // Now with two clauses, we will get a conjunction scorer
  // Make sure it returns null scores
  qBuilder.add(new TermQuery(new Term("field", "b")), Occur.FILTER);
  q = qBuilder.build();
  assertSameScoresWithoutFilters(searcher, q);

  // Now with a scoring clause, we need to make sure that
  // the boolean scores are the same as those from the term
  // query
  qBuilder.add(new TermQuery(new Term("field", "c")), Occur.SHOULD);
  q = qBuilder.build();
  assertSameScoresWithoutFilters(searcher, q);

  // FILTER and empty SHOULD
  qBuilder = new BooleanQuery.Builder();
  qBuilder.add(new TermQuery(new Term("field", "a")), Occur.FILTER);
  qBuilder.add(new TermQuery(new Term("field", "e")), Occur.SHOULD);
  q = qBuilder.build();
  assertSameScoresWithoutFilters(searcher, q);

  // mix of FILTER and MUST
  qBuilder = new BooleanQuery.Builder();
  qBuilder.add(new TermQuery(new Term("field", "a")), Occur.FILTER);
  qBuilder.add(new TermQuery(new Term("field", "d")), Occur.MUST);
  q = qBuilder.build();
  assertSameScoresWithoutFilters(searcher, q);

  // FILTER + minShouldMatch
  qBuilder = new BooleanQuery.Builder();
  qBuilder.add(new TermQuery(new Term("field", "b")), Occur.FILTER);
  qBuilder.add(new TermQuery(new Term("field", "a")), Occur.SHOULD);
  qBuilder.add(new TermQuery(new Term("field", "d")), Occur.SHOULD);
  qBuilder.setMinimumNumberShouldMatch(1);
  q = qBuilder.build();
  assertSameScoresWithoutFilters(searcher, q);

  reader.close();
  w.close();
  dir.close();
}
 

private void doTestSortedVsFieldCache(int minLength, int maxLength) throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
  Document doc = new Document();
  Field idField = new StringField("id", "", Field.Store.NO);
  Field indexedField = new StringField("indexed", "", Field.Store.NO);
  Field dvField = new SortedDocValuesField("dv", new BytesRef());
  doc.add(idField);
  doc.add(indexedField);
  doc.add(dvField);
  
  // index some docs
  int numDocs = atLeast(300);
  for (int i = 0; i < numDocs; i++) {
    idField.setStringValue(Integer.toString(i));
    final int length;
    if (minLength == maxLength) {
      length = minLength; // fixed length
    } else {
      length = TestUtil.nextInt(random(), minLength, maxLength);
    }
    String value = TestUtil.randomSimpleString(random(), length);
    indexedField.setStringValue(value);
    dvField.setBytesValue(new BytesRef(value));
    writer.addDocument(doc);
    if (random().nextInt(31) == 0) {
      writer.commit();
    }
  }
  
  // delete some docs
  int numDeletions = random().nextInt(numDocs/10);
  for (int i = 0; i < numDeletions; i++) {
    int id = random().nextInt(numDocs);
    writer.deleteDocuments(new Term("id", Integer.toString(id)));
  }
  writer.close();
  
  // compare
  DirectoryReader ir = DirectoryReader.open(dir);
  for (LeafReaderContext context : ir.leaves()) {
    LeafReader r = context.reader();
    SortedDocValues expected = FieldCache.DEFAULT.getTermsIndex(r, "indexed");
    SortedDocValues actual = r.getSortedDocValues("dv");
    assertEquals(r.maxDoc(), expected, actual);
  }
  ir.close();
  dir.close();
}