下面列出了org.apache.lucene.index.IndexWriter#addDocuments ( ) 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
@Override
public void setUp() throws Exception {
super.setUp();
directory = newDirectory();
final IndexWriterConfig config = new IndexWriterConfig(new MockAnalyzer(random()));
final IndexWriter indexWriter = new IndexWriter(directory, config);
for (int i = 0; i < AMOUNT_OF_SEGMENTS; i++) {
List<Document> segmentDocs = createDocsForSegment(i);
indexWriter.addDocuments(segmentDocs);
indexWriter.commit();
}
indexReader = DirectoryReader.open(indexWriter);
indexWriter.close();
indexSearcher = new IndexSearcher(indexReader);
parentsFilter = new QueryBitSetProducer(new WildcardQuery(new Term("parent", "*")));
}
public void testSharedCountCollectorManager() throws Exception {
Query q = new MatchAllDocsQuery();
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE));
Document doc = new Document();
w.addDocuments(Arrays.asList(doc, doc, doc, doc));
w.flush();
w.addDocuments(Arrays.asList(doc, doc));
w.flush();
IndexReader reader = DirectoryReader.open(w);
assertEquals(2, reader.leaves().size());
w.close();
TopDocsCollector<ScoreDoc> collector = doSearchWithThreshold( 5, 10, q, reader);
TopDocs tdc = doConcurrentSearchWithThreshold(5, 10, q, reader);
TopDocs tdc2 = collector.topDocs();
CheckHits.checkEqual(q, tdc.scoreDocs, tdc2.scoreDocs);
reader.close();
dir.close();
}
private void allowDuplicateUpdate(AddUpdateCommand cmd) throws IOException {
RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
try {
IndexWriter writer = iw.get();
Iterable<Document> nestedDocs = cmd.getLuceneDocsIfNested();
if (nestedDocs != null) {
writer.addDocuments(nestedDocs);
} else {
writer.addDocument(cmd.getLuceneDocument());
}
if (ulog != null) ulog.add(cmd);
} finally {
iw.decref();
}
}
private void addDocs(final List<ParseContext.Document> docs, final IndexWriter indexWriter) throws IOException {
if (docs.size() > 1) {
indexWriter.addDocuments(docs);
} else {
indexWriter.addDocument(docs.get(0));
}
numDocAppends.inc(docs.size());
}
public static Directory createIndex() throws CorruptIndexException, LockObtainFailedException, IOException {
Directory directory = new RAMDirectory();
IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION)));
writer.addDocuments(addPrime(Arrays.asList(newDocument(newStringField(ROW_ID, "1"), newStringField(PERSON_NAME, NAME1)),
newDocument(newStringField(ROW_ID, "1"), newStringField(PERSON_NAME, NAME1)), newDocument(newStringField(ROW_ID, "1"), newStringField(ADDRESS_STREET, STREET1)))));
writer.addDocuments(addPrime(Arrays.asList(newDocument(newStringField(ROW_ID, "2"), newStringField(PERSON_NAME, NAME2)),
newDocument(newStringField(ROW_ID, "2"), newStringField(ADDRESS_STREET, STREET1)))));
writer.addDocuments(addPrime(Arrays.asList(newDocument(newStringField(ROW_ID, "3"), newStringField(PERSON_NAME, NAME1)),
newDocument(newStringField(ROW_ID, "3"), newStringField(ADDRESS_STREET, STREET1)), newDocument(newStringField(ROW_ID, "3"), newStringField(ADDRESS_STREET, STREET2)))));
writer.close();
return directory;
}
private Directory createIndex(Random random, Collection<Query> sampler) throws CorruptIndexException, LockObtainFailedException, IOException {
Directory directory = new RAMDirectory();
String[] columnFamilies = genWords(random, MIN_NUM_COL_FAM, MAX_NUM_COL_FAM, "colfam");
Map<String, String[]> columns = new HashMap<String, String[]>();
for (int i = 0; i < columnFamilies.length; i++) {
columns.put(columnFamilies[i], genWords(random, MIN_NUM_COLS, MAX_NUM_COLS, "col"));
}
IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION)));
int numberOfDocs = random.nextInt(MAX_NUM_OF_DOCS) + 1;
for (int i = 0; i < numberOfDocs; i++) {
writer.addDocuments(generatSuperDoc(random, columns, sampler));
}
writer.close();
return directory;
}
private void addDocs(final List<ParseContext.Document> docs, final IndexWriter indexWriter) throws IOException {
if (docs.size() > 1) {
indexWriter.addDocuments(docs);
} else {
indexWriter.addDocument(docs.get(0));
}
numDocAppends.inc(docs.size());
}
public void testEmptyChildFilter() throws Exception {
final Directory dir = newDirectory();
final IndexWriterConfig config = new IndexWriterConfig(new MockAnalyzer(random()));
config.setMergePolicy(NoMergePolicy.INSTANCE);
// we don't want to merge - since we rely on certain segment setup
final IndexWriter w = new IndexWriter(dir, config);
final List<Document> docs = new ArrayList<>();
docs.add(makeJob("java", 2007));
docs.add(makeJob("python", 2010));
docs.add(makeResume("Lisa", "United Kingdom"));
w.addDocuments(docs);
docs.clear();
docs.add(makeJob("ruby", 2005));
docs.add(makeJob("java", 2006));
docs.add(makeResume("Frank", "United States"));
w.addDocuments(docs);
w.commit();
IndexReader r = DirectoryReader.open(w);
w.close();
IndexSearcher s = newSearcher(r);
BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "resume")));
CheckJoinIndex.check(r, parentsFilter);
BooleanQuery.Builder childQuery = new BooleanQuery.Builder();
childQuery.add(new BooleanClause(new TermQuery(new Term("skill", "java")), Occur.MUST));
childQuery.add(new BooleanClause(IntPoint.newRangeQuery("year", 2006, 2011), Occur.MUST));
ToParentBlockJoinQuery childJoinQuery = new ToParentBlockJoinQuery(childQuery.build(), parentsFilter, ScoreMode.Avg);
BooleanQuery.Builder fullQuery = new BooleanQuery.Builder();
fullQuery.add(new BooleanClause(childJoinQuery, Occur.MUST));
fullQuery.add(new BooleanClause(new MatchAllDocsQuery(), Occur.MUST));
TopDocs topDocs = s.search(fullQuery.build(), 2);
assertEquals(2, topDocs.totalHits.value);
assertEquals(asSet("Lisa", "Frank"),
asSet(s.doc(topDocs.scoreDocs[0].doc).get("name"), s.doc(topDocs.scoreDocs[1].doc).get("name")));
ParentChildrenBlockJoinQuery childrenQuery =
new ParentChildrenBlockJoinQuery(parentsFilter, childQuery.build(), topDocs.scoreDocs[0].doc);
TopDocs matchingChildren = s.search(childrenQuery, 1);
assertEquals(1, matchingChildren.totalHits.value);
assertEquals("java", s.doc(matchingChildren.scoreDocs[0].doc).get("skill"));
childrenQuery = new ParentChildrenBlockJoinQuery(parentsFilter, childQuery.build(), topDocs.scoreDocs[1].doc);
matchingChildren = s.search(childrenQuery, 1);
assertEquals(1, matchingChildren.totalHits.value);
assertEquals("java", s.doc(matchingChildren.scoreDocs[0].doc).get("skill"));
r.close();
dir.close();
}
public void testSetMinCompetitiveScore() throws Exception {
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE));
Document doc = new Document();
w.addDocuments(Arrays.asList(doc, doc, doc, doc));
w.flush();
w.addDocuments(Arrays.asList(doc, doc));
w.flush();
IndexReader reader = DirectoryReader.open(w);
assertEquals(2, reader.leaves().size());
w.close();
TopScoreDocCollector collector = TopScoreDocCollector.create(2, null, 2);
ScoreAndDoc scorer = new ScoreAndDoc();
LeafCollector leafCollector = collector.getLeafCollector(reader.leaves().get(0));
leafCollector.setScorer(scorer);
assertNull(scorer.minCompetitiveScore);
scorer.doc = 0;
scorer.score = 1;
leafCollector.collect(0);
assertNull(scorer.minCompetitiveScore);
scorer.doc = 1;
scorer.score = 2;
leafCollector.collect(1);
assertNull(scorer.minCompetitiveScore);
scorer.doc = 2;
scorer.score = 3;
leafCollector.collect(2);
assertEquals(Math.nextUp(2f), scorer.minCompetitiveScore, 0f);
scorer.doc = 3;
scorer.score = 0.5f;
// Make sure we do not call setMinCompetitiveScore for non-competitive hits
scorer.minCompetitiveScore = Float.NaN;
leafCollector.collect(3);
assertTrue(Float.isNaN(scorer.minCompetitiveScore));
scorer.doc = 4;
scorer.score = 4;
leafCollector.collect(4);
assertEquals(Math.nextUp(3f), scorer.minCompetitiveScore, 0f);
// Make sure the min score is set on scorers on new segments
scorer = new ScoreAndDoc();
leafCollector = collector.getLeafCollector(reader.leaves().get(1));
leafCollector.setScorer(scorer);
assertEquals(Math.nextUp(3f), scorer.minCompetitiveScore, 0f);
scorer.doc = 0;
scorer.score = 1;
leafCollector.collect(0);
assertEquals(Math.nextUp(3f), scorer.minCompetitiveScore, 0f);
scorer.doc = 1;
scorer.score = 4;
leafCollector.collect(1);
assertEquals(Math.nextUp(4f), scorer.minCompetitiveScore, 0f);
reader.close();
dir.close();
}
public void testTotalHits() throws Exception {
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE));
Document doc = new Document();
w.addDocuments(Arrays.asList(doc, doc, doc, doc));
w.flush();
w.addDocuments(Arrays.asList(doc, doc, doc, doc, doc, doc));
w.flush();
IndexReader reader = DirectoryReader.open(w);
assertEquals(2, reader.leaves().size());
w.close();
for (int totalHitsThreshold = 0; totalHitsThreshold < 20; ++ totalHitsThreshold) {
TopScoreDocCollector collector = TopScoreDocCollector.create(2, null, totalHitsThreshold);
ScoreAndDoc scorer = new ScoreAndDoc();
LeafCollector leafCollector = collector.getLeafCollector(reader.leaves().get(0));
leafCollector.setScorer(scorer);
scorer.doc = 0;
scorer.score = 3;
leafCollector.collect(0);
scorer.doc = 1;
scorer.score = 3;
leafCollector.collect(1);
leafCollector = collector.getLeafCollector(reader.leaves().get(1));
leafCollector.setScorer(scorer);
scorer.doc = 1;
scorer.score = 3;
leafCollector.collect(1);
scorer.doc = 5;
scorer.score = 4;
leafCollector.collect(1);
TopDocs topDocs = collector.topDocs();
assertEquals(4, topDocs.totalHits.value);
assertEquals(totalHitsThreshold < 4, scorer.minCompetitiveScore != null);
assertEquals(new TotalHits(4, totalHitsThreshold < 4 ? TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO : TotalHits.Relation.EQUAL_TO), topDocs.totalHits);
}
reader.close();
dir.close();
}
public void testSetMinCompetitiveScore() throws Exception {
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE));
Document doc = new Document();
w.addDocuments(Arrays.asList(doc, doc, doc, doc));
w.flush();
w.addDocuments(Arrays.asList(doc, doc));
w.flush();
IndexReader reader = DirectoryReader.open(w);
assertEquals(2, reader.leaves().size());
w.close();
Sort sort = new Sort(FIELD_SCORE, new SortField("foo", SortField.Type.LONG));
TopFieldCollector collector = TopFieldCollector.create(sort, 2, null, 2);
ScoreAndDoc scorer = new ScoreAndDoc();
LeafCollector leafCollector = collector.getLeafCollector(reader.leaves().get(0));
leafCollector.setScorer(scorer);
assertNull(scorer.minCompetitiveScore);
scorer.doc = 0;
scorer.score = 1;
leafCollector.collect(0);
assertNull(scorer.minCompetitiveScore);
scorer.doc = 1;
scorer.score = 2;
leafCollector.collect(1);
assertNull(scorer.minCompetitiveScore);
scorer.doc = 2;
scorer.score = 3;
leafCollector.collect(2);
assertEquals(2f, scorer.minCompetitiveScore, 0f);
scorer.doc = 3;
scorer.score = 0.5f;
// Make sure we do not call setMinCompetitiveScore for non-competitive hits
scorer.minCompetitiveScore = Float.NaN;
leafCollector.collect(3);
assertTrue(Float.isNaN(scorer.minCompetitiveScore));
scorer.doc = 4;
scorer.score = 4;
leafCollector.collect(4);
assertEquals(3f, scorer.minCompetitiveScore, 0f);
// Make sure the min score is set on scorers on new segments
scorer = new ScoreAndDoc();
leafCollector = collector.getLeafCollector(reader.leaves().get(1));
leafCollector.setScorer(scorer);
assertEquals(3f, scorer.minCompetitiveScore, 0f);
scorer.doc = 0;
scorer.score = 1;
leafCollector.collect(0);
assertEquals(3f, scorer.minCompetitiveScore, 0f);
scorer.doc = 1;
scorer.score = 4;
leafCollector.collect(1);
assertEquals(4f, scorer.minCompetitiveScore, 0f);
reader.close();
dir.close();
}
public void testTotalHitsWithScore() throws Exception {
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE));
Document doc = new Document();
w.addDocuments(Arrays.asList(doc, doc, doc, doc));
w.flush();
w.addDocuments(Arrays.asList(doc, doc, doc, doc, doc, doc));
w.flush();
IndexReader reader = DirectoryReader.open(w);
assertEquals(2, reader.leaves().size());
w.close();
for (int totalHitsThreshold = 0; totalHitsThreshold < 20; ++ totalHitsThreshold) {
Sort sort = new Sort(FIELD_SCORE, new SortField("foo", SortField.Type.LONG));
TopFieldCollector collector = TopFieldCollector.create(sort, 2, null, totalHitsThreshold);
ScoreAndDoc scorer = new ScoreAndDoc();
LeafCollector leafCollector = collector.getLeafCollector(reader.leaves().get(0));
leafCollector.setScorer(scorer);
scorer.doc = 0;
scorer.score = 3;
leafCollector.collect(0);
scorer.doc = 1;
scorer.score = 3;
leafCollector.collect(1);
leafCollector = collector.getLeafCollector(reader.leaves().get(1));
leafCollector.setScorer(scorer);
scorer.doc = 1;
scorer.score = 3;
leafCollector.collect(1);
scorer.doc = 5;
scorer.score = 4;
leafCollector.collect(1);
TopDocs topDocs = collector.topDocs();
assertEquals(totalHitsThreshold < 4, scorer.minCompetitiveScore != null);
assertEquals(new TotalHits(4, totalHitsThreshold < 4 ? TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO : TotalHits.Relation.EQUAL_TO), topDocs.totalHits);
}
reader.close();
dir.close();
}