下面列出了org.apache.lucene.index.LogMergePolicy#setMergeFactor ( ) 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
private static IndexWriter initWriter(int id, Random random, Path indexPath, boolean doCheckIndexOnClose) throws IOException {
Directory dir = SimpleReplicaNode.getDirectory(random, id, indexPath, doCheckIndexOnClose);
MockAnalyzer analyzer = new MockAnalyzer(random);
analyzer.setMaxTokenLength(TestUtil.nextInt(random, 1, IndexWriter.MAX_TERM_LENGTH));
IndexWriterConfig iwc = LuceneTestCase.newIndexWriterConfig(random, analyzer);
MergePolicy mp = iwc.getMergePolicy();
//iwc.setInfoStream(new PrintStreamInfoStream(System.out));
// Force more frequent merging so we stress merge warming:
if (mp instanceof TieredMergePolicy) {
TieredMergePolicy tmp = (TieredMergePolicy) mp;
tmp.setSegmentsPerTier(3);
tmp.setMaxMergeAtOnce(3);
} else if (mp instanceof LogMergePolicy) {
LogMergePolicy lmp = (LogMergePolicy) mp;
lmp.setMergeFactor(3);
}
IndexWriter writer = new IndexWriter(dir, iwc);
TestUtil.reduceOpenFiles(writer);
return writer;
}
/** just tries to configure things to keep the open file
* count lowish */
public static void reduceOpenFiles(IndexWriter w) {
// keep number of open files lowish
MergePolicy mp = w.getConfig().getMergePolicy();
mp.setNoCFSRatio(1.0);
if (mp instanceof LogMergePolicy) {
LogMergePolicy lmp = (LogMergePolicy) mp;
lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
} else if (mp instanceof TieredMergePolicy) {
TieredMergePolicy tmp = (TieredMergePolicy) mp;
tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
}
MergeScheduler ms = w.getConfig().getMergeScheduler();
if (ms instanceof ConcurrentMergeScheduler) {
// wtf... shouldnt it be even lower since it's 1 by default?!?!
((ConcurrentMergeScheduler) ms).setMaxMergesAndThreads(3, 2);
}
}
public static IndexWriter getIndexWriter(String indexPath, boolean create) throws IOException {
Directory dir = FSDirectory.open(Paths.get(indexPath));
Analyzer analyzer = new SmartChineseAnalyzer();
IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
LogMergePolicy mergePolicy = new LogByteSizeMergePolicy();
mergePolicy.setMergeFactor(50);
mergePolicy.setMaxMergeDocs(5000);
if (create){
iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
} else {
iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
}
return new IndexWriter(dir, iwc);
}
@Test
public void testOpenIfChangedManySegments() throws Exception {
// test openIfChanged() when the taxonomy contains many segments
Directory dir = newDirectory();
DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir) {
@Override
protected IndexWriterConfig createIndexWriterConfig(OpenMode openMode) {
IndexWriterConfig conf = super.createIndexWriterConfig(openMode);
LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
lmp.setMergeFactor(2);
return conf;
}
};
TaxonomyReader reader = new DirectoryTaxonomyReader(writer);
int numRounds = random().nextInt(10) + 10;
int numCategories = 1; // one for root
for (int i = 0; i < numRounds; i++) {
int numCats = random().nextInt(4) + 1;
for (int j = 0; j < numCats; j++) {
writer.addCategory(new FacetLabel(Integer.toString(i), Integer.toString(j)));
}
numCategories += numCats + 1 /* one for round-parent */;
TaxonomyReader newtr = TaxonomyReader.openIfChanged(reader);
assertNotNull(newtr);
reader.close();
reader = newtr;
// assert categories
assertEquals(numCategories, reader.getSize());
int roundOrdinal = reader.getOrdinal(new FacetLabel(Integer.toString(i)));
int[] parents = reader.getParallelTaxonomyArrays().parents();
assertEquals(0, parents[roundOrdinal]); // round's parent is root
for (int j = 0; j < numCats; j++) {
int ord = reader.getOrdinal(new FacetLabel(Integer.toString(i), Integer.toString(j)));
assertEquals(roundOrdinal, parents[ord]); // round's parent is root
}
}
reader.close();
writer.close();
dir.close();
}
public void testSubclassConcurrentMergeScheduler() throws IOException {
MockDirectoryWrapper dir = newMockDirectory();
dir.failOn(new FailOnlyOnMerge());
Document doc = new Document();
Field idField = newStringField("id", "", Field.Store.YES);
doc.add(idField);
IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()))
.setMergeScheduler(new MyMergeScheduler())
.setMaxBufferedDocs(2).setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH)
.setMergePolicy(newLogMergePolicy());
ByteArrayOutputStream baos = new ByteArrayOutputStream();
infoStream = new PrintStreamInfoStream(new PrintStream(baos, true, IOUtils.UTF_8));
iwc.setInfoStream(infoStream);
IndexWriter writer = new IndexWriter(dir, iwc);
LogMergePolicy logMP = (LogMergePolicy) writer.getConfig().getMergePolicy();
logMP.setMergeFactor(10);
for(int i=0;i<20;i++) {
writer.addDocument(doc);
}
try {
((MyMergeScheduler) writer.getConfig().getMergeScheduler()).sync();
} catch (IllegalStateException ise) {
// OK
}
writer.rollback();
try {
assertTrue(mergeThreadCreated);
assertTrue(mergeCalled);
assertTrue(excCalled);
} catch (AssertionError ae) {
System.out.println("TEST FAILED; IW infoStream output:");
System.out.println(baos.toString(IOUtils.UTF_8));
throw ae;
}
dir.close();
}
private void loadExternalFileDictionary(SolrCore core, SolrIndexSearcher searcher) {
try {
IndexSchema schema = null == searcher ? core.getLatestSchema() : searcher.getSchema();
// Get the field's analyzer
if (fieldTypeName != null && schema.getFieldTypeNoEx(fieldTypeName) != null) {
FieldType fieldType = schema.getFieldTypes().get(fieldTypeName);
// Do index-time analysis using the given fieldType's analyzer
Directory ramDir = new ByteBuffersDirectory();
LogMergePolicy mp = new LogByteSizeMergePolicy();
mp.setMergeFactor(300);
IndexWriter writer = new IndexWriter(
ramDir,
new IndexWriterConfig(fieldType.getIndexAnalyzer()).
setMaxBufferedDocs(150).
setMergePolicy(mp).
setOpenMode(IndexWriterConfig.OpenMode.CREATE)
// TODO: if we enable this, codec gets angry since field won't exist in the schema
// .setCodec(core.getCodec())
);
List<String> lines = core.getResourceLoader().getLines(sourceLocation, characterEncoding);
for (String s : lines) {
Document d = new Document();
d.add(new TextField(WORD_FIELD_NAME, s, Field.Store.NO));
writer.addDocument(d);
}
writer.forceMerge(1);
writer.close();
dictionary = new HighFrequencyDictionary(DirectoryReader.open(ramDir),
WORD_FIELD_NAME, 0.0f);
} else {
// check if character encoding is defined
if (characterEncoding == null) {
dictionary = new PlainTextDictionary(core.getResourceLoader().openResource(sourceLocation));
} else {
dictionary = new PlainTextDictionary(new InputStreamReader(core.getResourceLoader().openResource(sourceLocation), characterEncoding));
}
}
} catch (IOException e) {
log.error( "Unable to load spellings", e);
}
}