下面列出了org.apache.lucene.index.IndexWriterConfig#setIndexDeletionPolicy ( ) 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
@Test
public void testNoCommit() throws Exception {
Directory indexDir = newDirectory();
IndexWriterConfig conf = new IndexWriterConfig(null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
IndexWriter indexWriter = new IndexWriter(indexDir, conf);
Directory taxoDir = newDirectory();
SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
// should fail when there are no commits to snapshot
expectThrows(IllegalStateException.class, () -> {
new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
});
indexWriter.close();
IOUtils.close(taxoWriter, taxoDir, indexDir);
}
@Test
public void testSegmentsFileLast() throws Exception {
Directory indexDir = newDirectory();
IndexWriterConfig conf = new IndexWriterConfig(null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
IndexWriter indexWriter = new IndexWriter(indexDir, conf);
Directory taxoDir = newDirectory();
SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
try {
indexWriter.addDocument(newDocument(taxoWriter));
indexWriter.commit();
taxoWriter.commit();
Revision rev = new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
Map<String,List<RevisionFile>> sourceFiles = rev.getSourceFiles();
assertEquals(2, sourceFiles.size());
for (List<RevisionFile> files : sourceFiles.values()) {
String lastFile = files.get(files.size() - 1).fileName;
assertTrue(lastFile.startsWith(IndexFileNames.SEGMENTS));
}
indexWriter.close();
} finally {
IOUtils.close(indexWriter, taxoWriter, taxoDir, indexDir);
}
}
@Override
@Before
public void setUp() throws Exception {
super.setUp();
publishDir = newMockDirectory();
handlerDir = newMockDirectory();
sourceDirFactory = new PerSessionDirectoryFactory(createTempDir("replicationClientTest"));
replicator = new LocalReplicator();
callback = new IndexReadyCallback(handlerDir);
handler = new IndexReplicationHandler(handlerDir, callback);
client = new ReplicationClient(replicator, handler, sourceDirFactory);
IndexWriterConfig conf = newIndexWriterConfig(null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
publishWriter = new IndexWriter(publishDir, conf);
}
@Test
public void testRevisionRelease() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = new IndexWriterConfig(null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
IndexWriter writer = new IndexWriter(dir, conf);
try {
writer.addDocument(new Document());
writer.commit();
Revision rev1 = new IndexRevision(writer);
// releasing that revision should not delete the files
rev1.release();
assertTrue(slowFileExists(dir, IndexFileNames.SEGMENTS + "_1"));
rev1 = new IndexRevision(writer); // create revision again, so the files are snapshotted
writer.addDocument(new Document());
writer.commit();
assertNotNull(new IndexRevision(writer));
rev1.release(); // this release should trigger the delete of segments_1
assertFalse(slowFileExists(dir, IndexFileNames.SEGMENTS + "_1"));
} finally {
IOUtils.close(writer, dir);
}
}
@Test
public void testSegmentsFileLast() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = new IndexWriterConfig(null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
IndexWriter writer = new IndexWriter(dir, conf);
try {
writer.addDocument(new Document());
writer.commit();
Revision rev = new IndexRevision(writer);
@SuppressWarnings("unchecked")
Map<String, List<RevisionFile>> sourceFiles = rev.getSourceFiles();
assertEquals(1, sourceFiles.size());
List<RevisionFile> files = sourceFiles.values().iterator().next();
String lastFile = files.get(files.size() - 1).fileName;
assertTrue(lastFile.startsWith(IndexFileNames.SEGMENTS));
writer.close();
} finally {
IOUtils.close(dir);
}
}
@Before
@Override
public void setUp() throws Exception {
super.setUp();
if (VERBOSE) {
System.setProperty("org.eclipse.jetty.LEVEL", "DEBUG"); // sets stderr logging to DEBUG level
}
clientWorkDir = createTempDir("httpReplicatorTest");
handlerIndexDir = newDirectory();
serverIndexDir = newDirectory();
serverReplicator = new LocalReplicator();
startServer();
IndexWriterConfig conf = newIndexWriterConfig(null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
writer = new IndexWriter(serverIndexDir, conf);
reader = DirectoryReader.open(writer);
}
@Override
@Before
public void setUp() throws Exception {
super.setUp();
publishIndexDir = newDirectory();
publishTaxoDir = newDirectory();
handlerIndexDir = newMockDirectory();
handlerTaxoDir = newMockDirectory();
clientWorkDir = createTempDir("replicationClientTest");
sourceDirFactory = new PerSessionDirectoryFactory(clientWorkDir);
replicator = new LocalReplicator();
callback = new IndexAndTaxonomyReadyCallback(handlerIndexDir, handlerTaxoDir);
handler = new IndexAndTaxonomyReplicationHandler(handlerIndexDir, handlerTaxoDir, callback);
client = new ReplicationClient(replicator, handler, sourceDirFactory);
IndexWriterConfig conf = newIndexWriterConfig(null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
publishIndexWriter = new IndexWriter(publishIndexDir, conf);
publishTaxoWriter = new SnapshotDirectoryTaxonomyWriter(publishTaxoDir);
config = new FacetsConfig();
config.setHierarchical("A", true);
}
/**
* Constructor
* @param fs
* @param shard
* @param tempDir
* @param conf
* @throws IOException
*/
public ShardWriter(FileSystem fs, Shard shard, String tempDir, Configuration conf)
throws IOException {
logger.info("Construct a shard writer");
this.conf = conf;
this.fs = fs;
localFs = FileSystem.getLocal(conf);
perm = new Path(shard.getDirectory());
taxoPerm = new Path(shard.getDirectory() + ".taxonomy");
String indexDir = tempDir + "/" + "index";
String taxoDir = tempDir + "/" + "taxo";
temp = new Path(indexDir);
taxoTemp = new Path(taxoDir);
if (localFs.exists(temp)) {
File tempFile = new File(temp.getName());
if (tempFile.exists()) {
LindenReducer.deleteDir(tempFile);
}
}
if (!fs.exists(perm)) {
fs.mkdirs(perm);
} else {
moveToTrash(conf, perm);
fs.mkdirs(perm);
}
if (!fs.exists(taxoPerm)) {
fs.mkdirs(taxoPerm);
} else {
moveToTrash(conf, taxoPerm);
fs.mkdirs(taxoPerm);
}
IndexWriterConfig config = new IndexWriterConfig(Version.LATEST, null);
config.setIndexDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
writer = new IndexWriter(FSDirectory.open(new File(indexDir)), config);
taxoWriter = new DirectoryTaxonomyWriter(FSDirectory.open(new File(taxoDir)));
}
@Override
protected IndexWriterConfig createIndexWriterConfig(OpenMode openMode) {
IndexWriterConfig conf = super.createIndexWriterConfig(openMode);
sdp = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy());
conf.setIndexDeletionPolicy(sdp);
return conf;
}
@Test
public void testRevisionRelease() throws Exception {
Directory indexDir = newDirectory();
IndexWriterConfig conf = new IndexWriterConfig(null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
IndexWriter indexWriter = new IndexWriter(indexDir, conf);
Directory taxoDir = newDirectory();
SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
try {
indexWriter.addDocument(newDocument(taxoWriter));
indexWriter.commit();
taxoWriter.commit();
Revision rev1 = new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
// releasing that revision should not delete the files
rev1.release();
assertTrue(slowFileExists(indexDir, IndexFileNames.SEGMENTS + "_1"));
assertTrue(slowFileExists(taxoDir, IndexFileNames.SEGMENTS + "_1"));
rev1 = new IndexAndTaxonomyRevision(indexWriter, taxoWriter); // create revision again, so the files are snapshotted
indexWriter.addDocument(newDocument(taxoWriter));
indexWriter.commit();
taxoWriter.commit();
assertNotNull(new IndexAndTaxonomyRevision(indexWriter, taxoWriter));
rev1.release(); // this release should trigger the delete of segments_1
assertFalse(slowFileExists(indexDir, IndexFileNames.SEGMENTS + "_1"));
indexWriter.close();
} finally {
IOUtils.close(indexWriter, taxoWriter, taxoDir, indexDir);
}
}
@Before
@Override
public void setUp() throws Exception {
super.setUp();
sourceDir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
sourceWriter = new IndexWriter(sourceDir, conf);
replicator = new LocalReplicator();
}
@Test
public void testNoSnapshotDeletionPolicy() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = new IndexWriterConfig(null);
conf.setIndexDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
IndexWriter writer = new IndexWriter(dir, conf);
// should fail when IndexDeletionPolicy is not Snapshot
expectThrows(IllegalArgumentException.class, () -> {
new IndexRevision(writer);
});
writer.close();
IOUtils.close(dir);
}
@Test
public void testNoCommit() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = new IndexWriterConfig(null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
IndexWriter writer = new IndexWriter(dir, conf);
// should fail when there are no commits to snapshot"
expectThrows(IllegalStateException.class, () -> {
new IndexRevision(writer);
});
writer.close();
IOUtils.close(dir);
}
@Test
public void testOpen() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = new IndexWriterConfig(null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
IndexWriter writer = new IndexWriter(dir, conf);
try {
writer.addDocument(new Document());
writer.commit();
Revision rev = new IndexRevision(writer);
@SuppressWarnings("unchecked")
Map<String, List<RevisionFile>> sourceFiles = rev.getSourceFiles();
String source = sourceFiles.keySet().iterator().next();
for (RevisionFile file : sourceFiles.values().iterator().next()) {
IndexInput src = dir.openInput(file.fileName, IOContext.READONCE);
InputStream in = rev.open(source, file.fileName);
assertEquals(src.length(), in.available());
byte[] srcBytes = new byte[(int) src.length()];
byte[] inBytes = new byte[(int) src.length()];
int offset = 0;
if (random().nextBoolean()) {
int skip = random().nextInt(10);
if (skip >= src.length()) {
skip = 0;
}
in.skip(skip);
src.seek(skip);
offset = skip;
}
src.readBytes(srcBytes, offset, srcBytes.length - offset);
in.read(inBytes, offset, inBytes.length - offset);
assertArrayEquals(srcBytes, inBytes);
IOUtils.close(src, in);
}
writer.close();
} finally {
IOUtils.close(dir);
}
}
@Test
public void testSerialization() throws IOException {
Directory dir = newDirectory();
IndexWriterConfig conf = new IndexWriterConfig(null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
IndexWriter writer = new IndexWriter(dir, conf);
writer.addDocument(new Document());
writer.commit();
Revision rev = new IndexRevision(writer);
SessionToken session1 = new SessionToken("17", rev);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
session1.serialize(new DataOutputStream(baos));
byte[] b = baos.toByteArray();
SessionToken session2 = new SessionToken(new DataInputStream(new ByteArrayInputStream(b)));
assertEquals(session1.id, session2.id);
assertEquals(session1.version, session2.version);
assertEquals(1, session2.sourceFiles.size());
assertEquals(session1.sourceFiles.size(), session2.sourceFiles.size());
assertEquals(session1.sourceFiles.keySet(), session2.sourceFiles.keySet());
List<RevisionFile> files1 = session1.sourceFiles.values().iterator().next();
List<RevisionFile> files2 = session2.sourceFiles.values().iterator().next();
assertEquals(files1, files2);
writer.close();
IOUtils.close(dir);
}
private Path createIndex() throws IOException {
Path indexDir = createTempDir();
Directory dir = newFSDirectory(indexDir);
IndexWriterConfig config = new IndexWriterConfig(new MockAnalyzer(random()));
config.setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE);
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, config);
Document doc1 = new Document();
doc1.add(newStringField("f1", "1", Field.Store.NO));
writer.addDocument(doc1);
writer.commit();
Document doc2 = new Document();
doc2.add(newStringField("f1", "2", Field.Store.NO));
writer.addDocument(doc2);
Document doc3 = new Document();
doc3.add(newStringField("f1", "3", Field.Store.NO));
writer.addDocument(doc3);
writer.commit();
writer.close();
dir.close();
return indexDir;
}
/**
* This method deletes index files of the {@linkplain IndexCommit} for the specified generation number.
*
* @param core The Solr core
* @param dir The index directory storing the snapshot.
* @throws IOException in case of I/O errors.
*/
@SuppressWarnings({"try", "unused"})
private static void deleteSnapshotIndexFiles(SolrCore core, Directory dir, IndexDeletionPolicy delPolicy) throws IOException {
IndexWriterConfig conf = core.getSolrConfig().indexConfig.toIndexWriterConfig(core);
conf.setOpenMode(OpenMode.APPEND);
conf.setMergePolicy(NoMergePolicy.INSTANCE);//Don't want to merge any commits here!
conf.setIndexDeletionPolicy(delPolicy);
conf.setCodec(core.getCodec());
try (SolrIndexWriter iw = new SolrIndexWriter("SolrSnapshotCleaner", dir, conf)) {
// Do nothing. The only purpose of opening index writer is to invoke the Lucene IndexDeletionPolicy#onInit
// method so that we can cleanup the files associated with specified index commit.
// Note the index writer creates a new commit during the close() operation (which is harmless).
}
}
@Test
public void testCreateIndex() throws IOException {
long s = System.nanoTime();
IndexWriterConfig conf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, new KeywordAnalyzer());
IndexDeletionPolicyReader indexDeletionPolicy = new IndexDeletionPolicyReader(
new KeepOnlyLastCommitDeletionPolicy());
conf.setIndexDeletionPolicy(indexDeletionPolicy);
FSDirectory control = FSDirectory.open(fileControl);
Directory dir = getControlDir(control, directory);
// The serial merge scheduler can be useful for debugging.
// conf.setMergeScheduler(new SerialMergeScheduler());
IndexWriter writer = new IndexWriter(dir, conf);
int numDocs = 1000;
DirectoryReader reader = null;
long gen = 0;
for (int i = 0; i < 100; i++) {
if (reader == null) {
reader = DirectoryReader.open(writer, true);
gen = reader.getIndexCommit().getGeneration();
indexDeletionPolicy.register(gen);
} else {
DirectoryReader old = reader;
reader = DirectoryReader.openIfChanged(old, writer, true);
if (reader == null) {
reader = old;
} else {
long newGen = reader.getIndexCommit().getGeneration();
indexDeletionPolicy.register(newGen);
indexDeletionPolicy.unregister(gen);
old.close();
gen = newGen;
}
}
assertEquals(i * numDocs, reader.numDocs());
IndexSearcher searcher = new IndexSearcher(reader);
NumericRangeQuery<Integer> query = NumericRangeQuery.newIntRange("id", 42, 42, true, true);
TopDocs topDocs = searcher.search(query, 10);
assertEquals(i, topDocs.totalHits);
addDocuments(writer, numDocs);
}
writer.close(false);
reader.close();
long e = System.nanoTime();
System.out.println("Total time [" + (e - s) / 1000000.0 + " ms]");
}
private void createWriter() throws IOException {
IndexWriterConfig config = new IndexWriterConfig(Version.LATEST, null);
config.setIndexDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
writer = new IndexWriter(dir, config);
taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
}
@Test
public void testOpen() throws Exception {
Directory indexDir = newDirectory();
IndexWriterConfig conf = new IndexWriterConfig(null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
IndexWriter indexWriter = new IndexWriter(indexDir, conf);
Directory taxoDir = newDirectory();
SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
try {
indexWriter.addDocument(newDocument(taxoWriter));
indexWriter.commit();
taxoWriter.commit();
Revision rev = new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
for (Entry<String,List<RevisionFile>> e : rev.getSourceFiles().entrySet()) {
String source = e.getKey();
@SuppressWarnings("resource") // silly, both directories are closed in the end
Directory dir = source.equals(IndexAndTaxonomyRevision.INDEX_SOURCE) ? indexDir : taxoDir;
for (RevisionFile file : e.getValue()) {
IndexInput src = dir.openInput(file.fileName, IOContext.READONCE);
InputStream in = rev.open(source, file.fileName);
assertEquals(src.length(), in.available());
byte[] srcBytes = new byte[(int) src.length()];
byte[] inBytes = new byte[(int) src.length()];
int offset = 0;
if (random().nextBoolean()) {
int skip = random().nextInt(10);
if (skip >= src.length()) {
skip = 0;
}
in.skip(skip);
src.seek(skip);
offset = skip;
}
src.readBytes(srcBytes, offset, srcBytes.length - offset);
in.read(inBytes, offset, inBytes.length - offset);
assertArrayEquals(srcBytes, inBytes);
IOUtils.close(src, in);
}
}
indexWriter.close();
} finally {
IOUtils.close(indexWriter, taxoWriter, taxoDir, indexDir);
}
}