下面列出了怎么用org.apache.lucene.index.StoredFieldVisitor的API类实例代码及写法,或者点击链接到github查看源代码。
@Override
public void document(final int docID, final StoredFieldVisitor visitor) throws IOException {
if(complianceConfig.readHistoryEnabledForIndex(indexService.index().getName())) {
final ComplianceAwareStoredFieldVisitor cv = new ComplianceAwareStoredFieldVisitor(visitor);
if(flsEnabled) {
in.document(docID, new FlsStoredFieldVisitor(maskFields?new HashingStoredFieldVisitor(cv):cv));
} else {
in.document(docID, maskFields?new HashingStoredFieldVisitor(cv):cv);
}
cv.finished();
} else {
if(flsEnabled) {
in.document(docID, new FlsStoredFieldVisitor(maskFields?new HashingStoredFieldVisitor(visitor):visitor));
} else {
in.document(docID, maskFields?new HashingStoredFieldVisitor(visitor):visitor);
}
}
}
private void readField(BytesRef type, FieldInfo fieldInfo, StoredFieldVisitor visitor) throws IOException {
readLine();
assert StringHelper.startsWith(scratch.get(), VALUE);
if (type == TYPE_STRING) {
byte[] bytes = new byte[scratch.length() - VALUE.length];
System.arraycopy(scratch.bytes(), VALUE.length, bytes, 0, bytes.length);
visitor.stringField(fieldInfo, new String(bytes, 0, bytes.length, StandardCharsets.UTF_8));
} else if (type == TYPE_BINARY) {
byte[] copy = new byte[scratch.length()-VALUE.length];
System.arraycopy(scratch.bytes(), VALUE.length, copy, 0, copy.length);
visitor.binaryField(fieldInfo, copy);
} else if (type == TYPE_INT) {
scratchUTF16.copyUTF8Bytes(scratch.bytes(), VALUE.length, scratch.length()-VALUE.length);
visitor.intField(fieldInfo, Integer.parseInt(scratchUTF16.toString()));
} else if (type == TYPE_LONG) {
scratchUTF16.copyUTF8Bytes(scratch.bytes(), VALUE.length, scratch.length()-VALUE.length);
visitor.longField(fieldInfo, Long.parseLong(scratchUTF16.toString()));
} else if (type == TYPE_FLOAT) {
scratchUTF16.copyUTF8Bytes(scratch.bytes(), VALUE.length, scratch.length()-VALUE.length);
visitor.floatField(fieldInfo, Float.parseFloat(scratchUTF16.toString()));
} else if (type == TYPE_DOUBLE) {
scratchUTF16.copyUTF8Bytes(scratch.bytes(), VALUE.length, scratch.length()-VALUE.length);
visitor.doubleField(fieldInfo, Double.parseDouble(scratchUTF16.toString()));
}
}
@Override
public void document(int docID, StoredFieldVisitor visitor) throws IOException {
if (docID != 0) {
throw new IllegalArgumentException("no such doc ID " + docID);
}
if (visitor.needsField(FAKE_SOURCE_FIELD) == StoredFieldVisitor.Status.YES) {
assert operation.source().toBytesRef().offset == 0;
assert operation.source().toBytesRef().length == operation.source().toBytesRef().bytes.length;
visitor.binaryField(FAKE_SOURCE_FIELD, operation.source().toBytesRef().bytes);
}
if (operation.routing() != null && visitor.needsField(FAKE_ROUTING_FIELD) == StoredFieldVisitor.Status.YES) {
visitor.stringField(FAKE_ROUTING_FIELD, operation.routing().getBytes(StandardCharsets.UTF_8));
}
if (visitor.needsField(FAKE_ID_FIELD) == StoredFieldVisitor.Status.YES) {
BytesRef bytesRef = Uid.encodeId(operation.id());
byte[] id = new byte[bytesRef.length];
System.arraycopy(bytesRef.bytes, bytesRef.offset, id, 0, bytesRef.length);
visitor.stringField(FAKE_ID_FIELD, id);
}
}
@Override
public StoredFieldsReader getFieldsReader() {
StoredFieldsReader fieldsReader = super.getFieldsReader();
return new FilterStoredFieldsReader(fieldsReader) {
@Override
public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
if (recoverySourceToKeep != null && recoverySourceToKeep.get(docID)) {
super.visitDocument(docID, visitor);
} else {
super.visitDocument(docID, new FilterStoredFieldVisitor(visitor) {
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (recoverySourceField.equals(fieldInfo.name)) {
return Status.NO;
}
return super.needsField(fieldInfo);
}
});
}
}
};
}
private static void readField(DataInput in, StoredFieldVisitor visitor, FieldInfo info, int bits) throws IOException {
switch (bits & TYPE_MASK) {
case BYTE_ARR:
int length = in.readVInt();
byte[] data = new byte[length];
in.readBytes(data, 0, length);
visitor.binaryField(info, data);
break;
case STRING:
visitor.stringField(info, in.readString());
break;
case NUMERIC_INT:
visitor.intField(info, in.readZInt());
break;
case NUMERIC_FLOAT:
visitor.floatField(info, readZFloat(in));
break;
case NUMERIC_LONG:
visitor.longField(info, readTLong(in));
break;
case NUMERIC_DOUBLE:
visitor.doubleField(info, readZDouble(in));
break;
default:
throw new AssertionError("Unknown type flag: " + Integer.toHexString(bits));
}
}
@Override
public void visitDocument(int docID, StoredFieldVisitor visitor)
throws IOException {
final SerializedDocument doc = document(docID);
for (int fieldIDX = 0; fieldIDX < doc.numStoredFields; fieldIDX++) {
final long infoAndBits = doc.in.readVLong();
final int fieldNumber = (int) (infoAndBits >>> TYPE_BITS);
final FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber);
final int bits = (int) (infoAndBits & TYPE_MASK);
assert bits <= NUMERIC_DOUBLE: "bits=" + Integer.toHexString(bits);
switch(visitor.needsField(fieldInfo)) {
case YES:
readField(doc.in, visitor, fieldInfo, bits);
break;
case NO:
if (fieldIDX == doc.numStoredFields - 1) {// don't skipField on last field value; treat like STOP
return;
}
skipField(doc.in, bits);
break;
case STOP:
return;
}
}
}
/** @see SolrIndexSearcher#doc(int, StoredFieldVisitor) */
public void doc(int docId, StoredFieldVisitor visitor) throws IOException {
if (documentCache != null) {
// get cached document or retrieve it including all fields (and cache it)
Document cached = doc(docId);
visitFromCached(cached, visitor);
} else {
searcher.getIndexReader().document(docId, visitor);
}
}
/** Executes a stored field visitor against a hit from the document cache */
private void visitFromCached(Document document, StoredFieldVisitor visitor) throws IOException {
for (IndexableField f : document) {
final FieldInfo info = searcher.getFieldInfos().fieldInfo(f.name());
final StoredFieldVisitor.Status needsField = visitor.needsField(info);
if (needsField == StoredFieldVisitor.Status.STOP) return;
if (needsField == StoredFieldVisitor.Status.NO) continue;
BytesRef binaryValue = f.binaryValue();
if (binaryValue != null) {
visitor.binaryField(info, toByteArrayUnwrapIfPossible(binaryValue));
continue;
}
Number numericValue = f.numericValue();
if (numericValue != null) {
if (numericValue instanceof Double) {
visitor.doubleField(info, numericValue.doubleValue());
} else if (numericValue instanceof Integer) {
visitor.intField(info, numericValue.intValue());
} else if (numericValue instanceof Float) {
visitor.floatField(info, numericValue.floatValue());
} else if (numericValue instanceof Long) {
visitor.longField(info, numericValue.longValue());
} else {
throw new AssertionError();
}
continue;
}
// must be String
if (f instanceof LargeLazyField) { // optimization to avoid premature string conversion
visitor.stringField(info, toStringUnwrapIfPossible(((LargeLazyField) f).readBytes()));
} else {
visitor.stringField(info, f.stringValue());
}
}
}
private static String getRowId(IndexReader reader, int docId) throws CorruptIndexException, IOException {
reader.document(docId, new StoredFieldVisitor() {
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (ROW_ID.equals(fieldInfo.name)) {
return StoredFieldVisitor.Status.STOP;
}
return StoredFieldVisitor.Status.NO;
}
});
return reader.document(docId).get(ROW_ID);
}
public static ResetableDocumentStoredFieldVisitor getFieldSelector(final Selector selector) {
return new ResetableDocumentStoredFieldVisitor() {
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (ROW_ID.equals(fieldInfo.name)) {
return StoredFieldVisitor.Status.YES;
}
if (RECORD_ID.equals(fieldInfo.name)) {
return StoredFieldVisitor.Status.YES;
}
if (PRIME_DOC.equals(fieldInfo.name)) {
return StoredFieldVisitor.Status.NO;
}
if (FAMILY.equals(fieldInfo.name)) {
return StoredFieldVisitor.Status.YES;
}
if (selector.columnFamiliesToFetch == null && selector.columnsToFetch == null) {
return StoredFieldVisitor.Status.YES;
}
String columnFamily = getColumnFamily(fieldInfo.name);
if (selector.columnFamiliesToFetch != null) {
if (selector.columnFamiliesToFetch.contains(columnFamily)) {
return StoredFieldVisitor.Status.YES;
}
}
String columnName = getColumnName(fieldInfo.name);
if (selector.columnsToFetch != null) {
Set<String> columns = selector.columnsToFetch.get(columnFamily);
if (columns != null && columns.contains(columnName)) {
return StoredFieldVisitor.Status.YES;
}
}
return StoredFieldVisitor.Status.NO;
}
};
}
private static AtomicReader setDocSize(AtomicReader reader, final int count) {
return new FilterAtomicReader(reader) {
@Override
public Bits getLiveDocs() {
return new Bits() {
@Override
public boolean get(int index) {
return true;
}
@Override
public int length() {
return count;
}
};
}
@Override
public int numDocs() {
return count;
}
@Override
public int maxDoc() {
return count;
}
@Override
public void document(int docID, StoredFieldVisitor visitor) throws IOException {
// Do nothing
}
};
}
public ComplianceAwareStoredFieldVisitor(final StoredFieldVisitor delegate) {
super();
this.delegate = delegate;
}
public FlsStoredFieldVisitor(final StoredFieldVisitor delegate) {
super();
this.delegate = delegate;
}
public HashingStoredFieldVisitor(final StoredFieldVisitor delegate) {
super();
this.delegate = delegate;
}
@Override
public void document(int docID, StoredFieldVisitor visitor) throws IOException {
}
@Override
public void visitDocument(int n, StoredFieldVisitor visitor) throws IOException {
in.seek(offsets[n]);
while (true) {
readLine();
if (StringHelper.startsWith(scratch.get(), FIELD) == false) {
break;
}
int fieldNumber = parseIntAt(FIELD.length);
FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber);
readLine();
assert StringHelper.startsWith(scratch.get(), NAME);
readLine();
assert StringHelper.startsWith(scratch.get(), TYPE);
final BytesRef type;
if (equalsAt(TYPE_STRING, scratch.get(), TYPE.length)) {
type = TYPE_STRING;
} else if (equalsAt(TYPE_BINARY, scratch.get(), TYPE.length)) {
type = TYPE_BINARY;
} else if (equalsAt(TYPE_INT, scratch.get(), TYPE.length)) {
type = TYPE_INT;
} else if (equalsAt(TYPE_LONG, scratch.get(), TYPE.length)) {
type = TYPE_LONG;
} else if (equalsAt(TYPE_FLOAT, scratch.get(), TYPE.length)) {
type = TYPE_FLOAT;
} else if (equalsAt(TYPE_DOUBLE, scratch.get(), TYPE.length)) {
type = TYPE_DOUBLE;
} else {
throw new RuntimeException("unknown field type");
}
switch (visitor.needsField(fieldInfo)) {
case YES:
readField(type, fieldInfo, visitor);
break;
case NO:
readLine();
assert StringHelper.startsWith(scratch.get(), VALUE);
break;
case STOP: return;
}
}
}
@Override
public void visitDocument(int n, StoredFieldVisitor visitor) throws IOException {
AssertingCodec.assertThread("StoredFieldsReader", creationThread);
assert n >= 0 && n < maxDoc;
in.visitDocument(n, visitor);
}
/** Visit the stored fields for document <code>docID</code> */
public abstract void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException;
@Test
public void testEstimator() throws Exception {
JettySolrRunner jetty = cluster.getRandomJetty(random());
String randomCoreName = jetty.getCoreContainer().getAllCoreNames().iterator().next();
SolrCore core = jetty.getCoreContainer().getCore(randomCoreName);
RefCounted<SolrIndexSearcher> searcherRef = core.getSearcher();
try {
SolrIndexSearcher searcher = searcherRef.get();
// limit the max length
IndexSizeEstimator estimator = new IndexSizeEstimator(searcher.getRawReader(), 20, 50, true, true);
IndexSizeEstimator.Estimate estimate = estimator.estimate();
Map<String, Long> fieldsBySize = estimate.getFieldsBySize();
assertFalse("empty fieldsBySize", fieldsBySize.isEmpty());
assertEquals(fieldsBySize.toString(), fields.size(), fieldsBySize.size());
fieldsBySize.forEach((k, v) -> assertTrue("unexpected size of " + k + ": " + v, v > 0));
Map<String, Long> typesBySize = estimate.getTypesBySize();
assertFalse("empty typesBySize", typesBySize.isEmpty());
assertTrue("expected at least 8 types: " + typesBySize.toString(), typesBySize.size() >= 8);
typesBySize.forEach((k, v) -> assertTrue("unexpected size of " + k + ": " + v, v > 0));
Map<String, Object> summary = estimate.getSummary();
assertNotNull("summary", summary);
assertFalse("empty summary", summary.isEmpty());
assertEquals(summary.keySet().toString(), fields.size(), summary.keySet().size());
Map<String, Object> details = estimate.getDetails();
assertNotNull("details", details);
assertFalse("empty details", details.isEmpty());
// by type
assertEquals(details.keySet().toString(), 6, details.keySet().size());
// check sampling
estimator.setSamplingThreshold(searcher.getRawReader().maxDoc() / 2);
IndexSizeEstimator.Estimate sampledEstimate = estimator.estimate();
Map<String, Long> sampledFieldsBySize = sampledEstimate.getFieldsBySize();
assertFalse("empty fieldsBySize", sampledFieldsBySize.isEmpty());
// verify that the sampled values are within 50% of the original values
fieldsBySize.forEach((field, size) -> {
Long sampledSize = sampledFieldsBySize.get(field);
assertNotNull("sampled size for " + field + " is missing in " + sampledFieldsBySize, sampledSize);
double delta = (double) size * 0.5;
assertEquals("sampled size of " + field + " is wildly off", (double)size, (double)sampledSize, delta);
});
// verify the reader is still usable - SOLR-13694
IndexReader reader = searcher.getRawReader();
for (LeafReaderContext context : reader.leaves()) {
LeafReader leafReader = context.reader();
assertTrue("unexpected LeafReader class: " + leafReader.getClass().getName(), leafReader instanceof CodecReader);
Bits liveDocs = leafReader.getLiveDocs();
CodecReader codecReader = (CodecReader) leafReader;
StoredFieldsReader storedFieldsReader = codecReader.getFieldsReader();
StoredFieldVisitor visitor = new DocumentStoredFieldVisitor();
assertNotNull(storedFieldsReader);
for (int docId = 0; docId < leafReader.maxDoc(); docId++) {
if (liveDocs != null && !liveDocs.get(docId)) {
continue;
}
storedFieldsReader.visitDocument(docId, visitor);
}
}
} finally {
searcherRef.decref();
core.close();
}
}
public void doc(int docID, StoredFieldVisitor fieldVisitor) throws IOException {
_secureIndexReader.document(docID, fieldVisitor);
}
public ReadMaskStoredFieldVisitor(StoredFieldVisitor visitor, Map<String, String> readMaskFieldsAndMessages) {
_visitor = visitor;
_readMaskFieldsAndMessages = readMaskFieldsAndMessages;
}
@Override
public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
fieldsReader.visitDocument(docID, visitor);
}
FilterStoredFieldVisitor(StoredFieldVisitor visitor) {
this.visitor = visitor;
}
/**
* Sugar for <code>.getIndexReader().document(docID, fieldVisitor)</code>
* @see IndexReader#document(int, StoredFieldVisitor)
*/
public void doc(int docID, StoredFieldVisitor fieldVisitor) throws IOException {
reader.document(docID, fieldVisitor);
}
/**
* Visit a document's fields using a {@link StoredFieldVisitor}.
* This method does not currently add to the Solr document cache.
*
* @see IndexReader#document(int, StoredFieldVisitor)
* @see SolrDocumentFetcher
*/
@Override
public final void doc(int docId, StoredFieldVisitor visitor) throws IOException {
getDocFetcher().doc(docId, visitor);
}