下面列出了怎么用org.apache.lucene.index.FieldInfos的API类实例代码及写法,或者点击链接到github查看源代码。
static CodecReader wrap(CodecReader reader) throws IOException {
final FieldInfos fieldInfos = reader.getFieldInfos();
final FieldInfo versionInfo = fieldInfos.fieldInfo(VersionFieldMapper.NAME);
if (versionInfo != null && versionInfo.getDocValuesType() != DocValuesType.NONE) {
// the reader is a recent one, it has versions and they are stored
// in a numeric doc values field
return reader;
}
// The segment is an old one, look at the _uid field
final Terms terms = reader.terms(UidFieldMapper.NAME);
if (terms == null || !terms.hasPayloads()) {
// The segment doesn't have an _uid field or doesn't have payloads
// don't try to do anything clever. If any other segment has versions
// all versions of this segment will be initialized to 0
return reader;
}
// convert _uid payloads -> _version docvalues
return new VersionFieldUpgrader(reader);
}
/**
* Called by the default implementation of {@link #getOffsetSource(String)}.
* If there is no searcher then we simply always return null.
*/
protected FieldInfo getFieldInfo(String field) {
if (searcher == null) {
return null;
}
// Need thread-safety for lazy-init but lets avoid 'synchronized' by using double-check locking idiom
FieldInfos fieldInfos = this.fieldInfos; // note: it's volatile; read once
if (fieldInfos == null) {
synchronized (this) {
fieldInfos = this.fieldInfos;
if (fieldInfos == null) {
fieldInfos = FieldInfos.getMergedFieldInfos(searcher.getIndexReader());
this.fieldInfos = fieldInfos;
}
}
}
return fieldInfos.fieldInfo(field);
}
public GetFieldNamesResponse getFieldNames() throws IOException {
openReaderIfChanges();
GetFieldNamesResponse.Builder builder = GetFieldNamesResponse.newBuilder();
Set<String> fields = new HashSet<>();
for (LeafReaderContext subReaderContext : directoryReader.leaves()) {
FieldInfos fieldInfos = subReaderContext.reader().getFieldInfos();
for (FieldInfo fi : fieldInfos) {
String fieldName = fi.name;
fields.add(fieldName);
}
}
fields.forEach(builder::addFieldName);
return builder.build();
}
private void readFields(ChecksumIndexInput meta, FieldInfos infos) throws IOException {
for (int fieldNumber = meta.readInt(); fieldNumber != -1; fieldNumber = meta.readInt()) {
FieldInfo info = infos.fieldInfo(fieldNumber);
if (info == null) {
throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
}
byte type = meta.readByte();
if (type == Lucene80DocValuesFormat.NUMERIC) {
numerics.put(info.name, readNumeric(meta));
} else if (type == Lucene80DocValuesFormat.BINARY) {
binaries.put(info.name, readBinary(meta));
} else if (type == Lucene80DocValuesFormat.SORTED) {
sorted.put(info.name, readSorted(meta));
} else if (type == Lucene80DocValuesFormat.SORTED_SET) {
sortedSets.put(info.name, readSortedSet(meta));
} else if (type == Lucene80DocValuesFormat.SORTED_NUMERIC) {
sortedNumerics.put(info.name, readSortedNumeric(meta));
} else {
throw new CorruptIndexException("invalid type: " + type, meta);
}
}
}
private void readFields(IndexInput meta, FieldInfos infos) throws IOException {
for (int fieldNumber = meta.readInt(); fieldNumber != -1; fieldNumber = meta.readInt()) {
FieldInfo info = infos.fieldInfo(fieldNumber);
if (info == null) {
throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
} else if (!info.hasNorms()) {
throw new CorruptIndexException("Invalid field: " + info.name, meta);
}
NormsEntry entry = new NormsEntry();
entry.docsWithFieldOffset = meta.readLong();
entry.docsWithFieldLength = meta.readLong();
entry.jumpTableEntryCount = meta.readShort();
entry.denseRankPower = meta.readByte();
entry.numDocsWithField = meta.readInt();
entry.bytesPerNorm = meta.readByte();
switch (entry.bytesPerNorm) {
case 0: case 1: case 2: case 4: case 8:
break;
default:
throw new CorruptIndexException("Invalid bytesPerValue: " + entry.bytesPerNorm + ", field: " + info.name, meta);
}
entry.normsOffset = meta.readLong();
norms.put(info.number, entry);
}
}
@Override
public void finish(FieldInfos fis, int numDocs) throws IOException {
if (numBufferedDocs > 0) {
flush();
numDirtyChunks++; // incomplete: we had to force this flush
} else {
assert bufferedDocs.size() == 0;
}
if (docBase != numDocs) {
throw new RuntimeException("Wrote " + docBase + " docs, finish called with numDocs=" + numDocs);
}
indexWriter.finish(numDocs, fieldsStream.getFilePointer());
fieldsStream.writeVLong(numChunks);
fieldsStream.writeVLong(numDirtyChunks);
CodecUtil.writeFooter(fieldsStream);
assert bufferedDocs.size() == 0;
}
@Override
public FieldInfos getFieldInfos() {
if(!flsEnabled) {
return in.getFieldInfos();
}
return flsFieldInfos;
}
VersionFieldUpgrader(CodecReader in) {
super(in);
// Find a free field number
int fieldNumber = 0;
for (FieldInfo fi : in.getFieldInfos()) {
fieldNumber = Math.max(fieldNumber, fi.number + 1);
}
// TODO: lots of things can wrong here...
FieldInfo newInfo = new FieldInfo(VersionFieldMapper.NAME, // field name
fieldNumber, // field number
false, // store term vectors
false, // omit norms
false, // store payloads
IndexOptions.NONE, // index options
DocValuesType.NUMERIC, // docvalues
-1, // docvalues generation
Collections.<String, String>emptyMap() // attributes
);
newInfo.checkConsistency(); // fail merge immediately if above code is wrong
final ArrayList<FieldInfo> fieldInfoList = new ArrayList<>();
for (FieldInfo info : in.getFieldInfos()) {
if (!info.name.equals(VersionFieldMapper.NAME)) {
fieldInfoList.add(info);
}
}
fieldInfoList.add(newInfo);
infos = new FieldInfos(fieldInfoList.toArray(new FieldInfo[fieldInfoList.size()]));
}
public TermVectorLeafReader(String field, Terms terms) {
fields = new Fields() {
@Override
public Iterator<String> iterator() {
return Collections.singletonList(field).iterator();
}
@Override
public Terms terms(String fld) throws IOException {
if (!field.equals(fld)) {
return null;
}
return terms;
}
@Override
public int size() {
return 1;
}
};
IndexOptions indexOptions;
if (!terms.hasFreqs()) {
indexOptions = IndexOptions.DOCS;
} else if (!terms.hasPositions()) {
indexOptions = IndexOptions.DOCS_AND_FREQS;
} else if (!terms.hasOffsets()) {
indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
} else {
indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
}
FieldInfo fieldInfo = new FieldInfo(field, 0,
true, true, terms.hasPayloads(),
indexOptions, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, false);
fieldInfos = new FieldInfos(new FieldInfo[]{fieldInfo});
}
public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) throws IOException {
final String termsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, FSTTermsWriter.TERMS_EXTENSION);
this.postingsReader = postingsReader;
final IndexInput in = state.directory.openInput(termsFileName, state.context);
boolean success = false;
try {
CodecUtil.checkIndexHeader(in, FSTTermsWriter.TERMS_CODEC_NAME,
FSTTermsWriter.TERMS_VERSION_START,
FSTTermsWriter.TERMS_VERSION_CURRENT,
state.segmentInfo.getId(), state.segmentSuffix);
CodecUtil.checksumEntireFile(in);
this.postingsReader.init(in, state);
seekDir(in);
final FieldInfos fieldInfos = state.fieldInfos;
final int numFields = in.readVInt();
for (int i = 0; i < numFields; i++) {
int fieldNumber = in.readVInt();
FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber);
long numTerms = in.readVLong();
long sumTotalTermFreq = in.readVLong();
// if frequencies are omitted, sumTotalTermFreq=sumDocFreq and we only write one value
long sumDocFreq = fieldInfo.getIndexOptions() == IndexOptions.DOCS ? sumTotalTermFreq : in.readVLong();
int docCount = in.readVInt();
TermsReader current = new TermsReader(fieldInfo, in, numTerms, sumTotalTermFreq, sumDocFreq, docCount);
TermsReader previous = fields.put(fieldInfo.name, current);
checkFieldSummary(state.segmentInfo, in, current, previous);
}
success = true;
} finally {
if (success) {
IOUtils.close(in);
} else {
IOUtils.closeWhileHandlingException(in);
}
}
}
public SimpleTextStoredFieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
this.fieldInfos = fn;
boolean success = false;
try {
in = directory.openInput(IndexFileNames.segmentFileName(si.name, "", SimpleTextStoredFieldsWriter.FIELDS_EXTENSION), context);
success = true;
} finally {
if (!success) {
try {
close();
} catch (Throwable t) {} // ensure we throw our original exception
}
}
readIndex(si.maxDoc());
}
@Override
public void finish(FieldInfos fis, int numDocs) throws IOException {
if (numDocsWritten != numDocs) {
throw new RuntimeException("mergeVectors produced an invalid result: mergedDocs is " + numDocs + " but vec numDocs is " + numDocsWritten + " file=" + out.toString() + "; now aborting this merge to prevent index corruption");
}
write(END);
newLine();
SimpleTextUtil.writeChecksum(out, scratch);
}
protected void fillFieldMap(PostingsReaderBase postingsReader, SegmentReadState state, BlockDecoder blockDecoder,
boolean dictionaryOnHeap, IndexInput dictionaryInput, IndexInput blockInput,
Collection<FieldMetadata> fieldMetadataCollection, FieldInfos fieldInfos) throws IOException {
for (FieldMetadata fieldMetadata : fieldMetadataCollection) {
IndexDictionary.BrowserSupplier dictionaryBrowserSupplier = createDictionaryBrowserSupplier(state, dictionaryInput, fieldMetadata, blockDecoder, dictionaryOnHeap);
fieldToTermsMap.put(fieldMetadata.getFieldInfo().name,
new UniformSplitTerms(blockInput, fieldMetadata, postingsReader, blockDecoder, dictionaryBrowserSupplier));
}
}
/**
* @param indexInput {@link IndexInput} must be positioned to the fields metadata
* details by calling {@link #seekFieldsMetadata(IndexInput)} before this call.
* @param blockDecoder Optional block decoder, may be null if none.
*/
protected Collection<FieldMetadata> readFieldsMetadata(IndexInput indexInput, BlockDecoder blockDecoder, FieldInfos fieldInfos,
FieldMetadata.Serializer fieldMetadataReader, int maxNumDocs) throws IOException {
int numFields = indexInput.readVInt();
if (numFields < 0) {
throw new CorruptIndexException("Illegal number of fields= " + numFields, indexInput);
}
return (blockDecoder != null && version >= VERSION_ENCODABLE_FIELDS_METADATA) ?
readEncodedFieldsMetadata(numFields, indexInput, blockDecoder, fieldInfos, fieldMetadataReader, maxNumDocs)
: readUnencodedFieldsMetadata(numFields, indexInput, fieldInfos, fieldMetadataReader, maxNumDocs);
}
protected Collection<FieldMetadata> readEncodedFieldsMetadata(int numFields, DataInput metadataInput, BlockDecoder blockDecoder,
FieldInfos fieldInfos, FieldMetadata.Serializer fieldMetadataReader,
int maxNumDocs) throws IOException {
long encodedLength = metadataInput.readVLong();
if (encodedLength < 0) {
throw new CorruptIndexException("Illegal encoded length: " + encodedLength, metadataInput);
}
BytesRef decodedBytes = blockDecoder.decode(metadataInput, encodedLength);
DataInput decodedMetadataInput = new ByteArrayDataInput(decodedBytes.bytes, 0, decodedBytes.length);
return readUnencodedFieldsMetadata(numFields, decodedMetadataInput, fieldInfos, fieldMetadataReader, maxNumDocs);
}
@Override
protected void fillFieldMap(PostingsReaderBase postingsReader, SegmentReadState state, BlockDecoder blockDecoder,
boolean dictionaryOnHeap, IndexInput dictionaryInput, IndexInput blockInput,
Collection<FieldMetadata> fieldMetadataCollection, FieldInfos fieldInfos) throws IOException {
if (!fieldMetadataCollection.isEmpty()) {
FieldMetadata unionFieldMetadata = createUnionFieldMetadata(fieldMetadataCollection);
// Share the same immutable dictionary between all fields.
IndexDictionary.BrowserSupplier dictionaryBrowserSupplier = createDictionaryBrowserSupplier(state, dictionaryInput, unionFieldMetadata, blockDecoder, dictionaryOnHeap);
for (FieldMetadata fieldMetadata : fieldMetadataCollection) {
fieldToTermsMap.put(fieldMetadata.getFieldInfo().name,
new STUniformSplitTerms(blockInput, fieldMetadata, unionFieldMetadata, postingsReader, blockDecoder, fieldInfos, dictionaryBrowserSupplier));
}
}
}
public STMergingBlockReader(
IndexDictionary.BrowserSupplier dictionaryBrowserSupplier,
IndexInput blockInput,
PostingsReaderBase postingsReader,
FieldMetadata fieldMetadata,
BlockDecoder blockDecoder,
FieldInfos fieldInfos) throws IOException {
super(dictionaryBrowserSupplier, blockInput, postingsReader, fieldMetadata, blockDecoder, fieldInfos);
}
protected STUniformSplitTerms(IndexInput blockInput, FieldMetadata fieldMetadata,
FieldMetadata unionFieldMetadata, PostingsReaderBase postingsReader,
BlockDecoder blockDecoder, FieldInfos fieldInfos,
IndexDictionary.BrowserSupplier dictionaryBrowserSupplier) {
super(blockInput, fieldMetadata, postingsReader, blockDecoder, dictionaryBrowserSupplier);
this.unionFieldMetadata = unionFieldMetadata;
this.fieldInfos = fieldInfos;
}
private static FieldInfos mockFieldInfos() {
return new FieldInfos(
new FieldInfo[]{
mockFieldInfo("f1", 0),
mockFieldInfo("f2", 1),
mockFieldInfo("f3", 2),
mockFieldInfo("f4", 3),
});
}
/**
* Return a query that will return docs like the passed lucene document ID.
*
* @param docNum the documentID of the lucene doc to generate the 'More Like This" query for.
* @return a query that will return docs like the passed lucene document ID.
*/
public Query like(int docNum) throws IOException {
if (fieldNames == null) {
// gather list of valid fields from lucene
Collection<String> fields = FieldInfos.getIndexedFields(ir);
fieldNames = fields.toArray(new String[fields.size()]);
}
return createQuery(retrieveTerms(docNum));
}
public void initialize(BlurInputSplit blurInputSplit, Configuration configuration) throws IOException {
if (_setup) {
return;
}
_setup = true;
_table = blurInputSplit.getTable();
Path localCachePath = BlurInputFormat.getLocalCachePath(configuration);
LOG.info("Local cache path [{0}]", localCachePath);
_directory = BlurInputFormat.getDirectory(configuration, _table.toString(), blurInputSplit.getDir());
SegmentInfoPerCommit commit = segmentInfosRead(_directory, blurInputSplit.getSegmentsName(),
blurInputSplit.getSegmentInfoName());
SegmentInfo segmentInfo = commit.info;
if (localCachePath != null) {
_readingDirectory = copyFilesLocally(configuration, _directory, _table.toString(), blurInputSplit.getDir(),
localCachePath, commit.files(), blurInputSplit.getSegmentInfoName());
} else {
_readingDirectory = _directory;
}
Blur024Codec blur024Codec = new Blur024Codec();
IOContext iocontext = IOContext.READ;
String segmentName = segmentInfo.name;
FieldInfos fieldInfos = blur024Codec.fieldInfosFormat().getFieldInfosReader()
.read(_readingDirectory, segmentName, iocontext);
if (commit.getDelCount() > 0) {
_liveDocs = blur024Codec.liveDocsFormat().readLiveDocs(_readingDirectory, commit, iocontext);
}
_fieldsReader = blur024Codec.storedFieldsFormat().fieldsReader(_readingDirectory, segmentInfo, fieldInfos,
iocontext);
_maxDoc = commit.info.getDocCount();
}
@Override
public void finish(FieldInfos fis, int numDocs) throws IOException {
assert docCount == numDocs;
assert docStatus == (numDocs > 0 ? Status.FINISHED : Status.UNDEFINED);
assert fieldStatus != Status.STARTED;
assert termStatus != Status.STARTED;
in.finish(fis, numDocs);
}
@Override
public void finish(FieldInfos fis, int numDocs) throws IOException {
if (random.nextInt(100) == 0) {
throw new IOException("Fake IOException from TermVectorsWriter.finish()");
}
delegate.finish(fis, numDocs);
}
@Override
public void finish(FieldInfos fis, int numDocs) throws IOException {
if (random.nextInt(100) == 0) {
throw new IOException("Fake IOException from StoredFieldsWriter.finish()");
}
delegate.finish(fis, numDocs);
}
@Override
public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
if (random.nextInt(100) == 0) {
throw new IOException("Fake IOException from FieldInfosFormat.getFieldInfosWriter()");
}
delegate.write(directory, segmentInfo, segmentSuffix, infos, context);
}
@Override
public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
String value = si.getAttribute(MODE_KEY);
if (value == null) {
throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name);
}
Mode mode = Mode.valueOf(value);
return impl(mode).fieldsReader(directory, si, fn, context);
}
@Override
public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);
try (IndexOutput output = directory.createOutput(fileName, context)) {
CodecUtil.writeIndexHeader(output, Lucene50FieldInfosFormat.CODEC_NAME, Lucene50FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix);
output.writeVInt(infos.size());
for (FieldInfo fi : infos) {
fi.checkConsistency();
output.writeString(fi.name);
output.writeVInt(fi.number);
byte bits = 0x0;
if (fi.hasVectors()) bits |= STORE_TERMVECTOR;
if (fi.omitsNorms()) bits |= OMIT_NORMS;
if (fi.hasPayloads()) bits |= STORE_PAYLOADS;
output.writeByte(bits);
output.writeByte(indexOptionsByte(fi.getIndexOptions()));
// pack the DV type and hasNorms in one byte
output.writeByte(docValuesByte(fi.getDocValuesType()));
output.writeLong(fi.getDocValuesGen());
output.writeMapOfStrings(fi.attributes());
}
CodecUtil.writeFooter(output);
}
}
@Override
public final TermVectorsReader vectorsReader(Directory directory,
SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context)
throws IOException {
return new CompressingTermVectorsReader(directory, segmentInfo, segmentSuffix,
fieldInfos, context, formatName, compressionMode);
}
@Override
public void finish(FieldInfos fis, int numDocs) throws IOException {
if (!pendingDocs.isEmpty()) {
flush();
numDirtyChunks++; // incomplete: we had to force this flush
}
if (numDocs != this.numDocs) {
throw new RuntimeException("Wrote " + this.numDocs + " docs, finish called with numDocs=" + numDocs);
}
indexWriter.finish(numDocs, vectorsStream.getFilePointer());
vectorsStream.writeVLong(numChunks);
vectorsStream.writeVLong(numDirtyChunks);
CodecUtil.writeFooter(vectorsStream);
}
PerFieldMergeState(MergeState in) {
this.in = in;
this.orgMergeFieldInfos = in.mergeFieldInfos;
this.orgFieldInfos = new FieldInfos[in.fieldInfos.length];
this.orgFieldsProducers = new FieldsProducer[in.fieldsProducers.length];
System.arraycopy(in.fieldInfos, 0, this.orgFieldInfos, 0, this.orgFieldInfos.length);
System.arraycopy(in.fieldsProducers, 0, this.orgFieldsProducers, 0, this.orgFieldsProducers.length);
}