org.apache.lucene.index.SegmentReadState#org.apache.lucene.store.IndexInput源码实例Demo

@Test
public void test1() throws IOException {
  RAMDirectory directory = new RAMDirectory();

  String name = "test";

  IndexOutput output = directory.createOutput(name, IOContext.DEFAULT);
  byte[] bs = "hello world".getBytes();
  output.writeBytes(bs, bs.length);
  output.close();

  IndexInput input = directory.openInput(name, IOContext.DEFAULT);
  Cache cache = getCache();
  CacheIndexInput cacheInput = new CacheIndexInput(null, name, input, cache);
  byte[] buf = new byte[bs.length];
  cacheInput.readBytes(buf, 0, buf.length);
  cacheInput.close();

  assertArrayEquals(bs, buf);
  directory.close();
}
 

/**
 * Snapshot individual file
 * <p>
 * This is asynchronous method. Upon completion of the operation latch is getting counted down and any failures are
 * added to the {@code failures} list
 *
 * @param fileInfo file to be snapshotted
 */
private void snapshotFile(final BlobStoreIndexShardSnapshot.FileInfo fileInfo) throws IOException {
    final String file = fileInfo.physicalName();
    try (IndexInput indexInput = store.openVerifyingInput(file, IOContext.READONCE, fileInfo.metadata())) {
        for (int i = 0; i < fileInfo.numberOfParts(); i++) {
            final long partBytes = fileInfo.partBytes(i);

            final InputStreamIndexInput inputStreamIndexInput = new InputStreamIndexInput(indexInput, partBytes);
            InputStream inputStream = snapshotRateLimiter == null ? inputStreamIndexInput : new RateLimitingInputStream(inputStreamIndexInput, snapshotRateLimiter, snapshotThrottleListener);
            inputStream = new AbortableInputStream(inputStream, fileInfo.physicalName());
            blobContainer.writeBlob(fileInfo.partName(i), inputStream, partBytes);
        }
        Store.verify(indexInput);
        snapshotStatus.addProcessedFile(fileInfo.length());
    } catch (Throwable t) {
        failStoreIfCorrupted(t);
        snapshotStatus.addProcessedFile(0);
        throw t;
    }
}
 

public CompressedIndexInput(IndexInput in) throws IOException {
    super("compressed(" + in.toString() + ")");
    this.in = in;
    readHeader(in);
    this.version = in.readInt();
    long metaDataPosition = in.readLong();
    long headerLength = in.getFilePointer();
    in.seek(metaDataPosition);
    this.totalUncompressedLength = in.readVLong();
    int size = in.readVInt();
    offsets = BigArrays.NON_RECYCLING_INSTANCE.newLongArray(size);
    for (int i = 0; i < size; i++) {
        offsets.set(i, in.readVLong());
    }
    this.currentOffsetIdx = -1;
    this.currentUncompressedChunkPointer = 0;
    in.seek(headerLength);
}
 

/** 
 * This test creates compound file based on a single file.
 * Files of different sizes are tested: 0, 1, 10, 100 bytes.
 */
public void testSingleFile() throws IOException {
  int data[] = new int[] { 0, 1, 10, 100 };
  for (int i=0; i<data.length; i++) {
    String testfile = "_" + i + ".test";
    Directory dir = newDirectory();
    SegmentInfo si = newSegmentInfo(dir, "_" + i);
    createSequenceFile(dir, testfile, (byte) 0, data[i], si.getId(), "suffix");
    
    si.setFiles(Collections.singleton(testfile));
    si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
    Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
    
    IndexInput expected = dir.openInput(testfile, newIOContext(random()));
    IndexInput actual = cfs.openInput(testfile, newIOContext(random()));
    assertSameStreams(testfile, expected, actual);
    assertSameSeekBehavior(testfile, expected, actual);
    expected.close();
    actual.close();
    cfs.close();
    dir.close();
  }
}
 

private void assertAdvanceBeyondEnd(BitSet set, Directory dir) throws IOException {
  final int cardinality = set.cardinality();
  final byte denseRankPower = 9; // Not tested here so fixed to isolate factors
  long length;
  int jumpTableentryCount;
  try (IndexOutput out = dir.createOutput("bar", IOContext.DEFAULT)) {
    jumpTableentryCount = IndexedDISI.writeBitSet(new BitSetIterator(set, cardinality), out, denseRankPower);
  }

  try (IndexInput in = dir.openInput("bar", IOContext.DEFAULT)) {
    BitSetIterator disi2 = new BitSetIterator(set, cardinality);
    int doc = disi2.docID();
    int index = 0;
    while (doc < cardinality) {
      doc = disi2.nextDoc();
      index++;
    }

    IndexedDISI disi = new IndexedDISI(in, 0L, in.length(), jumpTableentryCount, denseRankPower, cardinality);
    // Advance 1 docID beyond end
    assertFalse("There should be no set bit beyond the valid docID range", disi.advanceExact(set.length()));
    disi.advance(doc); // Should be the special docID signifyin NO_MORE_DOCS from the BitSetIterator
    assertEquals("The index when advancing beyond the last defined docID should be correct",
        index, disi.index()+1); // disi.index()+1 as the while-loop also counts the NO_MORE_DOCS
  }
}
 

/**
 * Loads a {@link NRTSuggester} from {@link org.apache.lucene.store.IndexInput} on or off-heap
 * depending on the provided <code>fstLoadMode</code>
 */
public static NRTSuggester load(IndexInput input, FSTLoadMode fstLoadMode) throws IOException {
  final FST<Pair<Long, BytesRef>> fst;
  if (shouldLoadFSTOffHeap(input, fstLoadMode)) {
    OffHeapFSTStore store = new OffHeapFSTStore();
    IndexInput clone = input.clone();
    clone.seek(input.getFilePointer());
    fst = new FST<>(clone, clone, new PairOutputs<>(
        PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()), store);
    input.seek(clone.getFilePointer() + store.size());
  } else {
    fst = new FST<>(input, input, new PairOutputs<>(
        PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()));
  }

  /* read some meta info */
  int maxAnalyzedPathsPerOutput = input.readVInt();
  /*
   * Label used to denote the end of an input in the FST and
   * the beginning of dedup bytes
   */
  int endByte = input.readVInt();
  int payloadSep = input.readVInt();
  return new NRTSuggester(fst, maxAnalyzedPathsPerOutput, payloadSep);
}
 

/** simple encode/decode */
public void testSimple() throws Exception {
  Directory dir = newDirectory();
  int bitsPerValue = DirectWriter.bitsRequired(2);
  IndexOutput output = dir.createOutput("foo", IOContext.DEFAULT);
  DirectWriter writer = DirectWriter.getInstance(output, 5, bitsPerValue);
  writer.add(1);
  writer.add(0);
  writer.add(2);
  writer.add(1);
  writer.add(2);
  writer.finish();
  output.close();
  IndexInput input = dir.openInput("foo", IOContext.DEFAULT);
  LongValues reader = DirectReader.getInstance(input.randomAccessSlice(0, input.length()), bitsPerValue, 0);
  assertEquals(1, reader.get(0));
  assertEquals(0, reader.get(1));
  assertEquals(2, reader.get(2));
  assertEquals(1, reader.get(3));
  assertEquals(2, reader.get(4));
  input.close();
  dir.close();
}
 

/**
 * Adds the index input to list.
 *
 * @param name the name
 * @param in the in
 * @param postingsFormatName the postings format name
 * @return the string
 * @throws IOException Signals that an I/O exception has occurred.
 */
private String addIndexInputToList(String name, IndexInput in,
    String postingsFormatName) throws IOException {
  if (indexInputList.get(name) != null) {
    indexInputList.get(name).close();
  }
  if (in != null) {
    String localPostingsFormatName = postingsFormatName;
    if (localPostingsFormatName == null) {
      localPostingsFormatName = in.readString();
    } else if (!in.readString().equals(localPostingsFormatName)) {
      throw new IOException("delegate codec " + name + " doesn't equal "
          + localPostingsFormatName);
    }
    indexInputList.put(name, in);
    indexInputOffsetList.put(name, in.getFilePointer());
    return localPostingsFormatName;
  } else {
    log.debug("no " + name + " registered");
    return null;
  }
}
 

private static Directory copyFilesLocally(Configuration configuration, Directory dir, String table, Path shardDir,
    Path localCachePath, Collection<String> files, String segmentName) throws IOException {
  LOG.info("Copying files need to local cache for faster reads [{0}].", shardDir);
  Path localShardPath = new Path(new Path(new Path(localCachePath, table), shardDir.getName()), segmentName);
  HdfsDirectory localDir = new HdfsDirectory(configuration, localShardPath, null);
  for (String name : files) {
    if (!isValidFileToCache(name)) {
      continue;
    }
    LOG.info("Valid file for local copy [{0}].", name);
    if (!isValid(localDir, dir, name)) {
      LastModified lastModified = (LastModified) dir;
      long fileModified = lastModified.getFileModified(name);

      IndexInput input = dir.openInput(name, IOContext.READONCE);
      IndexOutput output = localDir.createOutput(name, IOContext.READONCE);
      output.copyBytes(input, input.length());
      output.close();
      IndexOutput lastMod = localDir.createOutput(name + LASTMOD, IOContext.DEFAULT);
      lastMod.writeLong(fileModified);
      lastMod.close();
    }
  }
  return localDir;
}
 

/**
 * Read values that have been written using variable-length encoding instead of bit-packing.
 */
static void readVIntBlock(IndexInput docIn, int[] docBuffer,
    int[] freqBuffer, int num, boolean indexHasFreq) throws IOException {
  if (indexHasFreq) {
    for(int i=0;i<num;i++) {
      final int code = docIn.readVInt();
      docBuffer[i] = code >>> 1;
      if ((code & 1) != 0) {
        freqBuffer[i] = 1;
      } else {
        freqBuffer[i] = docIn.readVInt();
      }
    }
  } else {
    for(int i=0;i<num;i++) {
      docBuffer[i] = docIn.readVInt();
    }
  }
}
 

public static void readRandomData(IndexInput baseInput, IndexInput testInput, Random random, int sampleSize,
    int maxBufSize, int maxOffset) throws IOException {
  assertEquals(baseInput.length(), testInput.length());
  int fileLength = (int) baseInput.length();
  for (int i = 0; i < sampleSize; i++) {
    int position = random.nextInt(fileLength - maxBufSize);
    int bufSize = random.nextInt(maxBufSize - maxOffset) + 1;
    byte[] buf1 = new byte[bufSize];
    byte[] buf2 = new byte[bufSize];

    int offset = random.nextInt(Math.min(maxOffset, bufSize));
    int len = Math.min(random.nextInt(bufSize - offset), fileLength - position);

    baseInput.seek(position);
    baseInput.readBytes(buf1, offset, len);
    testInput.seek(position);
    testInput.readBytes(buf2, offset, len);
    assertArrayEquals("Read [" + i + "] The position is [" + position + "] and bufSize [" + bufSize + "]", buf1, buf2);
  }
}
 

/** {@inheritDoc} */
@Override public IndexInput openInput(final String name, final IOContext context) throws IOException {
    ensureOpen();

    GridLuceneFile file = fileMap.get(name);

    if (file == null)
        throw new FileNotFoundException(name);

    // Lock for using in stream. Will be unlocked on stream closing.
    file.lockRef();

    if (!fileMap.containsKey(name)) {
        // Unblock for deferred delete.
        file.releaseRef();

        throw new FileNotFoundException(name);
    }

    return new GridLuceneInputStream(name, file);
}
 

private static DocValuesType getDocValuesType(IndexInput input, byte b) throws IOException {
  switch(b) {
  case 0:
    return DocValuesType.NONE;
  case 1:
    return DocValuesType.NUMERIC;
  case 2:
    return DocValuesType.BINARY;
  case 3:
    return DocValuesType.SORTED;
  case 4:
    return DocValuesType.SORTED_SET;
  case 5:
    return DocValuesType.SORTED_NUMERIC;
  default:
    throw new CorruptIndexException("invalid docvalues byte: " + b, input);
  }
}
 

public void testRename() throws IOException {
  String[] listAll = directory.listAll();
  for (String file : listAll) {
    directory.deleteFile(file);
  }
  
  IndexOutput output = directory.createOutput("testing.test", new IOContext());
  output.writeInt(12345);
  output.close();
  directory.rename("testing.test", "testing.test.renamed");
  assertFalse(slowFileExists(directory, "testing.test"));
  assertTrue(slowFileExists(directory, "testing.test.renamed"));
  IndexInput input = directory.openInput("testing.test.renamed", new IOContext());
  assertEquals(12345, input.readInt());
  assertEquals(input.getFilePointer(), input.length());
  input.close();
  directory.deleteFile("testing.test.renamed");
  assertFalse(slowFileExists(directory, "testing.test.renamed"));
}
 

@Override
protected int readSkipData(int level, IndexInput skipStream) throws IOException {
  int delta = skipStream.readVInt();
  docPointer[level] += skipStream.readVLong();

  if (posPointer != null) {
    posPointer[level] += skipStream.readVLong();
    posBufferUpto[level] = skipStream.readVInt();

    if (payloadByteUpto != null) {
      payloadByteUpto[level] = skipStream.readVInt();
    }

    if (payPointer != null) {
      payPointer[level] += skipStream.readVLong();
    }
  }
  readImpacts(level, skipStream);
  return delta;
}
 

public void testReadBogusCRC() throws Exception {
  ByteBuffersDataOutput out = new ByteBuffersDataOutput();
  IndexOutput output = new ByteBuffersIndexOutput(out, "temp", "temp");
  output.writeLong(-1L); // bad
  output.writeLong(1L << 32); // bad
  output.writeLong(-(1L << 32)); // bad
  output.writeLong((1L << 32) - 1); // ok
  output.close();
  IndexInput input = new BufferedChecksumIndexInput(new ByteBuffersIndexInput(out.toDataInput(), "temp"));
  // read 3 bogus values
  for (int i = 0; i < 3; i++) {
    expectThrows(CorruptIndexException.class, () -> {
      CodecUtil.readCRC(input);
    });
  }
  // good value
  CodecUtil.readCRC(input);
}
 

private void doTestImpactSerialization(List<Impact> impacts) throws IOException {
  CompetitiveImpactAccumulator acc = new CompetitiveImpactAccumulator();
  for (Impact impact : impacts) {
    acc.add(impact.freq, impact.norm);
  }
  try(Directory dir = newDirectory()) {
    try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {
      Lucene84SkipWriter.writeImpacts(acc, out);
    }
    try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {
      byte[] b = new byte[Math.toIntExact(in.length())];
      in.readBytes(b, 0, b.length);
      List<Impact> impacts2 = Lucene84ScoreSkipReader.readImpacts(new ByteArrayDataInput(b), new MutableImpactList());
      assertEquals(impacts, impacts2);
    }
  }
}
 

/** Read this many bytes from in */
public void copy(IndexInput in, long byteCount) throws IOException {
  while (byteCount > 0) {
    int left = blockSize - upto;
    if (left == 0) {
      if (currentBlock != null) {
        addBlock(currentBlock);
      }
      currentBlock = new byte[blockSize];
      upto = 0;
      left = blockSize;
    }
    if (left < byteCount) {
      in.readBytes(currentBlock, upto, left, false);
      upto = blockSize;
      byteCount -= left;
    } else {
      in.readBytes(currentBlock, upto, (int) byteCount, false);
      upto += byteCount;
      break;
    }
  }
}
 

private static void readInts24(IndexInput in, int count, int[] docIDs) throws IOException {
  int i;
  for (i = 0; i < count - 7; i += 8) {
    long l1 = in.readLong();
    long l2 = in.readLong();
    long l3 = in.readLong();
    docIDs[i] =  (int) (l1 >>> 40);
    docIDs[i+1] = (int) (l1 >>> 16) & 0xffffff;
    docIDs[i+2] = (int) (((l1 & 0xffff) << 8) | (l2 >>> 56));
    docIDs[i+3] = (int) (l2 >>> 32) & 0xffffff;
    docIDs[i+4] = (int) (l2 >>> 8) & 0xffffff;
    docIDs[i+5] = (int) (((l2 & 0xff) << 16) | (l3 >>> 48));
    docIDs[i+6] = (int) (l3 >>> 24) & 0xffffff;
    docIDs[i+7] = (int) l3 & 0xffffff;
  }
  for (; i < count; ++i) {
    docIDs[i] = (Short.toUnsignedInt(in.readShort()) << 8) | Byte.toUnsignedInt(in.readByte());
  }
}
 

@Test
public void testLongReadAndClone() throws IOException {
  FSDirectory control = FSDirectory.open(fileControl);
  Directory dir = getControlDir(control, directory);
  String name = writeFile(dir,10*1000*1000);
  IndexInput input = dir.openInput(name, IOContext.DEFAULT);
  readFile(input,1000*1000);
  IndexInput clone = input.clone();
  clone.readByte();
  input.close();
}
 

private final Vector readDeleteableFiles() throws IOException {
  Vector result = new Vector();
  if (!directory.fileExists("deletable"))
    return result;

  IndexInput input = directory.openInput("deletable");
  try {
    for (int i = input.readInt(); i > 0; i--)	  // read file names
      result.addElement(input.readString());
  } finally {
    input.close();
  }
  return result;
}
 

private void testEof(String name, Directory directory, long length) throws IOException {
  IndexInput input = directory.openInput(name, new IOContext());
  try {
  input.seek(length);
    try {
      input.readByte();
      fail("should throw eof");
    } catch (IOException e) {
    }
  } finally {
    input.close();
  }
}
 

/**
 * Returns the suggester for a field, if not loaded already, loads
 * the appropriate suggester from CompletionDictionary
 */
public synchronized NRTSuggester suggester() throws IOException {
  if (suggester == null) {
    try (IndexInput dictClone = dictIn.clone()) { // let multiple fields load concurrently
      dictClone.seek(offset);
      suggester = NRTSuggester.load(dictClone, fstLoadMode);
    }
  }
  return suggester;
}
 

TermsDict(TermsDictEntry entry, IndexInput data) throws IOException {
  this.entry = entry;
  RandomAccessInput addressesSlice = data.randomAccessSlice(entry.termsAddressesOffset, entry.termsAddressesLength);
  blockAddresses = DirectMonotonicReader.getInstance(entry.termsAddressesMeta, addressesSlice);
  bytes = data.slice("terms", entry.termsDataOffset, entry.termsDataLength);
  blockMask = (1L << entry.termsDictBlockShift) - 1;
  RandomAccessInput indexAddressesSlice = data.randomAccessSlice(entry.termsIndexAddressesOffset, entry.termsIndexAddressesLength);
  indexAddresses = DirectMonotonicReader.getInstance(entry.termsIndexAddressesMeta, indexAddressesSlice);
  indexBytes = data.slice("terms-index", entry.termsIndexOffset, entry.termsIndexLength);
  term = new BytesRef(entry.maxTermLength);
}
 

/**
 * Creates a CompletionTermsReader to load a field-specific suggester
 * from the index <code>dictIn</code> with <code>offset</code>
 */
CompletionsTermsReader(IndexInput dictIn, long offset, long minWeight, long maxWeight, byte type, FSTLoadMode fstLoadMode) {
  assert minWeight <= maxWeight;
  assert offset >= 0l && offset < dictIn.length();
  this.dictIn = dictIn;
  this.offset = offset;
  this.minWeight = minWeight;
  this.maxWeight = maxWeight;
  this.type = type;
  this.fstLoadMode = fstLoadMode;
}
 

public IndexInput openInput(String name, IOContext context) throws IOException {
  IndexInput indexInput = _internal.openInput(name, context);
  if (_cache.cacheFileForReading(this, name, context) || (_tableBlockCacheFileTypes != null && isCachableFile(name))) {
    return new CacheIndexInput(this, name, indexInput, _cache);
  }
  return indexInput;
}
 

public CachedIndexInput(IndexInput source, int blockSize, String name, String cacheName, Cache cache,
    IOContext context) {
  super("CachedIndexInput(" + source.toString() + ")", context);
  _source = source;
  _blockSize = blockSize;
  _fileLength = source.length();
  _cacheName = cacheName;
  _cache = cache;
  _store = BufferStore.instance(_blockSize);
}
 

@Override
public IndexInput openInput(String name, IOContext context) throws IOException {
  IndexInput in = super.openInput(name, context);
  final FixedBitSet set = readBytes.computeIfAbsent(name, n -> new FixedBitSet(Math.toIntExact(in.length())));
  if (set.length() != in.length()) {
    throw new IllegalStateException();
  }
  return new ReadBytesIndexInputWrapper(in, set::set);
}
 

@Override
public IndexInput openInput(String name, IOContext context) throws IOException {
  LOG.debug("openInput [{0}] [{1}] [{2}]", name, context, getPath());
  if (!fileExists(name)) {
    throw new FileNotFoundException("File [" + name + "] not found.");
  }
  long fileLength = fileLength(name);
  Path path = getPath(name);
  FSInputFileHandle fsInputFileHandle = new FSInputFileHandle(_fileSystem, path, fileLength, name, _resourceTracking,
      _asyncClosing && _useCache);
  HdfsIndexInput input = new HdfsIndexInput(this, fsInputFileHandle, fileLength, _metricsGroup, name,
      _sequentialReadControl.clone());
  return input;
}
 

/** 
 * Returns (but does not validate) the checksum previously written by {@link #checkFooter}.
 * @return actual checksum value
 * @throws IOException if the footer is invalid
 */
public static long retrieveChecksum(IndexInput in) throws IOException {
  if (in.length() < footerLength()) {
    throw new CorruptIndexException("misplaced codec footer (file truncated?): length=" + in.length() + " but footerLength==" + footerLength(), in);
  }
  in.seek(in.length() - footerLength());
  validateFooter(in);
  return readCRC(in);
}