/**
 * Sets the key to the value for the provided reader;
 * if the key is already set then this doesn't change it.
 */
public void put(AtomicReader reader, CacheKey key, Accountable value) {
  final Object readerKey = reader.getCoreCacheKey();
  synchronized (readerCache) {
    Map<CacheKey, Accountable> innerCache = readerCache.get(readerKey);
    if (innerCache == null) {
      // First time this reader is using FieldCache
      innerCache = new HashMap<>();
      readerCache.put(readerKey, innerCache);
      wrapper.initReader(reader);
    }
    if (innerCache.get(key) == null) {
      innerCache.put(key, value);
    } else {
      // Another thread beat us to it; leave the current
      // value
    }
  }
}
 

@Override
protected Accountable createValue(final AtomicReader reader, CacheKey key, boolean setDocsWithField)
    throws IOException {
  final Map<String, Integer> uidMap = new HashMap<>();

  Uninvert u = new Uninvert() {
    private String currentValue;

    @Override
    public void visitTerm(BytesRef term) {
      currentValue = term.utf8ToString();
    }

    @Override
    public void visitDoc(int docID) {
      uidMap.put(currentValue, docID);
    }

    @Override
    protected TermsEnum termsEnum(Terms terms) throws IOException {
      return terms.iterator(null);
    }
  };
  u.uninvert(reader, key.field, setDocsWithField);
  return new PerReaderUIDMaps(reader.getContext().ord, uidMap);
}
 

@Override
public Object get(int doc) {
  Map<Integer, Object> innerCache = cache.get(context.reader().getCoreCacheKey());
  if (innerCache == null) {
    synchronized (cache) {
      innerCache = cache.get(context.reader().getCoreCacheKey());
      if (innerCache == null) {
        innerCache = new HashMap<>();
        for (int i = 0; i < context.reader().maxDoc(); ++i) {
          innerCache.put(i, createValue(i));
        }
        LOGGER.info("Create cache for context ord:{}, base:{}, max doc:{}.", context.ord, context.docBase, context.reader().maxDoc());
        cache.put(context.reader().getCoreCacheKey(), innerCache);
        context.reader().addCoreClosedListener(new AtomicReader.CoreClosedListener() {
          @Override
          public void onClose(Object ownerCoreCacheKey) {
            cache.remove(ownerCoreCacheKey);
          }
        });
      }
    }
  }
  return innerCache.get(doc);
}
 

private DocIdSet docIdSetToCache(DocIdSet docIdSet, AtomicReader reader, String segmentName, Directory directory)
    throws IOException {
  if (docIdSet == null) {
    // this is better than returning null, as the nonnull result can be cached
    return DocIdSet.EMPTY_DOCIDSET;
  } else if (docIdSet.isCacheable()) {
    return docIdSet;
  } else {
    final DocIdSetIterator it = docIdSet.iterator();
    // null is allowed to be returned by iterator(),
    // in this case we wrap with the empty set,
    // which is cacheable.
    if (it == null) {
      return DocIdSet.EMPTY_DOCIDSET;
    } else {
      final IndexFileBitSet bits = new IndexFileBitSet(reader.maxDoc(), _id, segmentName, directory);
      if (!bits.exists()) {
        bits.create(it);
      }
      bits.load();
      return bits;
    }
  }
}
 

public void addScorers(AtomicReaderContext context, Scorer[] scorers) throws IOException {
  LOG.debug(getPrefix("adding scorers context [{0}] [{1}]"), context, Arrays.asList(scorers));
  if (scorers.length != _length) {
    throw new IOException("Scorer length is not correct expecting [" + _length + "] actual [" + scorers.length + "]");
  }
  Object key = getKey(context);
  Info info = _infoMap.get(key);
  if (info == null) {
    info = new Info(context, scorers, _locks, _instance);
    _infoMap.put(key, info);
  } else {
    AtomicReader reader = context.reader();
    LOG.warn(getPrefix("Info about reader context [{0}] already created, existing Info [{1}] current reader [{2}]."),
        context, info, reader);
  }
}
 

/**
 * This method is used by the index update combiner and process an
 * intermediate form into the current intermediate form. More specifically,
 * the input intermediate forms are a single-document ram index and/or a
 * single delete term.
 * @param form  the input intermediate form
 * @throws IOException
 */
public void process(IntermediateForm form, FacetsConfig facetsConfig) throws IOException {
  if (form.dir.ramBytesUsed() > 0 || form.taxoDir.ramBytesUsed() > 0) {
    if (writer == null) {
      createWriter();
    }

    if (facetsConfig != null) {
      DirectoryTaxonomyWriter.OrdinalMap map = new DirectoryTaxonomyWriter.MemoryOrdinalMap();
      // merge the taxonomies
      taxoWriter.addTaxonomy(form.taxoDir, map);
      int ordinalMap[] = map.getMap();
      DirectoryReader reader = DirectoryReader.open(form.dir);
      try {
        List<AtomicReaderContext> leaves = reader.leaves();
        int numReaders = leaves.size();
        AtomicReader wrappedLeaves[] = new AtomicReader[numReaders];
        for (int i = 0; i < numReaders; i++) {
          wrappedLeaves[i] = new OrdinalMappingAtomicReader(leaves.get(i).reader(), ordinalMap, facetsConfig);
        }
        writer.addIndexes(new MultiReader(wrappedLeaves));
      } finally {
        reader.close();
      }
    } else {
      writer.addIndexes(new Directory[] { form.dir });
    }
    numDocs++;
  }
}
 

private void initReader(AtomicReader reader) {
  if (reader instanceof SegmentReader) {
    reader.addCoreClosedListener(purgeCore);
  } else {
    // we have a slow reader of some sort, try to register a purge event
    // rather than relying on gc:
    Object key = reader.getCoreCacheKey();
    if (key instanceof AtomicReader) {
      ((AtomicReader) key).addReaderClosedListener(purgeReader);
    } else {
      // last chance
      reader.addReaderClosedListener(purgeReader);
    }
  }
}
 

@Override
protected Accountable createValue(final AtomicReader reader, CacheKey key, boolean setDocsWithField)
    throws IOException {
  int maxDoc = reader.maxDoc();

  final int[][] matrix = new int[maxDoc][];
  BinaryDocValues valuesIn = reader.getBinaryDocValues(key.field);
  if (valuesIn == null) {
    for (int i = 0; i < maxDoc; ++i) {
      matrix[i] = new int[0];
    }
    return new IntList(matrix);
  }
  for (int i = 0; i < maxDoc; ++i) {
    String str = valuesIn.get(i).utf8ToString();
    if (StringUtils.isEmpty(str)) {
      matrix[i] = new int[0];
      continue;
    }
    JSONArray array = JSON.parseArray(str);
    matrix[i] = new int[array.size()];
    for (int j = 0; j < array.size(); ++j) {
      matrix[i][j] = array.getInteger(j);
    }
  }
  return new IntList(matrix);
}
 

@Override
public AccessControlReader clone(AtomicReader in) throws IOException {
  try {
    FilterAccessControlReader filterAccessControlReader = (FilterAccessControlReader) super.clone();
    filterAccessControlReader._isClone = true;
    if (_readDocumentVisibilityFilter == null) {
      filterAccessControlReader._noReadAccess = true;
    } else {
      DocIdSet readDocIdSet = _readDocumentVisibilityFilter.getDocIdSet(in.getContext(), in.getLiveDocs());
      if (readDocIdSet == DocIdSet.EMPTY_DOCIDSET || readDocIdSet == null) {
        filterAccessControlReader._noReadAccess = true;
      } else {
        filterAccessControlReader._readBits = readDocIdSet.bits();
        if (filterAccessControlReader._readBits == null) {
          throw new IOException("Read Bits can not be null.");
        }
      }
      filterAccessControlReader._readDocIdSet = readDocIdSet;
    }

    if (_discoverDocumentVisibilityFilter == null) {
      filterAccessControlReader._noDiscoverAccess = true;
    } else {
      DocIdSet discoverDocIdSet = _discoverDocumentVisibilityFilter.getDocIdSet(in.getContext(), in.getLiveDocs());
      if (discoverDocIdSet == DocIdSet.EMPTY_DOCIDSET || discoverDocIdSet == null) {
        filterAccessControlReader._noDiscoverAccess = true;
      } else {
        filterAccessControlReader._discoverBits = discoverDocIdSet.bits();
        if (filterAccessControlReader._discoverBits == null) {
          throw new IOException("Read Bits can not be null.");
        }
      }
      filterAccessControlReader._discoverDocIdSet = discoverDocIdSet;
    }
    return filterAccessControlReader;
  } catch (CloneNotSupportedException e) {
    throw new IOException(e);
  }
}
 

@Override
protected Accountable createValue(final AtomicReader reader, CacheKey key, boolean setDocsWithField)
    throws IOException {
  int maxDoc = reader.maxDoc();

  final float[][] matrix = new float[maxDoc][];
  BinaryDocValues valuesIn = reader.getBinaryDocValues(key.field);
  if (valuesIn == null) {
    for (int i = 0; i < maxDoc; ++i) {
      matrix[i] = new float[0];
    }
    return new FloatList(matrix);
  }
  for (int i = 0; i < maxDoc; ++i) {
    String str = valuesIn.get(i).utf8ToString();
    if (StringUtils.isEmpty(str)) {
      matrix[i] = new float[0];
      continue;
    }
    JSONArray array = JSON.parseArray(str);
    matrix[i] = new float[array.size()];
    for (int j = 0; j < array.size(); ++j) {
      matrix[i][j] = array.getFloat(j);
    }
  }
  return new FloatList(matrix);
}
 

@Override
protected Accountable createValue(final AtomicReader reader, CacheKey key, boolean setDocsWithField)
    throws IOException {
  int maxDoc = reader.maxDoc();

  final double[][] matrix = new double[maxDoc][];
  BinaryDocValues valuesIn = reader.getBinaryDocValues(key.field);
  if (valuesIn == null) {
    for (int i = 0; i < maxDoc; ++i) {
      matrix[i] = new double[0];
    }
    return new DoubleList(matrix);
  }
  for (int i = 0; i < maxDoc; ++i) {
    String str = valuesIn.get(i).utf8ToString();
    if (StringUtils.isEmpty(str)) {
      matrix[i] = new double[0];
      continue;
    }
    JSONArray array = JSON.parseArray(str);
    matrix[i] = new double[array.size()];
    for (int j = 0; j < array.size(); ++j) {
      matrix[i][j] = array.getFloat(j);
    }
  }
  return new DoubleList(matrix);
}
 

@Override
protected Accountable createValue(final AtomicReader reader, CacheKey key, boolean setDocsWithField)
    throws IOException {
  int maxDoc = reader.maxDoc();

  final String[][] matrix = new String[maxDoc][];
  BinaryDocValues valuesIn = reader.getBinaryDocValues(key.field);
  if (valuesIn == null) {
    for (int i = 0; i < maxDoc; ++i) {
      matrix[i] = new String[0];
    }
    return new StringList(matrix);
  }
  for (int i = 0; i < maxDoc; ++i) {
    String str = valuesIn.get(i).utf8ToString();
    if (StringUtils.isEmpty(str)) {
      matrix[i] = new String[0];
      continue;
    }
    JSONArray array = JSON.parseArray(str);
    matrix[i] = new String[array.size()];
    for (int j = 0; j < array.size(); ++j) {
      matrix[i][j] = array.getString(j);
    }
  }
  return new StringList(matrix);
}
 

@Override
public DocIdSet getDocIdSet(String fieldName, BytesRef term, AtomicReader reader) {
  Key key = new Key(fieldName, term, reader.getCoreCacheKey());
  DocIdSet docIdSet = _cache.get(key);
  if (docIdSet != null) {
    LOG.debug("Cache hit for key [" + key + "]");
  } else {
    LOG.debug("Cache miss for key [" + key + "]");
  }
  return docIdSet;
}
 

private static AtomicReader setDocSize(AtomicReader reader, final int count) {
  return new FilterAtomicReader(reader) {
    @Override
    public Bits getLiveDocs() {
      return new Bits() {
        @Override
        public boolean get(int index) {
          return true;
        }

        @Override
        public int length() {
          return count;
        }
      };
    }

    @Override
    public int numDocs() {
      return count;
    }

    @Override
    public int maxDoc() {
      return count;
    }

    @Override
    public void document(int docID, StoredFieldVisitor visitor) throws IOException {
      // Do nothing
    }
  };
}
 

protected void runGisDocValueTest(String s) throws IOException {
  DirectoryReader reader = DirectoryReader.open(_dir);
  AtomicReader atomicReader = reader.leaves().get(0).reader();
  SortedDocValues sortedDocValues = atomicReader.getSortedDocValues("fam.geo");
  BytesRef result = new BytesRef();
  sortedDocValues.get(0, result);
  assertEquals(s, result.utf8ToString());
  System.out.println(result.utf8ToString());
  reader.close();
}
 

private static boolean actualContain(AtomicReader reader, String field, int locDocId) {
  try {
    // index really contains such field of this doc
    return FieldCache.DEFAULT.getDocsWithField(reader, field).get(locDocId);
  } catch (IOException e) {
    return false;
  }
}
 

/**
 * Creates a corpus adapter which uses the Lucene index with the given path
 * and searches on the field with the given field name.
 * 
 * @param indexPath
 * @param fieldName
 * @return
 * @throws CorruptIndexException
 * @throws IOException
 */
public static LuceneCorpusAdapter create(String indexPath, String fieldName)
        throws CorruptIndexException, IOException {
    DirectoryReader dirReader = DirectoryReader.open(new NIOFSDirectory(new File(indexPath)));
    List<AtomicReaderContext> leaves = dirReader.leaves();
    AtomicReader reader[] = new AtomicReader[leaves.size()];
    AtomicReaderContext contexts[] = new AtomicReaderContext[leaves.size()];
    for (int i = 0; i < reader.length; i++) {
        contexts[i] = leaves.get(i);
        reader[i] = contexts[i].reader();
    }
    return new LuceneCorpusAdapter(dirReader, reader, contexts, fieldName);
}
 

protected LuceneCorpusAdapter(DirectoryReader dirReader, AtomicReader reader[], AtomicReaderContext contexts[],
        String fieldName) {
    this.dirReader = dirReader;
    this.reader = reader;
    this.contexts = contexts;
    this.fieldName = fieldName;
}
 

private long getTotalNumberOfRowIds(DirectoryReader reader) throws IOException {
  long total = 0;
  List<AtomicReaderContext> leaves = reader.leaves();
  for (AtomicReaderContext context : leaves) {
    AtomicReader atomicReader = context.reader();
    Terms terms = atomicReader.terms(BlurConstants.ROW_ID);
    long expectedInsertions = terms.size();
    if (expectedInsertions < 0) {
      return -1;
    }
    total += expectedInsertions;
  }
  return total;
}
 

@Override
public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
  AtomicReader reader = context.reader();
  List<DocIdSet> list = new ArrayList<DocIdSet>();

  Fields fields = reader.fields();
  Terms terms = fields.terms(_fieldName);
  if (terms == null) {
    // if field is not present then show nothing.
    return DocIdSet.EMPTY_DOCIDSET;
  }
  TermsEnum iterator = terms.iterator(null);
  BytesRef bytesRef;
  DocumentVisibilityEvaluator visibilityEvaluator = new DocumentVisibilityEvaluator(_authorizations);
  while ((bytesRef = iterator.next()) != null) {
    if (isVisible(visibilityEvaluator, bytesRef)) {
      DocIdSet docIdSet = _filterCacheStrategy.getDocIdSet(_fieldName, bytesRef, reader);
      if (docIdSet != null) {
        list.add(docIdSet);
      } else {
        // Do not use acceptDocs because we want the acl cache to be version
        // agnostic.
        DocsEnum docsEnum = iterator.docs(null, null);
        list.add(buildCache(reader, docsEnum, bytesRef));
      }
    }
  }
  return getLogicalOr(list);
}
 

public SecureDirectoryReader(DirectoryReader in, final AccessControlReader accessControlReader) {
  super(in, new SubReaderWrapper() {

    @Override
    public AtomicReader wrap(AtomicReader reader) {
      try {
        return new SecureAtomicReader(reader, accessControlReader);
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
    }
  });
  _original = in;
}
 

public MergeSortRowIdLookup(IndexReader indexReader) throws IOException {
  if (indexReader instanceof AtomicReader) {
    addAtomicReader((AtomicReader) indexReader);
  } else {
    for (AtomicReaderContext context : indexReader.leaves()) {
      addAtomicReader(context.reader());
    }
  }
}
 

private void checkTerms(IndexSearcher searcher, String fieldName) throws IOException {
  IndexReader reader = searcher.getIndexReader();
  for (AtomicReaderContext context : reader.leaves()) {
    AtomicReader atomicReader = context.reader();
    Fields fields = atomicReader.fields();
    Terms terms = fields.terms(fieldName);
    TermsEnum iterator = terms.iterator(null);
    BytesRef bytesRef = iterator.next();
    if (bytesRef != null) {
      System.out.println(bytesRef.utf8ToString());
      fail("There are only restricted terms for this field [" + fieldName + "]");
    }
  }
}
 

private static void applyColumns(Set<String> alreadyProcessed, OpenBitSet bits,
    Map<String, Set<String>> columnsToFetch, AtomicReader atomicReader, int primeDocRowId, int numberOfDocsInRow,
    Bits liveDocs) throws IOException {
  for (String family : columnsToFetch.keySet()) {
    if (!alreadyProcessed.contains(family)) {
      applyFamily(bits, family, atomicReader, primeDocRowId, numberOfDocsInRow, liveDocs);
      alreadyProcessed.add(family);
    }
  }
}
 

private static void applyFamilies(Set<String> alreadyProcessed, OpenBitSet bits, Set<String> columnFamiliesToFetch,
    AtomicReader segmentReader, int primeDocRowId, int numberOfDocsInRow, Bits liveDocs) throws IOException {
  for (String family : columnFamiliesToFetch) {
    if (!alreadyProcessed.contains(family)) {
      applyFamily(bits, family, segmentReader, primeDocRowId, numberOfDocsInRow, liveDocs);
      alreadyProcessed.add(family);
    }
  }
}
 

private static void applyFamily(OpenBitSet bits, String family, AtomicReader atomicReader, int primeDocRowId,
    int numberOfDocsInRow, Bits liveDocs) throws IOException {
  Fields fields = atomicReader.fields();
  Terms terms = fields.terms(BlurConstants.FAMILY);
  TermsEnum iterator = terms.iterator(null);
  BytesRef text = new BytesRef(family);
  int lastDocId = primeDocRowId + numberOfDocsInRow;
  if (iterator.seekExact(text, true)) {
    DocsEnum docs = iterator.docs(liveDocs, null, DocsEnum.FLAG_NONE);
    int doc = primeDocRowId;
    while ((doc = docs.advance(doc)) < lastDocId) {
      bits.set(doc - primeDocRowId);
    }
  }
}
 

public TermDocIterable(DocsEnum docsEnum, AtomicReader reader, ResetableDocumentStoredFieldVisitor fieldSelector) {
  if (docsEnum == null) {
    throw new NullPointerException("docsEnum can not be null.");
  }
  this.docsEnum = docsEnum;
  this.reader = reader;
  this.fieldSelector = fieldSelector;
}
 

private AtomicReader createIndexReader() throws IOException {
  RAMDirectory directory = new RAMDirectory();
  IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION)));
  for (int i = 0; i < BLOCKS; i++) {
    addDocumentBlock(i, COUNT_PER_BLOCK, writer);
  }
  writer.close();
  return SlowCompositeReaderWrapper.wrap(DirectoryReader.open(directory));
}
 

@Override
public Boolean execute(IndexContext context) throws IOException, InterruptedException {
  try {
    IndexReader indexReader = context.getIndexReader();
    while (true) {
      long hash = 0;
      for (AtomicReaderContext atomicReaderContext : indexReader.leaves()) {
        AtomicReader reader = atomicReaderContext.reader();
        for (String field : reader.fields()) {
          Terms terms = reader.terms(field);
          BytesRef bytesRef;
          TermsEnum iterator = terms.iterator(null);
          while ((bytesRef = iterator.next()) != null) {
            hash += bytesRef.hashCode();
          }
        }
      }
      System.out.println("hashcode = " + hash);
    }
  } catch (IOException e) {
    e.printStackTrace();
    throw e;
  } catch (Throwable t) {
    t.printStackTrace();
    if (t instanceof InterruptedException) {
      throw t;
    } else if (t instanceof RuntimeException) {
      throw (RuntimeException) t;
    }
    throw new RuntimeException(t);
  }
}
 

private AtomicReader getAtomicReader(DirectoryReader reader) throws IOException {
  List<AtomicReaderContext> leaves = reader.leaves();
  if (leaves.size() == 1) {
    return leaves.get(0).reader();
  }
  throw new IOException("Reader [" + reader + "] has more than one segment after optimize.");
}