下面列出了org.apache.zookeeper.KeeperException.SessionExpiredException#org.apache.solr.util.RefCounted 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
private void initRefCntWriter() {
// TODO: since we moved to a read-write lock, and don't rely on the count to close the writer, we don't really
// need this class any more. It could also be a singleton created at the same time as SolrCoreState
// or we could change the API of SolrCoreState to just return the writer and then add a releaseWriter() call.
if (refCntWriter == null && indexWriter != null) {
refCntWriter = new RefCounted<IndexWriter>(indexWriter) {
@Override
public void decref() {
iwLock.readLock().unlock();
super.decref(); // This is now redundant (since we switched to read-write locks), we don't really need to maintain our own reference count.
}
@Override
public void close() {
// We rely on other code to actually close the IndexWriter, and there's nothing special to do when the ref count hits 0
}
};
}
}
/** Opens a new realtime searcher and clears the id caches.
* This may also be called when we updates are being buffered (from PeerSync/IndexFingerprint)
*/
public void openRealtimeSearcher() {
synchronized (this) {
// We must cause a new IndexReader to be opened before anything looks at these caches again
// so that a cache miss will read fresh data.
try {
RefCounted<SolrIndexSearcher> holder = uhandler.core.openNewSearcher(true, true);
holder.decref();
} catch (Exception e) {
SolrException.log(log, "Error opening realtime searcher", e);
return;
}
if (map != null) map.clear();
if (prevMap != null) prevMap.clear();
if (prevMap2 != null) prevMap2.clear();
}
}
/** currently for testing only */
public void deleteAll() {
synchronized (this) {
try {
RefCounted<SolrIndexSearcher> holder = uhandler.core.openNewSearcher(true, true);
holder.decref();
} catch (Exception e) {
SolrException.log(log, "Error opening realtime searcher for deleteByQuery", e);
}
if (map != null) map.clear();
if (prevMap != null) prevMap.clear();
if (prevMap2 != null) prevMap2.clear();
oldDeletes.clear();
deleteByQueries.clear();
}
}
/** Opens a new realtime searcher and returns it's (possibly cached) fingerprint */
public static IndexFingerprint getFingerprint(SolrCore core, long maxVersion) throws IOException {
RTimer timer = new RTimer();
core.getUpdateHandler().getUpdateLog().openRealtimeSearcher();
RefCounted<SolrIndexSearcher> newestSearcher = core.getUpdateHandler().getUpdateLog().uhandler.core.getRealtimeSearcher();
try {
IndexFingerprint f = newestSearcher.get().getIndexFingerprint(maxVersion);
final double duration = timer.stop();
log.info("IndexFingerprint millis:{} result:{}",duration, f);
return f;
} finally {
if (newestSearcher != null) {
newestSearcher.decref();
}
}
}
private void allowDuplicateUpdate(AddUpdateCommand cmd) throws IOException {
RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
try {
IndexWriter writer = iw.get();
Iterable<Document> nestedDocs = cmd.getLuceneDocsIfNested();
if (nestedDocs != null) {
writer.addDocuments(nestedDocs);
} else {
writer.addDocument(cmd.getLuceneDocument());
}
if (ulog != null) ulog.add(cmd);
} finally {
iw.decref();
}
}
private void doNormalUpdate(AddUpdateCommand cmd) throws IOException {
RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
try {
IndexWriter writer = iw.get();
updateDocOrDocValues(cmd, writer);
// Add to the transaction log *after* successfully adding to the
// index, if there was no error.
// This ordering ensures that if we log it, it's definitely been
// added to the the index.
// This also ensures that if a commit sneaks in-between, that we
// know everything in a particular
// log version was definitely committed.
if (ulog != null) ulog.add(cmd);
} finally {
iw.decref();
}
}
@Override
public void delete(DeleteUpdateCommand cmd) throws IOException {
TestInjection.injectDirectUpdateLatch();
deleteByIdCommands.increment();
deleteByIdCommandsCumulative.mark();
if ((cmd.getFlags() & UpdateCommand.IGNORE_INDEXWRITER) != 0 ) {
if (ulog != null) ulog.delete(cmd);
return;
}
Term deleteTerm = getIdTerm(cmd.getIndexedId(), false);
// SolrCore.verbose("deleteDocuments",deleteTerm,writer);
RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
try {
iw.get().deleteDocuments(deleteTerm);
} finally {
iw.decref();
}
// SolrCore.verbose("deleteDocuments",deleteTerm,"DONE");
if (ulog != null) ulog.delete(cmd);
updateDeleteTrackers(cmd);
}
public void prepareCommit(CommitUpdateCommand cmd) throws IOException {
boolean error=true;
try {
log.debug("start {}", cmd);
RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
try {
SolrIndexWriter.setCommitData(iw.get(), cmd.getVersion());
iw.get().prepareCommit();
} finally {
iw.decref();
}
log.debug("end_prepareCommit");
error=false;
}
finally {
if (error) {
numErrors.increment();
numErrorsCumulative.mark();
}
}
}
@Override
public Weight createWeight(IndexSearcher searcher, org.apache.lucene.search.ScoreMode scoreMode, float boost) throws IOException {
SolrRequestInfo info = SolrRequestInfo.getRequestInfo();
CoreContainer container = info.getReq().getCore().getCoreContainer();
final SolrCore fromCore = container.getCore(fromIndex);
if (fromCore == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Cross-core join: no such core " + fromIndex);
}
RefCounted<SolrIndexSearcher> fromHolder = null;
fromHolder = fromCore.getRegisteredSearcher();
final Query joinQuery;
try {
joinQuery = JoinUtil.createJoinQuery(fromField, true,
toField, fromQuery, fromHolder.get(), this.scoreMode);
} finally {
fromCore.close();
fromHolder.decref();
}
return joinQuery.rewrite(searcher.getIndexReader()).createWeight(searcher, scoreMode, boost);
}
private SimpleOrderedMap<Object> getMergeInformation(SolrQueryRequest req, SegmentInfos infos, List<String> mergeCandidates) throws IOException {
SimpleOrderedMap<Object> result = new SimpleOrderedMap<>();
RefCounted<IndexWriter> refCounted = req.getCore().getSolrCoreState().getIndexWriter(req.getCore());
try {
IndexWriter indexWriter = refCounted.get();
if (indexWriter instanceof SolrIndexWriter) {
result.addAll(((SolrIndexWriter)indexWriter).getRunningMerges());
}
//get chosen merge policy
MergePolicy mp = indexWriter.getConfig().getMergePolicy();
//Find merges
MergeSpecification findMerges = mp.findMerges(MergeTrigger.EXPLICIT, infos, indexWriter);
if (findMerges != null && findMerges.merges != null && findMerges.merges.size() > 0) {
for (OneMerge merge : findMerges.merges) {
//TODO: add merge grouping
for (SegmentCommitInfo mergeSegmentInfo : merge.segments) {
mergeCandidates.add(mergeSegmentInfo.info.name);
}
}
}
return result;
} finally {
refCounted.decref();
}
}
final private void cloudDebugLog(SolrCore core, String op) {
if (!log.isDebugEnabled()) {
return;
}
try {
RefCounted<SolrIndexSearcher> searchHolder = core.getNewestSearcher(false);
SolrIndexSearcher searcher = searchHolder.get();
try {
final int totalHits = searcher.count(new MatchAllDocsQuery());
final String nodeName = core.getCoreContainer().getZkController().getNodeName();
log.debug("[{}] {} [{} total hits]", nodeName, op, totalHits);
} finally {
searchHolder.decref();
}
} catch (Exception e) {
log.debug("Error in solrcloud_debug block", e);
}
}
private void initSearcher(SolrCore prev) throws IOException {
// use the (old) writer to open the first searcher
RefCounted<IndexWriter> iwRef = null;
if (prev != null) {
iwRef = prev.getUpdateHandler().getSolrCoreState().getIndexWriter(null);
if (iwRef != null) {
final IndexWriter iw = iwRef.get();
final SolrCore core = this;
newReaderCreator = () -> indexReaderFactory.newReader(iw, core);
}
}
try {
getSearcher(false, false, null, true);
} finally {
newReaderCreator = null;
if (iwRef != null) {
iwRef.decref();
}
}
}
private RefCounted<SolrIndexSearcher> newHolder(SolrIndexSearcher newSearcher, final List<RefCounted<SolrIndexSearcher>> searcherList) {
RefCounted<SolrIndexSearcher> holder = new RefCounted<SolrIndexSearcher>(newSearcher) {
@Override
public void close() {
try {
synchronized (searcherLock) {
// it's possible for someone to get a reference via the _searchers queue
// and increment the refcount while RefCounted.close() is being called.
// we check the refcount again to see if this has happened and abort the close.
// This relies on the RefCounted class allowing close() to be called every
// time the counter hits zero.
if (refcount.get() > 0) return;
searcherList.remove(this);
}
resource.close();
} catch (Exception e) {
// do not allow decref() operations to fail since they are typically called in finally blocks
// and throwing another exception would be very unexpected.
SolrException.log(log, "Error closing searcher:" + this, e);
}
}
};
holder.incref(); // set ref count to 1 to account for this._searcher
return holder;
}
public List<SimpleOrderedMap<Object>> generateTree(ResponseBuilder rb, NamedList<Integer> facetValues) throws IOException {
List<SimpleOrderedMap<Object>> retVal = null;
// First get the searcher for the required collection
RefCounted<SolrIndexSearcher> searcherRef = getSearcherReference(rb);
try {
// Build the facet tree(s)
Collection<TreeFacetField> fTrees = treeBuilder.processFacetTree(searcherRef.get(), extractFacetValues(facetValues));
LOGGER.debug("Extracted {} facet trees", fTrees.size());
if (pruner != null) {
// Prune the trees
fTrees = pruner.prune(fTrees);
}
// Convert the trees into a SimpleOrderedMap
retVal = convertTreeFacetFields(fTrees);
} finally {
// Make sure the search ref count is decreased
searcherRef.decref();
}
return retVal;
}
/**
* Get a reference to the searcher for the required collection. If the collection is
* not the same as the search collection, we assume it is under the same Solr instance.
* @param rb the response builder holding the facets.
* @return a counted reference to the searcher.
* @throws SolrException if the collection cannot be found.
*/
private RefCounted<SolrIndexSearcher> getSearcherReference(ResponseBuilder rb) throws SolrException {
RefCounted<SolrIndexSearcher> searcherRef;
SolrCore currentCore = rb.req.getCore();
if (StringUtils.isBlank(collection)) {
searcherRef = currentCore.getSearcher();
} else {
// Using an alternative core - find it
SolrCore reqCore = currentCore.getCoreDescriptor().getCoreContainer().getCore(collection);
if (reqCore == null) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Collection \"" + collection
+ "\" cannot be found");
}
searcherRef = reqCore.getSearcher();
}
return searcherRef;
}
@Override
public Set<Long> getErrorDocIds() throws IOException
{
Set<Long> errorDocIds = new HashSet<>();
RefCounted<SolrIndexSearcher> refCounted = null;
try
{
refCounted = this.core.getSearcher();
SolrIndexSearcher searcher = refCounted.get();
TermQuery errorQuery = new TermQuery(new Term(FIELD_DOC_TYPE, DOC_TYPE_ERROR_NODE));
DocListCollector docListCollector = new DocListCollector();
searcher.search(errorQuery, docListCollector);
IntArrayList docList = docListCollector.getDocs();
int size = docList.size();
for (int i = 0; i < size; ++i)
{
int doc = docList.get(i);
Document document = searcher.doc(doc, REQUEST_ONLY_ID_FIELD);
IndexableField id = document.getField(FIELD_SOLR4_ID);
String idString = id.stringValue();
if (idString.startsWith(PREFIX_ERROR))
{
idString = idString.substring(PREFIX_ERROR.length());
}
errorDocIds.add(Long.valueOf(idString));
}
}
finally
{
ofNullable(refCounted).ifPresent(RefCounted::decref);
}
return errorDocIds;
}
private boolean isInIndex(long id, LRU cache, String fieldName, boolean populateCache, SolrCore core) throws IOException
{
if(cache.containsKey(id))
{
return true;
}
else
{
RefCounted<SolrIndexSearcher> refCounted = null;
try
{
if(populateCache)
{
cache.put(id, null); // Safe to add this here because we reset this on rollback.
}
refCounted = core.getSearcher();
SolrIndexSearcher searcher = refCounted.get();
FieldType fieldType = searcher.getSchema().getField(fieldName).getType();
TermQuery q = new TermQuery(new Term(fieldName, fieldType.readableToIndexed(Long.toString(id))));
TopDocs topDocs = searcher.search(q, 1);
return topDocs.totalHits > 0;
}
finally
{
ofNullable(refCounted).ifPresent(RefCounted::decref);
}
}
}
/**
* Queries the index and asserts if the count matches documents returned.
* @param queryString
* @param count
* @throws IOException
* @throws org.apache.lucene.queryparser.classic.ParseException
*/
private void assertFTSQuery(String queryString,
int count,
String... name) throws IOException, ParseException
{
SolrServletRequest solrQueryRequest = null;
RefCounted<SolrIndexSearcher>refCounted = null;
try
{
solrQueryRequest = new SolrServletRequest(getCore(), null);
refCounted = getCore().getSearcher(false, true, null);
SolrIndexSearcher solrIndexSearcher = refCounted.get();
SearchParameters searchParameters = new SearchParameters();
searchParameters.setQuery(queryString);
Query query = dataModel.getFTSQuery(new Pair<SearchParameters, Boolean>(searchParameters, Boolean.FALSE),
solrQueryRequest, FTSQueryParser.RerankPhase.SINGLE_PASS);
TopDocs docs = solrIndexSearcher.search(query, count * 2 + 10);
Assert.assertEquals(count, docs.totalHits);
}
finally
{
refCounted.decref();
solrQueryRequest.close();
}
}
public static void waitForDocCount(Query query, long expectedNumFound, long waitMillis)
throws Exception
{
Date date = new Date();
long timeout = date.getTime() + waitMillis;
RefCounted<SolrIndexSearcher> ref = null;
int totalHits = 0;
while(new Date().getTime() < timeout)
{
try
{
ref = getCore().getSearcher();
SolrIndexSearcher searcher = ref.get();
TopDocs topDocs = searcher.search(query, 10);
totalHits = topDocs.totalHits;
if (topDocs.totalHits == expectedNumFound)
{
LOG.warn("Query \"" + query + "\" returned " + totalHits + " as expected");
return;
}
else
{
LOG.warn("Query \"" + query + "\" returned " + totalHits + ", expected " + expectedNumFound);
Thread.sleep(2000);
}
}
finally
{
ref.decref();
}
}
throw new Exception("Wait error expected "+expectedNumFound+" found "+totalHits+" : "+query.toString());
}
protected static void waitForWarming(SolrCore core) throws InterruptedException {
RefCounted<SolrIndexSearcher> registeredSearcher = core.getRegisteredSearcher();
RefCounted<SolrIndexSearcher> newestSearcher = core.getNewestSearcher(false);
while (registeredSearcher == null || registeredSearcher.get() != newestSearcher.get()) {
if (registeredSearcher != null) {
registeredSearcher.decref();
}
newestSearcher.decref();
Thread.sleep(50);
registeredSearcher = core.getRegisteredSearcher();
newestSearcher = core.getNewestSearcher(false);
}
registeredSearcher.decref();
newestSearcher.decref();
}
@Override
public RefCounted<IndexWriter> getIndexWriter(SolrCore core)
throws IOException {
if (core != null && (!core.indexEnabled || core.readOnly)) {
throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE,
"Indexing is temporarily disabled");
}
boolean succeeded = false;
lock(iwLock.readLock());
try {
// Multiple readers may be executing this, but we only want one to open the writer on demand.
synchronized (this) {
if (core == null) {
// core == null is a signal to just return the current writer, or null if none.
initRefCntWriter();
if (refCntWriter == null) return null;
} else {
if (indexWriter == null) {
indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2");
}
initRefCntWriter();
}
refCntWriter.incref();
succeeded = true; // the returned RefCounted<IndexWriter> will release the readLock on a decref()
return refCntWriter;
}
} finally {
// if we failed to return the IW for some other reason, we should unlock.
if (!succeeded) {
iwLock.readLock().unlock();
}
}
}
public Throwable getTragicException() throws IOException {
RefCounted<IndexWriter> ref = getIndexWriter(null);
if (ref == null) return null;
try {
return ref.get().getTragicException();
} finally {
ref.decref();
}
}
Long getMaxVersionFromIndex() {
RefCounted<SolrIndexSearcher> newestSearcher = (uhandler != null && uhandler.core != null)
? uhandler.core.getRealtimeSearcher() : null;
if (newestSearcher == null)
throw new IllegalStateException("No searcher available to lookup max version from index!");
try {
seedBucketsWithHighestVersion(newestSearcher.get());
return getCurrentMaxVersion();
} finally {
newestSearcher.decref();
}
}
/**
* Returns the latest version from the index, searched by the given id (bytes) as seen from the realtime searcher.
* Returns null if no document can be found in the index for the given id.
*/
@SuppressWarnings({"unchecked"})
public Long getVersionFromIndex(BytesRef idBytes) {
// TODO: we could cache much of this and invalidate during a commit.
// TODO: most DocValues classes are threadsafe - expose which.
RefCounted<SolrIndexSearcher> newestSearcher = ulog.uhandler.core.getRealtimeSearcher();
try {
SolrIndexSearcher searcher = newestSearcher.get();
long lookup = searcher.lookupId(idBytes);
if (lookup < 0) return null; // this means the doc doesn't exist in the index yet
ValueSource vs = versionField.getType().getValueSource(versionField, null);
@SuppressWarnings({"rawtypes"})
Map context = ValueSource.newContext(searcher);
vs.createWeight(context, searcher);
FunctionValues fv = vs.getValues(context, searcher.getTopReaderContext().leaves().get((int) (lookup >> 32)));
long ver = fv.longVal((int) lookup);
return ver;
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error reading version from index", e);
} finally {
if (newestSearcher != null) {
newestSearcher.decref();
}
}
}
private DocFoundAndOldUserAndSolrVersions getOldUserVersionsFromFieldCache(BytesRef indexedDocId) {
SolrInputDocument oldDoc = RealTimeGetComponent.getInputDocumentFromTlog(core, indexedDocId, null, null, true);
if (oldDoc == RealTimeGetComponent.DELETED) {
return DocFoundAndOldUserAndSolrVersions.NOT_FOUND;
}
if (oldDoc == null) {
// need to look up in index now...
RefCounted<SolrIndexSearcher> newestSearcher = core.getRealtimeSearcher();
try {
SolrIndexSearcher searcher = newestSearcher.get();
long lookup = searcher.lookupId(indexedDocId);
if (lookup < 0) {
// doc not in index either...
return DocFoundAndOldUserAndSolrVersions.NOT_FOUND;
}
final LeafReaderContext segmentContext = searcher.getTopReaderContext().leaves().get((int)(lookup>>32));
final int docIdInSegment = (int)lookup;
long oldSolrVersion = getFunctionValues(segmentContext, solrVersionField, searcher).longVal(docIdInSegment);
Object[] oldUserVersions = getObjectValues(segmentContext, userVersionFields, searcher, docIdInSegment);
return new DocFoundAndOldUserAndSolrVersions(oldUserVersions, oldSolrVersion);
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error reading version from index", e);
} finally {
if (newestSearcher != null) { //TODO can this ever be null?
newestSearcher.decref();
}
}
} else {
return getUserVersionAndSolrVersionFromDocument(oldDoc);
}
}
private void deleteAll() throws IOException {
if (log.isInfoEnabled()) {
log.info("{} REMOVING ALL DOCUMENTS FROM INDEX", core.getLogId());
}
RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
try {
iw.get().deleteAll();
} finally {
iw.decref();
}
}
@Override
public int mergeIndexes(MergeIndexesCommand cmd) throws IOException {
TestInjection.injectDirectUpdateLatch();
mergeIndexesCommands.mark();
int rc;
log.info("start {}", cmd);
List<DirectoryReader> readers = cmd.readers;
if (readers != null && readers.size() > 0) {
List<CodecReader> mergeReaders = new ArrayList<>();
for (DirectoryReader reader : readers) {
for (LeafReaderContext leaf : reader.leaves()) {
mergeReaders.add(SlowCodecReaderWrapper.wrap(leaf.reader()));
}
}
RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
try {
iw.get().addIndexes(mergeReaders.toArray(new CodecReader[mergeReaders.size()]));
} finally {
iw.decref();
}
rc = 1;
} else {
rc = 0;
}
log.info("end_mergeIndexes");
// TODO: consider soft commit issues
if (rc == 1 && commitTracker.getTimeUpperBound() > 0) {
commitTracker.scheduleCommitWithin(commitTracker.getTimeUpperBound());
} else if (rc == 1 && softCommitTracker.getTimeUpperBound() > 0) {
softCommitTracker.scheduleCommitWithin(softCommitTracker.getTimeUpperBound());
}
return rc;
}
private void openNewSearcherAndUpdateCommitPoint() throws IOException {
RefCounted<SolrIndexSearcher> searcher = null;
IndexCommit commitPoint;
// must get the latest solrCore object because the one we have might be closed because of a reload
// todo stop keeping solrCore around
SolrCore core = solrCore.getCoreContainer().getCore(solrCore.getName());
try {
@SuppressWarnings({"rawtypes"})
Future[] waitSearcher = new Future[1];
searcher = core.getSearcher(true, true, waitSearcher, true);
if (waitSearcher[0] != null) {
try {
waitSearcher[0].get();
} catch (InterruptedException | ExecutionException e) {
SolrException.log(log, e);
}
}
commitPoint = searcher.get().getIndexReader().getIndexCommit();
} finally {
if (searcher != null) {
searcher.decref();
}
core.close();
}
// update the commit point in replication handler
replicationHandler.indexCommitPoint = commitPoint;
}
public void seedVersionBuckets() {
UpdateHandler uh = getUpdateHandler();
if (uh != null && uh.getUpdateLog() != null) {
RefCounted<SolrIndexSearcher> newestSearcher = getRealtimeSearcher();
if (newestSearcher != null) {
try {
uh.getUpdateLog().seedBucketsWithHighestVersion(newestSearcher.get());
} finally {
newestSearcher.decref();
}
} else {
log.warn("No searcher available! Cannot seed version buckets with max from index.");
}
}
}
/**
* Returns the current registered searcher with its reference count incremented, or null if none are registered.
*/
public RefCounted<SolrIndexSearcher> getRegisteredSearcher() {
synchronized (searcherLock) {
if (_searcher != null) {
_searcher.incref();
}
return _searcher;
}
}