@Override
void flatten(Query sourceQuery, IndexReader reader, Collection<Query> flatQueries, float boost) throws IOException {
    if (sourceQuery instanceof SpanTermQuery) {
        super.flatten(new TermQuery(((SpanTermQuery) sourceQuery).getTerm()), reader, flatQueries, boost);
    } else if (sourceQuery instanceof ConstantScoreQuery) {
        flatten(((ConstantScoreQuery) sourceQuery).getQuery(), reader, flatQueries, boost);
    } else if (sourceQuery instanceof FunctionScoreQuery) {
        flatten(((FunctionScoreQuery) sourceQuery).getSubQuery(), reader, flatQueries, boost);
    } else if (sourceQuery instanceof MultiPhrasePrefixQuery) {
        flatten(sourceQuery.rewrite(reader), reader, flatQueries, boost);
    } else if (sourceQuery instanceof FiltersFunctionScoreQuery) {
        flatten(((FiltersFunctionScoreQuery) sourceQuery).getSubQuery(), reader, flatQueries, boost);
    } else if (sourceQuery instanceof MultiPhraseQuery) {
        MultiPhraseQuery q = ((MultiPhraseQuery) sourceQuery);
        convertMultiPhraseQuery(0, new int[q.getTermArrays().size()], q, q.getTermArrays(), q.getPositions(), reader, flatQueries);
    } else if (sourceQuery instanceof BlendedTermQuery) {
        final BlendedTermQuery blendedTermQuery = (BlendedTermQuery) sourceQuery;
        flatten(blendedTermQuery.rewrite(reader), reader, flatQueries, boost);
    } else {
        super.flatten(sourceQuery, reader, flatQueries, boost);
    }
}
 

public void testNearSpanSimpleQuery() throws Exception {
  doSearching(new SpanNearQuery(new SpanQuery[] {
      new SpanTermQuery(new Term(FIELD_NAME, "beginning")),
      new SpanTermQuery(new Term(FIELD_NAME, "kennedy")) }, 3, false));

  TestHighlightRunner helper = new TestHighlightRunner() {

    @Override
    public void run() throws Exception {
      mode = QUERY;
      doStandardHighlights(analyzer, searcher, hits, query, HighlighterTest.this);
    }
  };

  helper.run();

  assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
      numHighlights == 2);
}
 

public void testSpanHighlighting() throws Exception {
  Query query1 = new SpanNearQuery(new SpanQuery[] {
      new SpanTermQuery(new Term(FIELD_NAME, "wordx")),
      new SpanTermQuery(new Term(FIELD_NAME, "wordy")) }, 1, false);
  Query query2 = new SpanNearQuery(new SpanQuery[] {
      new SpanTermQuery(new Term(FIELD_NAME, "wordy")),
      new SpanTermQuery(new Term(FIELD_NAME, "wordc")) }, 1, false);
  BooleanQuery.Builder bquery = new BooleanQuery.Builder();
  bquery.add(query1, Occur.SHOULD);
  bquery.add(query2, Occur.SHOULD);
  doSearching(bquery.build());
  TestHighlightRunner helper = new TestHighlightRunner() {

    @Override
    public void run() throws Exception {
      mode = QUERY;
      doStandardHighlights(analyzer, searcher, hits, query, HighlighterTest.this);
    }
  };

  helper.run();
  assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
      numHighlights == 7);
}
 

public void testNotSpanSimpleQuery() throws Exception {
  doSearching(new SpanNotQuery(new SpanNearQuery(new SpanQuery[] {
      new SpanTermQuery(new Term(FIELD_NAME, "shot")),
      new SpanTermQuery(new Term(FIELD_NAME, "kennedy")) }, 3, false), new SpanTermQuery(
      new Term(FIELD_NAME, "john"))));
  TestHighlightRunner helper = new TestHighlightRunner() {

    @Override
    public void run() throws Exception {
      mode = QUERY;
      doStandardHighlights(analyzer, searcher, hits, query, HighlighterTest.this);
    }
  };

  helper.run();
  assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
      numHighlights == 4);
}
 

public void testGetBestFragmentsQueryScorer() throws Exception {
  TestHighlightRunner helper = new TestHighlightRunner() {

    @Override
    public void run() throws Exception {
      numHighlights = 0;
      SpanQuery clauses[] = { new SpanTermQuery(new Term("contents", "john")),
          new SpanTermQuery(new Term("contents", "kennedy")), };

      SpanNearQuery snq = new SpanNearQuery(clauses, 1, true);
      doSearching(snq);
      doStandardHighlights(analyzer, searcher, hits, query, HighlighterTest.this);
      // Currently highlights "John" and "Kennedy" separately
      assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
          numHighlights == 2);
    }
  };

  helper.start();
}
 

public void testGetBestFragmentsFilteredQuery() throws Exception {
  TestHighlightRunner helper = new TestHighlightRunner() {

    @Override
    public void run() throws Exception {
      numHighlights = 0;
      SpanQuery clauses[] = { new SpanTermQuery(new Term("contents", "john")),
          new SpanTermQuery(new Term("contents", "kennedy")), };
      SpanNearQuery snq = new SpanNearQuery(clauses, 1, true);
      BooleanQuery.Builder bq = new BooleanQuery.Builder();
      bq.add(snq, Occur.MUST);
      bq.add(TermRangeQuery.newStringRange("contents", "john", "john", true, true), Occur.FILTER);

      doSearching(bq.build());
      doStandardHighlights(analyzer, searcher, hits, query, HighlighterTest.this);
      // Currently highlights "John" and "Kennedy" separately
      assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
          numHighlights == 2);
    }
  };

  helper.start();
}
 

@Override
public MtasSpanQuery rewrite(IndexReader reader) throws IOException {
  Query q = query.rewrite(reader);
  if (q instanceof SpanOrQuery) {
    SpanQuery[] clauses = ((SpanOrQuery) q).getClauses();
    if (clauses.length > MTAS_WILDCARD_EXPAND_BOUNDARY) {
      // forward index solution ?
      throw new IOException("Wildcard expression \""
          + CodecUtil.termValue(term.text()) + "\" expands to "
          + clauses.length + " terms, too many (boundary "
          + MTAS_WILDCARD_EXPAND_BOUNDARY + ")!");
    }
    MtasSpanQuery[] newClauses = new MtasSpanQuery[clauses.length];
    for (int i = 0; i < clauses.length; i++) {
      if (clauses[i] instanceof SpanTermQuery) {
        newClauses[i] = new MtasSpanTermQuery((SpanTermQuery) clauses[i],
            singlePosition);
      } else {
        throw new IOException("no SpanTermQuery after rewrite");
      }
    }
    return new MtasSpanOrQuery(newClauses).rewrite(reader);
  } else {
    throw new IOException("no SpanOrQuery after rewrite");
  }
}
 

/**
 * Instantiates a new mtas extended span term query.
 *
 * @param query the query
 * @param singlePosition the single position
 */
public MtasExtendedSpanTermQuery(SpanTermQuery query,
    boolean singlePosition) {
  super(query.getTerm());
  localTerm = query.getTerm();
  this.singlePosition = singlePosition;
  int i = localTerm.text().indexOf(MtasToken.DELIMITER);
  if (i >= 0) {
    prefix = localTerm.text().substring(0, i);
    value = localTerm.text().substring((i + MtasToken.DELIMITER.length()));
    value = (value.length() > 0) ? value : null;
  } else {
    prefix = localTerm.text();
    value = null;
  }
}
 

public void testFilteredOutSpan() throws IOException {
  indexWriter.addDocument(newDoc("freezing cold stuff like stuff freedom of speech"));
  initReaderSearcherHighlighter();

  WildcardQuery wildcardQuery = new WildcardQuery(new Term("body", "free*"));
  SpanMultiTermQueryWrapper<WildcardQuery> wildcardSpanQuery = new SpanMultiTermQueryWrapper<>(wildcardQuery);
  SpanTermQuery termQuery = new SpanTermQuery(new Term("body", "speech"));
  SpanQuery spanQuery = new SpanNearQuery(new SpanQuery[]{wildcardSpanQuery, termQuery}, 3, false);

  BooleanQuery query = new BooleanQuery.Builder()
      .add(spanQuery, BooleanClause.Occur.MUST)
      .build();

  TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
  String[] snippets = highlighter.highlight("body", query, topDocs);
  // spans' MatchesIterator exposes each underlying term; thus doesn't enclose intermediate "of"
  assertArrayEquals(new String[]{"freezing cold stuff like stuff <b>freedom</b> of <b>speech</b>"}, snippets);
}
 

public void testReverseOrderSpanCollection() throws IOException {
  // Processing order may depend on various optimizations or other weird factor.
  indexWriter.addDocument(newDoc("alpha bravo - alpha charlie"));
  indexWriter.addDocument(newDoc("alpha charlie - alpha bravo"));
  initReaderSearcherHighlighter();

  SpanNearQuery query = new SpanNearQuery(new SpanQuery[]{
      new SpanNearQuery(new SpanQuery[]{
          new SpanTermQuery(new Term("body", "alpha")),
          new SpanTermQuery(new Term("body", "bravo"))
      }, 0, true),
      new SpanNearQuery(new SpanQuery[]{
          new SpanTermQuery(new Term("body", "alpha")),
          new SpanTermQuery(new Term("body", "charlie"))
      }, 0, true)
  }, 10, false);

  TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
  String[] snippets = highlighter.highlight("body", query, topDocs);

  assertArrayEquals(new String[]{
          "<b>alpha</b> <b>bravo</b> - <b>alpha</b> <b>charlie</b>",
          "<b>alpha</b> <b>charlie</b> - <b>alpha</b> <b>bravo</b>",
      },
      snippets);
}
 

public void testSpanNot() throws Exception {
  SpanQuery[] clauses = new SpanQuery[2];
  clauses[0] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "one"));
  clauses[1] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "three"));
  SpanQuery spq = new SpanNearQuery(clauses, 5, true);
  SpanNotQuery snq = new SpanNotQuery(spq, new SpanTermQuery(new Term(PayloadHelper.FIELD, "two")));



  Directory directory = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), directory,
                                                   newIndexWriterConfig(new PayloadAnalyzer()).setSimilarity(similarity));

  Document doc = new Document();
  doc.add(newTextField(PayloadHelper.FIELD, "one two three one four three", Field.Store.YES));
  writer.addDocument(doc);
  IndexReader reader = getOnlyLeafReader(writer.getReader());
  writer.close();

  checkSpans(snq.createWeight(newSearcher(reader, false), ScoreMode.COMPLETE_NO_SCORES, 1f).getSpans(reader.leaves().get(0), SpanWeight.Postings.PAYLOADS), 1, new int[]{2});
  reader.close();
  directory.close();
}
 

@Override
public MtasSpanQuery rewrite(IndexReader reader) throws IOException {
  Query q = query.rewrite(reader);
  if (q instanceof SpanOrQuery) {
    SpanQuery[] clauses = ((SpanOrQuery) q).getClauses();
    MtasSpanQuery[] newClauses = new MtasSpanQuery[clauses.length];
    for (int i = 0; i < clauses.length; i++) {
      if (clauses[i] instanceof SpanTermQuery) {
        newClauses[i] = new MtasSpanTermQuery((SpanTermQuery) clauses[i],
            singlePosition).rewrite(reader);
      } else {
        throw new IOException("no SpanTermQuery after rewrite");
      }
    }
    return new MtasSpanOrQuery(newClauses).rewrite(reader);
  } else {
    throw new IOException("no SpanOrQuery after rewrite");
  }
}
 

@Test
public void testNearQuery() throws IOException {

  //   2     4
  // twenty two
  //  2     4      4     4
  // one hundred twenty two

  SpanNearQuery q = new SpanNearQuery(new SpanQuery[]{
                      new SpanTermQuery(new Term("field", "twenty")),
                      new SpanTermQuery(new Term("field", "two"))
                    }, 0, true);

  checkQuery(q, new MaxPayloadFunction(), new int[]{ 22, 122, 222 }, new float[]{ 4.0f, 4.0f, 4.0f });
  checkQuery(q, new MinPayloadFunction(), new int[]{ 122, 222, 22 }, new float[]{ 4.0f, 4.0f, 2.0f });
  checkQuery(q, new AveragePayloadFunction(), new int[] { 122, 222, 22 }, new float[] { 4.0f, 4.0f, 3.0f });

}
 

@Test
public void testSpanContainingQuery() throws Exception {

  // twenty WITHIN ((one OR hundred) NEAR two)~2
  SpanContainingQuery q = new SpanContainingQuery(
      new SpanNearQuery(new SpanQuery[]{
          new SpanOrQuery(new SpanTermQuery(new Term("field", "one")), new SpanTermQuery(new Term("field", "hundred"))),
          new SpanTermQuery(new Term("field", "two"))
      }, 2, true),
      new SpanTermQuery(new Term("field", "twenty"))
  );

  checkQuery(q, new AveragePayloadFunction(), new int[] { 222, 122 }, new float[]{ 4.0f, 3.666666f });
  checkQuery(q, new MaxPayloadFunction(), new int[]{ 122, 222 }, new float[]{ 4.0f, 4.0f });
  checkQuery(q, new MinPayloadFunction(), new int[]{ 222, 122 }, new float[]{ 4.0f, 2.0f });

}
 

@Test
public void testEquality() {
  SpanQuery sq1 = new SpanTermQuery(new Term("field", "one"));
  SpanQuery sq2 = new SpanTermQuery(new Term("field", "two"));
  PayloadFunction minFunc = new MinPayloadFunction();
  PayloadFunction maxFunc = new MaxPayloadFunction();
  PayloadScoreQuery query1 = new PayloadScoreQuery(sq1, minFunc, PayloadDecoder.FLOAT_DECODER, true);
  PayloadScoreQuery query2 = new PayloadScoreQuery(sq2, minFunc, PayloadDecoder.FLOAT_DECODER, true);
  PayloadScoreQuery query3 = new PayloadScoreQuery(sq2, maxFunc, PayloadDecoder.FLOAT_DECODER, true);
  PayloadScoreQuery query4 = new PayloadScoreQuery(sq2, maxFunc, PayloadDecoder.FLOAT_DECODER, false);
  PayloadScoreQuery query5 = new PayloadScoreQuery(sq1, minFunc, PayloadDecoder.FLOAT_DECODER);

  assertEquals(query1, query5);
  assertFalse(query1.equals(query2));
  assertFalse(query1.equals(query3));
  assertFalse(query1.equals(query4));
  assertFalse(query2.equals(query3));
  assertFalse(query2.equals(query4));
  assertFalse(query3.equals(query4));
}
 

@Override
public MtasSpanQuery rewrite(IndexReader reader) throws IOException {    
  Query q = query.rewrite(reader);
  if (q instanceof SpanOrQuery) {
    SpanQuery[] clauses = ((SpanOrQuery) q).getClauses();
    if (clauses.length > MTAS_REGEXP_EXPAND_BOUNDARY) {
      // forward index solution ?
      throw new IOException("Regexp \"" + CodecUtil.termValue(term.text())
          + "\" expands to " + clauses.length + " terms, too many (boundary "
          + MTAS_REGEXP_EXPAND_BOUNDARY + ")!");
    }
    MtasSpanQuery[] newClauses = new MtasSpanQuery[clauses.length];
    for (int i = 0; i < clauses.length; i++) {
      if (clauses[i] instanceof SpanTermQuery) {
        newClauses[i] = new MtasSpanTermQuery((SpanTermQuery) clauses[i],
            singlePosition).rewrite(reader);
      } else {
        throw new IOException("no SpanTermQuery after rewrite");
      }
    }
    return new MtasSpanOrQuery(newClauses).rewrite(reader);
  } else {
    throw new IOException("no SpanOrQuery after rewrite");
  }
}
 

public void testNoPayload() throws Exception {
  SpanQuery q1 = new PayloadScoreQuery(new SpanTermQuery(new Term(PayloadHelper.NO_PAYLOAD_FIELD, "zero")),
          new MaxPayloadFunction(), PayloadDecoder.FLOAT_DECODER);
  SpanQuery q2 = new PayloadScoreQuery(new SpanTermQuery(new Term(PayloadHelper.NO_PAYLOAD_FIELD, "foo")),
          new MaxPayloadFunction(), PayloadDecoder.FLOAT_DECODER);
  BooleanClause c1 = new BooleanClause(q1, BooleanClause.Occur.MUST);
  BooleanClause c2 = new BooleanClause(q2, BooleanClause.Occur.MUST_NOT);
  BooleanQuery.Builder query = new BooleanQuery.Builder();
  query.add(c1);
  query.add(c2);
  TopDocs hits = searcher.search(query.build(), 100);
  assertTrue("hits is null and it shouldn't be", hits != null);
  assertTrue("hits Size: " + hits.totalHits.value + " is not: " + 1, hits.totalHits.value == 1);
  int[] results = new int[1];
  results[0] = 0;//hits.scoreDocs[0].doc;
  CheckHits.checkHitCollector(random(), query.build(), PayloadHelper.NO_PAYLOAD_FIELD, searcher, results);
}
 

public void testUnorderedPayloadChecks() throws Exception {

    SpanTermQuery term5 = new SpanTermQuery(new Term("field", "five"));
    SpanTermQuery term100 = new SpanTermQuery(new Term("field", "hundred"));
    SpanTermQuery term4 = new SpanTermQuery(new Term("field", "four"));
    SpanNearQuery nearQuery = new SpanNearQuery(new SpanQuery[]{term5, term100, term4}, 0, false);

    List<BytesRef> payloads = new ArrayList<>();
    payloads.add(new BytesRef("pos: " + 2));
    payloads.add(new BytesRef("pos: " + 1));
    payloads.add(new BytesRef("pos: " + 0));

    SpanPayloadCheckQuery payloadQuery = new SpanPayloadCheckQuery(nearQuery, payloads);
    checkHits(payloadQuery, new int[]{ 405 });

    payloads.clear();
    payloads.add(new BytesRef("pos: " + 0));
    payloads.add(new BytesRef("pos: " + 1));
    payloads.add(new BytesRef("pos: " + 2));

    payloadQuery = new SpanPayloadCheckQuery(nearQuery, payloads);
    checkHits(payloadQuery, new int[]{ 504 });

  }
 

public void testEquality() {
  SpanQuery sq1 = new SpanTermQuery(new Term("field", "one"));
  SpanQuery sq2 = new SpanTermQuery(new Term("field", "two"));
  BytesRef payload1 = new BytesRef("pay1");
  BytesRef payload2 = new BytesRef("pay2");
  SpanQuery query1 = new SpanPayloadCheckQuery(sq1, Collections.singletonList(payload1));
  SpanQuery query2 = new SpanPayloadCheckQuery(sq2, Collections.singletonList(payload1));
  SpanQuery query3 = new SpanPayloadCheckQuery(sq1, Collections.singletonList(payload2));
  SpanQuery query4 = new SpanPayloadCheckQuery(sq2, Collections.singletonList(payload2));
  SpanQuery query5 = new SpanPayloadCheckQuery(sq1, Collections.singletonList(payload1));

  assertEquals(query1, query5);
  assertFalse(query1.equals(query2));
  assertFalse(query1.equals(query3));
  assertFalse(query1.equals(query4));
  assertFalse(query2.equals(query3));
  assertFalse(query2.equals(query4));
  assertFalse(query3.equals(query4));
}
 

public void testWildcardProximityRewrites() throws Exception {
  final SpanNearQuery snq = SpanNearQuery.newOrderedNearQuery(FIELD)
      .addClause(new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term(FIELD, "term*"))))
      .addClause(new SpanTermQuery(new Term(FIELD, "foo")))
      .build();

  try (Monitor monitor = newMonitor()) {

    monitor.register(new MonitorQuery("1", snq));

    MatchingQueries<HighlightsMatch> matches = monitor.match(buildDoc("term1 foo"), HighlightsMatch.MATCHER);
    HighlightsMatch m = matches.matches("1");
    assertNotNull(m);
    assertEquals(2, m.getHitCount());
  }
}
 

@Override
public SpanQuery getSpanQuery(Element e) throws ParserException {
  String fieldName = DOMUtils.getAttributeWithInheritanceOrFail(e, "fieldName");
  String value = DOMUtils.getNonBlankTextOrFail(e);

  List<SpanQuery> clausesList = new ArrayList<>();

  try (TokenStream ts = analyzer.tokenStream(fieldName, value)) {
    TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
    ts.reset();
    while (ts.incrementToken()) {
      SpanTermQuery stq = new SpanTermQuery(new Term(fieldName, BytesRef.deepCopyOf(termAtt.getBytesRef())));
      clausesList.add(stq);
    }
    ts.end();
    SpanOrQuery soq = new SpanOrQuery(clausesList.toArray(new SpanQuery[clausesList.size()]));
    float boost = DOMUtils.getAttribute(e, "boost", 1.0f);
    return new SpanBoostQuery(soq, boost);
  }
  catch (IOException ioe) {
    throw new ParserException("IOException parsing value:" + value);
  }
}
 

private static void checkChooseOneWordQuery(boolean span, Query query, String fieldName, String ... expectedTermTexts) {
  final Term term;
  if (span) {
    assertTrue(query instanceof SpanTermQuery);
    final SpanTermQuery stq = (SpanTermQuery)query;
    term = stq.getTerm();
  } else {
    assertTrue(query instanceof TermQuery);
    final TermQuery tq = (TermQuery)query;
    term = tq.getTerm();
  }
  final String text = term.text();
  boolean foundExpected = false;
  for (String expected : expectedTermTexts) {
    foundExpected |= expected.equals(text);
  }
  assertEquals(fieldName, term.field());
  assertTrue("expected term text ("+text+") not found in ("+expectedTermTexts+")", foundExpected);
}
 

@Test
public void testSpanTermQuery() throws Exception {
    // 相当于TermQuery,区别是使用SpanTermQuery可以得到词项的跨度信息
    Query query = new SpanTermQuery(new Term("title", "Toy"));
    TopDocs search = searcher.search(query, 1000);
    Assert.assertEquals(1, search.totalHits.value);
}
 

@Test
public void testSpanFirstQuery() throws Exception {
    // 临近查询(匹配域中[0,n]范围内的词项)
    SpanQuery spanQuery = new SpanTermQuery(new Term("title", "Story"));
    SpanFirstQuery firstQuery = new SpanFirstQuery(spanQuery, 5);
    TopDocs search = searcher.search(firstQuery, 1000);
    Assert.assertEquals(5, search.totalHits.value);
}
 

@Test
public void testFlatQueryShouldBeGeneratedFromSequentiallyShiftedTokens() throws Exception
{
    // prepare test data
    LinkedList<PackedTokenAttributeImpl> tokenSequenceWithRepeatedGroup = new LinkedList<PackedTokenAttributeImpl>();
    tokenSequenceWithRepeatedGroup.add(getTokenAttribute(TEST_QUERY.substring(0, 4), 0, 4));
    tokenSequenceWithRepeatedGroup.add(getTokenAttribute(TEST_QUERY.substring(5, 6), 5, 6));
    tokenSequenceWithRepeatedGroup.add(getTokenAttribute(TEST_QUERY.substring(6, 10), 6, 10));
    tokenSequenceWithRepeatedGroup.add(getTokenAttribute(TEST_QUERY.substring(10, 11), 10, 11));
    
    assertTrue("All tokens in test data must be sequentially shifted",
            parser.isAllTokensSequentiallyShifted(tokenSequenceWithRepeatedGroup));
    assertTrue(parser.getEnablePositionIncrements());
    
    LinkedList<LinkedList<PackedTokenAttributeImpl>> fixedTokenSequences = new LinkedList<LinkedList<PackedTokenAttributeImpl>>();
    fixedTokenSequences.add(tokenSequenceWithRepeatedGroup);
    
    // call method to test
    SpanQuery q = parser.generateSpanOrQuery(TEST_FIELD, fixedTokenSequences);
    
    // check results
    assertNotNull(q);
    assertTrue(q instanceof SpanNearQuery);
    SpanNearQuery spanNearQuery = (SpanNearQuery) q;
    assertEquals("Slop between term must be 0", 0, spanNearQuery.getSlop());
    assertTrue("Terms must be in order", spanNearQuery.isInOrder());
    
    SpanQuery[] termClauses = spanNearQuery.getClauses();
    assertEquals("Flat query must be generated (Query: " + q + ")", tokenSequenceWithRepeatedGroup.size(), termClauses.length);
    for (int i = 0; i < termClauses.length; i++)
    {
        assertTrue(termClauses[i] instanceof SpanTermQuery);
        assertEquals("All tokens must become spanQuery terms",
                tokenSequenceWithRepeatedGroup.get(i).toString(), ((SpanTermQuery) termClauses[i]).getTerm().text());
    }
}
 

public void testSpanNearQuery() throws IOException, InvalidTokenOffsetsException {
  try (Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)) {
    final Query query = new SpanNearQuery(new SpanQuery[] {
        new SpanTermQuery(new Term("test", "foo")),
        new SpanTermQuery(new Term("test", "bar"))}, 0, true);
    final Highlighter highlighter = new Highlighter(new SimpleHTMLFormatter(), new QueryScorer(query));
    assertEquals("this is a <B>foo</B> <B>bar</B> example",
        highlighter.getBestFragment(analyzer, "test", "this is a foo bar example"));
    assertNull(highlighter.getBestFragment(analyzer, "test", "this does not match"));
  }
}
 

/**
 * Test it does *not* highlight the same term's not next to the span-near.  "charlie" in this case.
 * This particular example exercises "Rewrite" plus "MTQ" in the same query.
 */
public void testRewriteAndMtq() throws IOException {
  indexWriter.addDocument(newDoc("alpha bravo charlie - charlie bravo alpha"));
  initReaderSearcherHighlighter();

  SpanNearQuery snq = new SpanNearQuery(
      new SpanQuery[]{
          new SpanTermQuery(new Term("body", "bravo")),
          new SpanMultiTermQueryWrapper<>(new PrefixQuery(new Term("body", "ch")))}, // REWRITES
      0, true);

  BooleanQuery query = new BooleanQuery.Builder()
      .add(snq, BooleanClause.Occur.MUST)
      .add(new PrefixQuery(new Term("body", "al")), BooleanClause.Occur.MUST) // MTQ
      .add(newPhraseQuery("body", "alpha bravo"), BooleanClause.Occur.MUST)
      // add queries for other fields; we shouldn't highlight these because of that.
      .add(newPhraseQuery("title", "bravo alpha"), BooleanClause.Occur.SHOULD)
      .build();

  TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
  String[] snippets = highlighter.highlight("body", query, topDocs);

  if (highlighter.getFlags("body").contains(HighlightFlag.WEIGHT_MATCHES)) {
    assertArrayEquals(new String[]{"<b>alpha bravo</b> <b>charlie</b> - charlie bravo <b>alpha</b>"}, snippets);
  } else {
    assertArrayEquals(new String[]{"<b>alpha</b> <b>bravo</b> <b>charlie</b> - charlie bravo <b>alpha</b>"}, snippets);
  }

  // do again, this time with MTQ disabled.  We should only find "alpha bravo".
  highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
  highlighter.setHandleMultiTermQuery(false);//disable but leave phrase processing enabled

  topDocs = searcher.search(query, 10, Sort.INDEXORDER);
  snippets = highlighter.highlight("body", query, topDocs);

  assertArrayEquals(new String[]{"<b>alpha</b> <b>bravo</b> charlie - charlie bravo alpha"},
      snippets);
}
 

/**
 * Like {@link #testRewriteAndMtq} but no rewrite.
 */
public void testMtq() throws IOException {
  indexWriter.addDocument(newDoc("alpha bravo charlie - charlie bravo alpha"));
  initReaderSearcherHighlighter();

  SpanNearQuery snq = new SpanNearQuery(
      new SpanQuery[]{
          new SpanTermQuery(new Term("body", "bravo")),
          new SpanTermQuery(new Term("body", "charlie"))}, // does NOT rewrite
      0, true);

  BooleanQuery query = new BooleanQuery.Builder()
      .add(snq, BooleanClause.Occur.MUST)
      .add(new PrefixQuery(new Term("body", "al")), BooleanClause.Occur.MUST) // MTQ
      .add(newPhraseQuery("body", "alpha bravo"), BooleanClause.Occur.MUST)
      // add queries for other fields; we shouldn't highlight these because of that.
      .add(newPhraseQuery("title", "bravo alpha"), BooleanClause.Occur.SHOULD)
      .build();

  TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
  String[] snippets = highlighter.highlight("body", query, topDocs);

  if (highlighter.getFlags("body").contains(HighlightFlag.WEIGHT_MATCHES)) {
    assertArrayEquals(new String[]{"<b>alpha bravo</b> <b>charlie</b> - charlie bravo <b>alpha</b>"}, snippets);
  } else {
    assertArrayEquals(new String[]{"<b>alpha</b> <b>bravo</b> <b>charlie</b> - charlie bravo <b>alpha</b>"}, snippets);
  }

  // do again, this time with MTQ disabled.
  highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
  highlighter.setHandleMultiTermQuery(false);//disable but leave phrase processing enabled

  topDocs = searcher.search(query, 10, Sort.INDEXORDER);
  snippets = highlighter.highlight("body", query, topDocs);

  //note: without MTQ, the WEIGHT_MATCHES is disabled which affects the snippet boundaries
  assertArrayEquals(new String[]{"<b>alpha</b> <b>bravo</b> <b>charlie</b> - charlie bravo alpha"},
      snippets);
}
 

public void testSpanNot() throws Exception {
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);

  Field body = new Field("body", "", fieldType);
  Document doc = new Document();
  doc.add(body);

  body.setStringValue("This is a test.");
  iw.addDocument(doc);
  body.setStringValue("Test a one sentence document.");
  iw.addDocument(doc);

  IndexReader ir = iw.getReader();
  iw.close();

  IndexSearcher searcher = newSearcher(ir);
  UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
  SpanQuery include = new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*")));
  SpanQuery exclude = new SpanTermQuery(new Term("body", "bogus"));
  Query query = new SpanNotQuery(include, exclude);
  TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
  assertEquals(2, topDocs.totalHits.value);
  String snippets[] = highlighter.highlight("body", query, topDocs);
  assertEquals(2, snippets.length);
  assertEquals("This is a <b>test</b>.", snippets[0]);
  assertEquals("<b>Test</b> a one sentence document.", snippets[1]);

  ir.close();
}
 

private static Query[] getPrebuiltQueries(String field) {
  //  be wary of unanalyzed text
  return new Query[] {
      new SpanFirstQuery(new SpanTermQuery(new Term(field, "ford")), 5),
      new SpanNearQuery(new SpanQuery[]{new SpanTermQuery(new Term(field, "night")), new SpanTermQuery(new Term(field, "trading"))}, 4, false),
      new SpanNearQuery(new SpanQuery[]{new SpanFirstQuery(new SpanTermQuery(new Term(field, "ford")), 10), new SpanTermQuery(new Term(field, "credit"))}, 10, false),
      new WildcardQuery(new Term(field, "fo*")),
  };
}