public IToken evaluate( ICharacterScanner scanner){

      buffer.setLength(0);

      charsRead = 0;
      int c = read(scanner);

      if (c == matchString.charAt(0)) {
         do {
            c = read(scanner);
         } while (isOK((char) c));

         if (charsRead == matchString.length()) {
            return fToken;
         } else {
            rewind(scanner);
            return Token.UNDEFINED;
         }

      }

      scanner.unread();
      return Token.UNDEFINED;
   }
 

protected boolean sequenceDetected( ICharacterScanner scanner, char[] sequence, boolean eofAllowed){
   int c = scanner.read();
   if (sequence[0] == '<') {
      if (c == '?') {
         // processing instruction - abort
         scanner.unread();
         return false;
      }
      if (c == '!') {
         scanner.unread();
         // comment - abort
         return false;
      }
   } else if (sequence[0] == '>') {
      scanner.unread();
   }
   return super.sequenceDetected(scanner, sequence, eofAllowed);
}
 

private String readNextLine(ICharacterScanner scanner)
{
	char[][] lineDelims = TextUtils.rsort(scanner.getLegalLineDelimiters());
	StringBuilder builder = new StringBuilder();
	int c;
	while ((c = scanner.read()) != ICharacterScanner.EOF)
	{
		// FIXME We need to properly handle multi-char line delims!
		if (isLineDelim((char) c, lineDelims))
		{
			break;
		}
		builder.append((char) c);
	}
	if (c == ICharacterScanner.EOF && builder.length() == 0)
		return null;
	scanner.unread();
	return builder.toString();
}
 

/**
 * Does the actual evaluation of the stream.
 * 
 * @param scanner The scanner
 * @param count The initial count of {
 * @return <code>fToken</code> on success, <code>Token.UNDEFINED</code> if
 * the match doesn't succeed
 */
private IToken doEvaluate(ICharacterScanner scanner, int count) {
    boolean inString = false;
    int c = scanner.read();

    if (((char) c) == '{') {
        do {
            c = scanner.read();
            if (((char) c) == '{' && !inString)
                count++;
            else if (((char) c) == '}' && !inString)
                count--;
            else if (((char) c) == '"' && !inString)
                inString = true;
            else if (((char) c) == '"' && inString)
                inString = false;
            else if (c == ICharacterScanner.EOF)
            	return Token.UNDEFINED;
        } while (count > 0);
        return fToken;
    }
    scanner.unread();
    return Token.UNDEFINED;
}
 

public IToken evaluate(ICharacterScanner scanner, boolean resume) {
    if (resume) {
        boolean inString = false;
        do {
            int c = scanner.read();
            if (((char) c) == ',' && !inString)
                break;
            else if (((char) c) == '@') {
                scanner.unread();
                return Token.UNDEFINED;
            } else if (((char) c) == '"' && !inString)
                inString = true;
            else if (((char) c) == '"' && inString)
                inString = false;
            else if (c == ICharacterScanner.EOF)
                return Token.UNDEFINED;
        } while (true);
    }
    return doEvaluate(scanner, 1);
}
 

protected void runRuleTest(String text, Integer[] tokenIndexes, Integer[] tokenLen) {
	StringCharacterScanner scanner = new StringCharacterScanner(text);
	 
	FullPatternRule fpRule = new FullPatternRule(new Token(null), sequences, new SampleJavaWordDetector());
	 
	while (scanner.peekNext() != ICharacterScanner.EOF) {
		int beginOffset = scanner.textOffset;
		IToken token = fpRule.evaluate(scanner);
		if(token.isUndefined()) {
			assertTrue(scanner.textOffset == beginOffset);
			assertTrue(ArrayUtil.contains(tokenIndexes, beginOffset) == false);
			scanner.read(); // advance
		} else {
			int indexOf = ArrayUtil.indexOf(tokenIndexes, beginOffset);
			assertTrue(indexOf != -1);
			assertTrue(scanner.textOffset == beginOffset + tokenLen[indexOf]);
		}
	}
}
 

public IToken evaluate(ICharacterScanner scanner) {
    int c = scanner.read();
    if ((c != ICharacterScanner.EOF) && wordDetector.isWordStart((char)c)) {
        readLength = 1;
        
        do {
            c = scanner.read();
            readLength++;
        } while ((c != ICharacterScanner.EOF) && wordDetector.isWordPart((char)c));
        for (char ch: endCharacters) {
            if (c == ch) {
                return successToken;
            }
        }
        
        while (readLength > 1) {
            readLength--;
            scanner.unread();
        }
    }
    scanner.unread();
    return Token.UNDEFINED;
}
 

@Override
public boolean wordOK(String word, ICharacterScanner scanner)
{
	// Table 1. Vendor Extension Prefixes
	// Prefix Organisation
	// -ms- Microsoft
	// mso- Microsoft Office
	// -moz- Mozilla Foundation (Gecko-based browsers)
	// -o- Opera Software
	// -atsc- Advanced Television Standards Committee
	// -wap- The WAP Forum
	// -webkit- Safari (and other WebKit-based browsers)
	// -khtml-

	return word.startsWith("-moz-") || word.startsWith("-webkit-") || word.startsWith("-ms-")
			|| word.startsWith("-o-") || word.startsWith("-atsc-") || word.startsWith("-khtml-")
			|| word.startsWith("-wap-");
}
 

protected boolean endSequenceDetected(ICharacterScanner scanner) {
    int c;
    while ((c = scanner.read()) != ICharacterScanner.EOF) {
        if (c == fEscapeCharacter) {
            // Skip the escaped character.
            scanner.read();
        } else if (c == '<') {
        	fEmbeddedStart++;
        } else if (c == '>') {
        	if (fEmbeddedStart == 0) {
        		return true;
        	}
        	fEmbeddedStart--;
        }
    }
    return true;
}
 

@Override
public IToken evaluate(ICharacterScanner scanner) {
	int ch = scanner.read();
	if (ch == '<') {
		ch = scanner.read();
		if (ch == '/') {
			return token;
		}
		scanner.unread();
		return token;
	} else if (ch == '>') {
		ch = scanner.read();
		if (ch == '/') {
			return token;
		}
		scanner.unread();
		return token;
	}
	scanner.unread();
	return Token.UNDEFINED;
}
 

@Override
protected boolean endSequenceDetected(ICharacterScanner scanner) {
  int c = 0, readCount = 1;
  while ((c = scanner.read()) != ICharacterScanner.EOF) {
    if (!isValidJavaRefCharacter((char) c)) {
      scanner.unread();
      return true;
    }
    readCount++;
  }

  while (--readCount > 0) {
    scanner.unread();
  }

  return super.endSequenceDetected(scanner);
}
 

@Override
protected boolean endSequenceDetected(ICharacterScanner scanner) {
	int c;
	int embeddedDTD = 0;
	CollectingCharacterScanner collectingCharacterScanner = new CollectingCharacterScanner(scanner, String.valueOf(fStartSequence));
	while ((c = collectingCharacterScanner.read()) != ICharacterScanner.EOF) {
		if (c == fEscapeCharacter) {
			// Skip the escaped character.
			collectingCharacterScanner.read();
		} else if (c == '[') {
			if (breakOnDTD) {
				break;
			}
			++embeddedDTD;
		} else if (c == ']') {
			--embeddedDTD;
		} else if (c == '>' && embeddedDTD <= 0) {
			break;
		}
	}
	if (fToken instanceof ExtendedToken) {
		((ExtendedToken) fToken).setContents(collectingCharacterScanner.getContents());
	}
	return true;
}
 

private boolean readInterface(ICharacterScanner scanner) {
	int ch= scanner.read();
	int i= 0;
	while (i < INTERFACE.length() && INTERFACE.charAt(i) == ch) {
		i++;
		ch= scanner.read();
	}
	if (i < INTERFACE.length())
		return false;

	if (fWordDetector.isWordPart((char) ch))
		return false;

	if (ch != ICharacterScanner.EOF)
		scanner.unread();

	return true;
}
 

public static boolean sequenceDetected(ICharacterScanner characterScanner, char[] sequence, boolean ignoreCase) {
	for (int i = 1; i < sequence.length; ++i) {
		int c = characterScanner.read();
		if ((ignoreCase && Character.toLowerCase(c) != Character.toLowerCase(sequence[i])) || (!ignoreCase && c != sequence[i])) {
			// Non-matching character detected, rewind the scanner back to the start.
			// Do not unread the first character.
			characterScanner.unread();
			for (int j = i - 1; j > 0; --j) {
				characterScanner.unread();
			}
			return false;
		}
	}
	for (int j = sequence.length - 1; j > 0; --j) {
		characterScanner.unread();
	}
	return true;
}
 

private void rewind( ICharacterScanner scanner){
   int rewindLength = charsRead;
   while (rewindLength > 0) {
      scanner.unread();
      rewindLength--;
   }
}
 

public IToken evaluate(ICharacterScanner scanner) {
	int c= scanner.read();
	if (fDetector.isWordStart((char) c)) {
		if (fColumn == UNDEFINED || (fColumn == scanner.getColumn() - 1)) {

			fBuffer.clear();
			do {
				fBuffer.append((char) c);
				c= scanner.read();
			} while (c != ICharacterScanner.EOF && fDetector.isWordPart((char) c));
			scanner.unread();

			for (int i= 0, n= fMatchers.size(); i < n; i++) {
				IToken token= fMatchers.get(i).evaluate(scanner, fBuffer);
				if (!token.isUndefined())
					return token;
			}

			if (fDefaultToken.isUndefined())
				unreadBuffer(scanner);

			return fDefaultToken;
		}
	}

	scanner.unread();
	return Token.UNDEFINED;
}
 

public IToken evaluate(ICharacterScanner scanner) {
	if (scanner instanceof SequenceCharacterScanner) {
		// when checking for the rule, do not search for potential sequence
		SequenceCharacterScanner seqScanner = (SequenceCharacterScanner) scanner;
		try {
			seqScanner.setSequenceIgnored(true);
			return rule.evaluate(scanner);
		} finally {
			seqScanner.setSequenceIgnored(false);
		}
	}
	return rule.evaluate(scanner);
}
 

public IToken evaluate(ICharacterScanner scanner) {
    int c = scanner.read();
    if (((char) c) == '=' || ((char) c) == '#' || ((char) c) == ','
        || ((char) c) == '{' || ((char) c) == '}') {
        return fToken;
    } else if (((char) c) == '\\') {
        c = scanner.read();
        if (((char) c) == '"')
            return fToken;
        scanner.unread();
    }
    scanner.unread();
    return Token.UNDEFINED;
}
 

public IToken evaluate(ICharacterScanner scanner) {
	if (!fIsVersionMatch)
		return Token.UNDEFINED;

	ResettableScanner resettable= new ResettableScanner(scanner);
	if (resettable.read() == '@')
		return readAnnotation(resettable);

	resettable.reset();
	return Token.UNDEFINED;
}
 

private int ignoreComment() {
    int ch = fScanner.read();
    int r=1;
    while (ch != '\r' && ch != '\n' && ch != ICharacterScanner.EOF) {
        ch = fScanner.read();
        r++;
    }
    return r;
}
 

private IToken checkForEnv() {
    int o = checkForCommand(BEGIN, 1);
    if (o == 0) {
        fTokenLength += 2;
        return fTokens[TEX];
    }
    int offsetEnd = fTokenOffset;
    offsetEnd += 6;
    int ch = fScanner.read();
    offsetEnd++;
    while (Character.isWhitespace(ch)) {
        ch = fScanner.read();
        offsetEnd++;
    }
    if (ch != '{'){
        unReadScanner(offsetEnd - fTokenOffset - 2);
        fTokenLength += 2;
        return fTokens[TEX];
    }
    final StringBuilder b = new StringBuilder();
    ch = fScanner.read();
    offsetEnd++;
    while (ch != '}' && ch != ICharacterScanner.EOF && ch != '{' && ch != '\\'){
        b.append((char)ch);
        ch = fScanner.read();
        offsetEnd++;
    }
    String envName = b.toString();
    if (getEnvIndex(envName) != TEX) {
        return checkForEndEnv(envName, offsetEnd);
    }
    else {
        unReadScanner(offsetEnd - fTokenOffset - 2);
        fTokenLength += 2;
        return fTokens[TEX];
    }
}
 

@Override
protected boolean wordOK(String word, ICharacterScanner scanner)
{
	if (pattern == null)
	{
		pattern = Pattern.compile("<(/)?"); //$NON-NLS-1$
	}
	return pattern.matcher(word).matches();
}
 

/**
 * Tests if the current character is '\' character. If it is
 * calls test for next character.  
 * @see org.eclipse.jface.text.rules.IPredicateRule#evaluate(org.eclipse.jface.text.rules.ICharacterScanner, boolean)
 * @param scanner 	the scanner to read characters
 * @param resume	shall method start from next character
 * @return 			the success token if "\X" (X is one of the predefined 
 * 					characters) matches, Token.UNDEFINED otherwise
 */
public IToken evaluate(ICharacterScanner scanner, boolean resume) {
	
	if(resume){
		if (evaluateNext(scanner)){
			return successToken;
		}else{
			return Token.UNDEFINED;				
		}				
	}else{
		return evaluate(scanner);
	}
}
 

public IToken evaluate(ICharacterScanner scanner) {

			int character= scanner.read();
			if (isOperator((char) character)) {
				do {
					character= scanner.read();
				} while (isOperator((char) character));
				scanner.unread();
				return fToken;
			} else {
				scanner.unread();
				return Token.UNDEFINED;
			}
		}
 

protected boolean wordOK(String word, ICharacterScanner scanner)
{
	if (pattern == null)
	{
		pattern = Pattern.compile("[-+]?\\s*[0-9]+(\\.[0-9]+)?"); //$NON-NLS-1$
	}

	return pattern.matcher(word).matches();
}
 

protected IToken currentOr(char altA, char altB, ICharacterScanner scanner) {
	int second = scanner.read();
	if(second == altA || second == altB) {
		return getSuccessToken();
	}
	
	scanner.unread(); 
	return getSuccessToken();
}
 

public IToken evaluate(ICharacterScanner scanner) {
    int c = scanner.read();
    if ((c != ICharacterScanner.EOF) && Character.isDigit((char)c)) {
        readLength = 1;
        
        do {
            c = scanner.read();
            readLength++;
        } while ((c != ICharacterScanner.EOF) && Character.isDigit((char)c));
        if (c == '.') {
            do {
                c = scanner.read();
            } while ((c != ICharacterScanner.EOF) && Character.isDigit((char)c));
            if (c == 'E') {
                c = scanner.read();
                if ((c == '+') || (c == '-')) {
                    c = scanner.read();
                }
                while ((c != ICharacterScanner.EOF) && Character.isDigit((char)c)) {
                    c = scanner.read();
                }
            }
            scanner.unread();
            return successToken;
        }
        
        while (readLength > 1) {
            readLength--;
            scanner.unread();
        }
    }
    scanner.unread();
    return Token.UNDEFINED;
}
 

@Override
public IToken evaluate(ICharacterScanner scanner) {
	IToken result= super.evaluate(scanner);
	if (result == fToken)
		return evaluateToken();
	return result;
}
 

protected boolean wordOK(String word, ICharacterScanner scanner)
{
	if (pattern == null)
	{
		pattern = Pattern.compile("!\\s*important", Pattern.CASE_INSENSITIVE); //$NON-NLS-1$
	}

	return pattern.matcher(word).matches();
}
 

private void unread(ICharacterScanner scanner, int toUnread)
{
	while (toUnread > 0)
	{
		scanner.unread();
		toUnread--;
	}
}