org.eclipse.jface.text.rules.Token#EOF源码实例Demo

@Override
public IToken nextToken() {
	if (currentTemplateTextToken != null) {
		if (currentTemplateTextToken.hasNext())
			return currentTemplateTextToken.nextToken();
		else
			currentTemplateTextToken = null;
	}
	if (!getIterator().hasNext())
		return Token.EOF;
	ILexerTokenRegion next = getIterator().next();
	int tokenType = next.getLexerTokenType();
	switch (tokenType) {
	case RULE_TEMPLATE_HEAD:
	case RULE_TEMPLATE_MIDDLE: {
		currentTemplateTextToken = createTemplateToken(tokenType, next);
		return currentTemplateTextToken.nextToken();
	}
	default:
		setCurrentToken(next);
		return createToken(next);
	}
}
 

@Override
public IToken nextToken() {
	if (nextToken==null) {
		return Token.EOF;
	}
	currentPartitionOffset = nextToken.getOffset();
	currentPartitionLength = nextToken.getLength();
	String tokenPartition = mapper.getPartitionType(nextToken.getLexerTokenType());
	while (tokens.hasNext()) {
		nextToken = tokens.next();
		String partitionOfNext = mapper.getPartitionType(nextToken.getLexerTokenType());
		currentPartitionLength = nextToken.getOffset()-currentPartitionOffset;
		if (!partitionOfNext.equals(tokenPartition) || !shouldMergePartitions(tokenPartition)) {
			return new Token(tokenPartition);
		}
	}
	if (nextToken != null)
		currentPartitionLength = nextToken.getOffset() + nextToken.getLength() - currentPartitionOffset; 
	nextToken = null;
	return new Token(tokenPartition);
}
 

@Override
public IToken nextToken() {
	if (currentRichTextToken != null) {
		if (currentRichTextToken.hasNext())
			return currentRichTextToken.nextToken();
		else
			currentRichTextToken = null;
	}
	if (!getIterator().hasNext())
		return Token.EOF;
	ILexerTokenRegion next = getIterator().next();
	int tokenType = next.getLexerTokenType();
	if (tokenType >= 0 && allTokenTypesAsString[tokenType] != null) {
		currentRichTextToken = createRichTextToken(allTokenTypesAsString[tokenType], next);
		return currentRichTextToken.nextToken();
	} else {
		setCurrentToken(next);
		return createToken(next);	
	}
}
 

public IToken nextToken() {
IToken token;
tokenOffset = offset;
column = UNDEFINED;
Iterator<?> ruleIterator = rules.iterator();
while(ruleIterator.hasNext()) {
    IRule rule = (IRule)ruleIterator.next();
    token = rule.evaluate(this);
    if(!token.isUndefined()) {
        return token;
    }
    if(ruleIterator.hasNext()) {
    	offset = tokenOffset;
    }
}
if(rules.size() <= 0) {
	read();
}
if (read() == EOF) {
	return Token.EOF;
} else {
	unread();
	return defaultToken;
}
  }
 

public IToken nextToken()
{
	if (tokenScanner != null)
	{
		return tokenScanner.nextToken();
	}

	// return value then reset to EOF
	try
	{
		return token;
	}
	finally
	{
		token = Token.EOF;
	}
}
 

protected IToken doNextToken() {
	fTokenOffset= fOffset;
	fColumn= UNDEFINED;

	for(int i = 0; i < fRules.length; i++) {
		fOffset = fTokenOffset; // Revert position changes that rules might have changed
		
		IToken token = fRules[i].evaluate(this);
		if(!token.isUndefined()) {
			return token; 
		}
	}
	
	if (read() == EOF)
		return Token.EOF;
	return fDefaultReturnToken;
}
 

public static void configure(TextLayout textLayout) {
	String text = textLayout.getText();
	Document doc = new Document(text);
	ITokenScanner scanner = getRecipeScanner(doc);
	scanner.setRange(doc, 0, doc.getLength());
	IToken token;
	while ((token = scanner.nextToken()) != Token.EOF) {
		int offset = scanner.getTokenOffset();
		int length = scanner.getTokenLength();
		Object data = token.getData();
		if (data != null && data instanceof TextStyle) {
			TextStyle textStyle = (TextStyle) data;
			textLayout.setStyle(textStyle, offset, offset + length - 1);
		}
	}
}
 

public static void configure(TextLayout textLayout) {
	String text = textLayout.getText();
	Document doc = new Document(text);
	ITokenScanner scanner = getRecipeScanner(doc);
	scanner.setRange(doc, 0, doc.getLength());
	IToken token;
	while ((token = scanner.nextToken()) != Token.EOF) {
		int offset = scanner.getTokenOffset();
		int length = scanner.getTokenLength();
		Object data = token.getData();
		if (data != null && data instanceof TextStyle) {
			TextStyle textStyle = (TextStyle) data;
			textLayout.setStyle(textStyle, offset, offset + length - 1);
		}
	}
	scanner = null;
	doc = null;
}
 

public static void configure(TextLayout textLayout) {
	String text = textLayout.getText();
	Document doc = new Document(text);
	ITokenScanner scanner = getRecipeScanner(doc);
	scanner.setRange(doc, 0, doc.getLength());
	IToken token;
	while ((token = scanner.nextToken()) != Token.EOF) {
		int offset = scanner.getTokenOffset();
		int length = scanner.getTokenLength();
		Object data = token.getData();
		if (data != null && data instanceof TextStyle) {
			TextStyle textStyle = (TextStyle) data;
			textLayout.setStyle(textStyle, offset, offset + length - 1);
		}
	}
}
 

public static void configure(TextLayout textLayout) {
	String text = textLayout.getText();
	Document doc = new Document(text);
	ITokenScanner scanner = getRecipeScanner(doc);
	scanner.setRange(doc, 0, doc.getLength());
	IToken token;
	while ((token = scanner.nextToken()) != Token.EOF) {
		int offset = scanner.getTokenOffset();
		int length = scanner.getTokenLength();
		Object data = token.getData();
		if (data != null && data instanceof TextStyle) {
			TextStyle textStyle = (TextStyle) data;
			textLayout.setStyle(textStyle, offset, offset + length - 1);
		}
	}
}
 

public static void configure(TextLayout textLayout) {
	String text = textLayout.getText();
	Document doc = new Document(text);
	ITokenScanner scanner = getRecipeScanner(doc);
	scanner.setRange(doc, 0, doc.getLength());
	IToken token;
	while ((token = scanner.nextToken()) != Token.EOF) {
		int offset = scanner.getTokenOffset();
		int length = scanner.getTokenLength();
		Object data = token.getData();
		if (data != null && data instanceof TextStyle) {
			TextStyle textStyle = (TextStyle) data;
			textLayout.setStyle(textStyle, offset, offset + length - 1);
		}
	}
	scanner = null;
	doc = null;
}
 

@Override
public IToken nextToken() {
	while (true) {
		lastScannedIndex++;
		if (lastScannedIndex >= concepts.size())
			return Token.EOF;

		SyntaxConcept concept = getLastConcept();
		TextAttribute attr = ClassificationFormat.getTextAttribute(concept);
		if (attr != null) {
			Logger.debug("token: '" + concept.type + "' @" + concept.line + "," + concept.column + "; value: "
					+ concept.value);
			return new Token(attr);
		} else {
			Logger.debug("ignored: '" + concept.type + "' @" + concept.line + "," + concept.column + "; value: "
					+ concept.value);
		}
	}
}
 

@Override
public IToken nextToken() {
	if (!tokenReturned) {
		tokenReturned = true;
		return new Token(getAttribute(HighlightingStyles.DEFAULT_ID));
	}
	return Token.EOF;
}
 

@Override
public IToken nextToken() {
	if (!reentrantIterator.hasNext())
		return Token.EOF;
	currentToken = reentrantIterator.next();
	return createToken(currentToken);
}
 

@Override
public IToken nextToken() {
    try {
        IToken returnToken;
        if (token == null) {
            returnToken = tokenManager.createFrom(flex.nextToken());
            tokenOffset = rangeOffset + flex.getTokenOffset();
            tokenLength = flex.yylength();
            
            if (returnToken == null) {
                token = tokenManager.createFrom(flex.nextToken());
                while (token == null) {
                    tokenLength += flex.yylength();
                    token = tokenManager.createFrom(flex.nextToken());
                }
                returnToken = defaultReturnToken;
            }
        } else {
            returnToken = token;
            tokenOffset = rangeOffset + flex.getTokenOffset();
            tokenLength = flex.yylength();
            token = null; 
        }
        return returnToken;
    }
    catch (IOException e) { /*Can't happen*/ 
    }
    return Token.EOF;
}
 

@Override
 public IToken nextToken() {
     try {
         TokenDescriptor tokenDesc = flex.nextToken();
return tokenManager.createFrom(tokenDesc);
     }
     catch (IOException e) { 
         LogHelper.logError(e); 
     }
     return Token.EOF;
 }
 

public IToken evaluate(ICharacterScanner scanner, boolean resume)
{
	// Should we try to match the first char first?
	if (firstChar != Character.MIN_VALUE)
	{
		int readChar = scanner.read();
		scanner.unread();
		if (readChar == ICharacterScanner.EOF)
		{
			return Token.EOF;
		}
		if (firstChar != readChar)
		{
			return Token.UNDEFINED;
		}
	}
	String line = readNextLine(scanner);
	if (line == null)
		return Token.EOF;
	Matcher matcher = regexp.matcher(line);
	if (matcher.find() && matcher.start() == 0)
	{
		// Unread back to end of regexp match!
		String match = matcher.group();
		if (match.length() < line.length())
		{
			int toUnread = line.length() - match.length();
			unread(scanner, toUnread);
		}
		return successToken;
	}
	unread(scanner, line.length());
	return Token.UNDEFINED;
}
 

public IToken nextToken() {
	current = queue.poll();
	if (current != null) {
		return current.token;
	}
	return Token.EOF;
}
 

/**
 * Used to determine whether there is any text after the current offset
 * within the same partition, excluding the current word Also returns true
 * if there is no white
 */
private boolean useContractedElementCompletion( int documentOffset,
        IDocument document){
    
    boolean textReached = false;
    boolean isRemainingWhiteSpace = true;
    
    try {
        ITypedRegion region = document.getPartition(documentOffset);
        
        int partitionOffset = region.getOffset();
        int partitionLength = region.getLength();
        
        int readLength = documentOffset - partitionOffset;
        int remainingLength = partitionLength - readLength;
        
        if( document.getLength() >= documentOffset + 1) {
            String firstTwo = document.get(partitionOffset, 2);
            if( firstTwo.equals("<<"))
                return false;
        }
        
        scanner.setRange(document, documentOffset, remainingLength);
        
        IToken token = null;
        while ((token = scanner.nextToken()) != Token.WHITESPACE
                && token != Token.EOF) {
            isRemainingWhiteSpace = false;
            continue;
        }
        
        while ((token = scanner.nextToken()) == Token.WHITESPACE
                && token != Token.EOF) {
            isRemainingWhiteSpace = true;
            continue;
        }
        
        char c = (char) 0;
        
        while ((c == scanner.read())) {
            if( c == XMLTagScanner.EOF)
                break;
            if( c == '<') {
                break;
            }
            if( !Character.isWhitespace(c))
                textReached = true;
            
        }
        
    } catch (BadLocationException e) {
       ExceptionHandler.handle(e);
    }
    
    if( textReached)
        return true;
    if( !isRemainingWhiteSpace && !textReached)
        return true;
    else
        return false;
    
}
 

@Override
protected IToken mapToken(Symbol symbol) throws IOException, Exception
{
	CSSTokenTypeSymbol tokenTypeSymbol = (CSSTokenTypeSymbol) symbol;
	CSSTokenType tokenData;
	// System.out.println(symbol.value + " - " + symbol.getId());
	switch (tokenTypeSymbol.token)
	{
		case IDENTIFIER:
			if (HTML_TAGS.contains(symbol.value))
			{
				tokenData = CSSTokenType.ELEMENT;
			}
			else if (FUNCTIONS.contains(symbol.value))
			{
				tokenData = CSSTokenType.FUNCTION;
			}
			else if (PROPERTY_VALUES.contains(symbol.value))
			{
				tokenData = CSSTokenType.VALUE;
			}
			else if (STANDARD_COLORS.contains(symbol.value))
			{
				tokenData = CSSTokenType.COLOR;
			}
			else if (DEPRECATED_COLORS.contains(symbol.value))
			{
				tokenData = CSSTokenType.DEPRECATED_COLOR;
			}
			else if (PROPERTY_NAMES.contains(symbol.value))
			{
				tokenData = CSSTokenType.PROPERTY;
			}
			else if (MEDIA.contains(symbol.value))
			{
				tokenData = CSSTokenType.MEDIA;
			}

			// slower ones as last
			else if (FONT_NAMES.contains(((String) symbol.value).toLowerCase()))
			{
				tokenData = CSSTokenType.FONT;
			}
			else if (CSSCodeScannerRuleBased.VENDOR_WORD_RULE.wordOK((String) symbol.value, null))
			{
				tokenData = CSSTokenType.PROPERTY;
			}
			else
			{
				tokenData = CSSTokenType.IDENTIFIER;
			}
			break;

		case EOF:
			return Token.EOF;

		default:
			tokenData = tokenTypeSymbol.token;

	}
	return makeTokenWithContext(tokenData, false);
}