下面列出了org.eclipse.jface.text.rules.RuleBasedScanner#org.eclipse.jface.text.rules.Token 实例代码，或者点击链接到github查看源代码，也可以在右侧发表评论。
public static void configure(TextLayout textLayout) {
String text = textLayout.getText();
Document doc = new Document(text);
ITokenScanner scanner = getRecipeScanner(doc);
scanner.setRange(doc, 0, doc.getLength());
IToken token;
while ((token = scanner.nextToken()) != Token.EOF) {
int offset = scanner.getTokenOffset();
int length = scanner.getTokenLength();
Object data = token.getData();
if (data != null && data instanceof TextStyle) {
TextStyle textStyle = (TextStyle) data;
textLayout.setStyle(textStyle, offset, offset + length - 1);
}
}
scanner = null;
doc = null;
}
public static void configure(TextLayout textLayout) {
String text = textLayout.getText();
Document doc = new Document(text);
ITokenScanner scanner = getRecipeScanner(doc);
scanner.setRange(doc, 0, doc.getLength());
IToken token;
while ((token = scanner.nextToken()) != Token.EOF) {
int offset = scanner.getTokenOffset();
int length = scanner.getTokenLength();
Object data = token.getData();
if (data != null && data instanceof TextStyle) {
TextStyle textStyle = (TextStyle) data;
textLayout.setStyle(textStyle, offset, offset + length - 1);
}
}
}
public XMLPartitionScanner() {
IToken xmlComment = new Token(XML_COMMENT);
IToken xmlPI = new Token(XML_PI);
IToken startTag = new Token(XML_START_TAG);
IToken endTag = new Token(XML_END_TAG);
IToken docType = new Token(XML_DOCTYPE);
IToken text = new Token(XML_TEXT);
IPredicateRule[] rules = new IPredicateRule[7];
rules[0] = new NonMatchingRule();
rules[1] = new MultiLineRule("<!--", "-->", xmlComment);
rules[2] = new MultiLineRule("<?", "?>", xmlPI);
rules[3] = new MultiLineRule("</", ">", endTag);
rules[4] = new StartTagRule(startTag);
rules[5] = new MultiLineRule("<!DOCTYPE", ">", docType);
rules[6] = new XMLTextPredicateRule(text);
setPredicateRules(rules);
}
@Test public void testWholePart() throws Exception {
PartitionTokenScanner scanner = getPartitionTokenScanner(t(2, 3),t(4,3),t(2,1),t(34,3));
scanner.setPartialRange(null, 0, 42, "3", 0);
assertEquals("3",scanner.nextToken().getData());
assertEquals(0,scanner.getTokenOffset());
assertEquals(6,scanner.getTokenLength());
assertEquals("1",scanner.nextToken().getData());
assertEquals(6,scanner.getTokenOffset());
assertEquals(2,scanner.getTokenLength());
assertEquals("3",scanner.nextToken().getData());
assertEquals(8,scanner.getTokenOffset());
assertEquals(34,scanner.getTokenLength());
assertSame(Token.EOF, scanner.nextToken());
}
/**
*
*/
public CompositePartitionScanner(ISubPartitionScanner defaultPartitionScanner, ISubPartitionScanner primaryPartitionScanner,
IPartitionerSwitchStrategy partitionerSwitchStrategy) {
this.defaultPartitionScanner = defaultPartitionScanner;
this.primaryPartitionScanner = primaryPartitionScanner;
defaultPartitionScanner.initCharacterScanner(this, partitionerSwitchStrategy.getDefaultSwitchStrategy());
primaryPartitionScanner.initCharacterScanner(this, partitionerSwitchStrategy.getPrimarySwitchStrategy());
String[][] pairs = partitionerSwitchStrategy.getSwitchTagPairs();
switchRules = new IPredicateRule[pairs.length][];
for (int i = 0; i < pairs.length; ++i) {
switchRules[i] = new IPredicateRule[] { new SingleTagRule(pairs[i][0], new Token(START_SWITCH_TAG)), new SingleTagRule(pairs[i][1], new Token(END_SWITCH_TAG)) };
}
currentPartitionScanner = defaultPartitionScanner;
setDefaultReturnToken(new Token(IDocument.DEFAULT_CONTENT_TYPE));
}
private void adaptToStyleChange(Token token, PropertyChangeEvent event, int styleAttribute) {
boolean eventValue = false;
Object value = event.getNewValue();
if (value instanceof Boolean)
eventValue = ((Boolean) value).booleanValue();
else if (IPreferenceStore.TRUE.equals(value))
eventValue = true;
Object data = token.getData();
if (data instanceof TextAttribute) {
TextAttribute oldAttr = (TextAttribute) data;
boolean activeValue = (oldAttr.getStyle() & styleAttribute) == styleAttribute;
if (activeValue != eventValue)
token.setData(new TextAttribute(oldAttr.getForeground(), oldAttr.getBackground(),
eventValue ? oldAttr.getStyle() | styleAttribute : oldAttr.getStyle() & ~styleAttribute));
}
}
@Override
public IToken nextToken() {
while (true) {
lastScannedIndex++;
if (lastScannedIndex >= concepts.size())
return Token.EOF;
SyntaxConcept concept = getLastConcept();
TextAttribute attr = ClassificationFormat.getTextAttribute(concept);
if (attr != null) {
Logger.debug("token: '" + concept.type + "' @" + concept.line + "," + concept.column + "; value: "
+ concept.value);
return new Token(attr);
} else {
Logger.debug("ignored: '" + concept.type + "' @" + concept.line + "," + concept.column + "; value: "
+ concept.value);
}
}
}
/**
* createLexemeProvider
*
* @param document
* @param offset
* @return
*/
ILexemeProvider<XMLTokenType> createLexemeProvider(IDocument document, int offset)
{
int documentLength = document.getLength();
// account for last position returning an empty IDocument default partition
int lexemeProviderOffset = (offset >= documentLength) ? documentLength - 1 : offset;
return new XMLLexemeProvider(document, lexemeProviderOffset, new XMLTagScanner()
{
@Override
protected IToken createToken(XMLTokenType type)
{
return new Token(type);
}
});
}
private void adaptToStyleChange(Token token, PropertyChangeEvent event, int styleAttribute) {
boolean eventValue= false;
Object value= event.getNewValue();
if (value instanceof Boolean)
eventValue= ((Boolean) value).booleanValue();
else if (IPreferenceStore.TRUE.equals(value))
eventValue= true;
Object data= token.getData();
if (data instanceof TextAttribute) {
TextAttribute oldAttr= (TextAttribute) data;
boolean activeValue= (oldAttr.getStyle() & styleAttribute) == styleAttribute;
if (activeValue != eventValue)
token.setData(new TextAttribute(oldAttr.getForeground(), oldAttr.getBackground(), eventValue ? oldAttr.getStyle() | styleAttribute : oldAttr.getStyle() & ~styleAttribute));
}
}
public IToken evaluate(ICharacterScanner scanner, boolean resume) {
if (resume) {
boolean inString = false;
do {
int c = scanner.read();
if (((char) c) == ',' && !inString)
break;
else if (((char) c) == '@') {
scanner.unread();
return Token.UNDEFINED;
} else if (((char) c) == '"' && !inString)
inString = true;
else if (((char) c) == '"' && inString)
inString = false;
else if (c == ICharacterScanner.EOF)
return Token.UNDEFINED;
} while (true);
}
return doEvaluate(scanner, 1);
}
@Override
public IToken evaluate(ICharacterScanner scanner) {
int ch = scanner.read();
if (ch == '<') {
ch = scanner.read();
if (ch == '/') {
return token;
}
scanner.unread();
return token;
} else if (ch == '>') {
ch = scanner.read();
if (ch == '/') {
return token;
}
scanner.unread();
return token;
}
scanner.unread();
return Token.UNDEFINED;
}
/**
* createScannerSpecificRules
*
* @return
*/
protected Collection<? extends IRule> createScannerSpecificRules()
{
List<IRule> rules = new ArrayList<IRule>();
WordRule wordRule = new WordRule(new KeywordIdentifierDetector(), Token.UNDEFINED);
wordRule.addWord("em", createToken(CSSTokenType.EMS));
wordRule.addWord("ex", createToken(CSSTokenType.EXS));
wordRule.addWord("px", createToken(CSSTokenType.LENGTH));
wordRule.addWord("cm", createToken(CSSTokenType.LENGTH));
wordRule.addWord("mm", createToken(CSSTokenType.LENGTH));
wordRule.addWord("in", createToken(CSSTokenType.LENGTH));
wordRule.addWord("pt", createToken(CSSTokenType.LENGTH));
wordRule.addWord("pc", createToken(CSSTokenType.LENGTH));
wordRule.addWord("deg", createToken(CSSTokenType.ANGLE));
wordRule.addWord("rad", createToken(CSSTokenType.ANGLE));
wordRule.addWord("grad", createToken(CSSTokenType.ANGLE));
wordRule.addWord("ms", createToken(CSSTokenType.TIME));
wordRule.addWord("s", createToken(CSSTokenType.TIME));
wordRule.addWord("hz", createToken(CSSTokenType.FREQUENCY));
wordRule.addWord("khz", createToken(CSSTokenType.FREQUENCY));
wordRule.addWord("Hz", createToken(CSSTokenType.FREQUENCY));
wordRule.addWord("kHz", createToken(CSSTokenType.FREQUENCY));
addWordsToRule(wordRule, FUNCTIONS, CSSTokenType.FUNCTION);
rules.add(wordRule);
return rules;
}
/**
* A default constructor.
* @param manager
*/
public TexArgScanner(ColorManager manager) {
IToken commentToken = new Token(new TextAttribute(manager
.getColor(ColorManager.COMMENT),
null,
manager.getStyle(ColorManager.COMMENT_STYLE)));
//Commands are colored in argument color with command styles
IToken commandToken = new Token(
new TextAttribute(
manager.getColor(ColorManager.CURLY_BRACKETS),
null,
manager.getStyle(ColorManager.COMMAND_STYLE)));
List<IRule> rules = new ArrayList<IRule>();
rules.add(new EndOfLineRule("%", commentToken, '\\'));
rules.add(new WhitespaceRule(new WhitespaceDetector()));
rules.add(new WordRule(new TexWord(), commandToken));
IRule[] result = new IRule[rules.size()];
rules.toArray(result);
setRules(result);
}
@Override
public IToken evaluate(ICharacterScanner scanner, boolean resume) {
if (scanner instanceof ITokenScanner) {
if (isShowInactiveCode && !CollectionUtils.isEmpty(inactiveCodeRegions)) {
ITokenScanner tokenScanner = (ITokenScanner) scanner;
ITextRegion reg = findInactiveCodeRegion(tokenScanner.getTokenOffset());
if (reg != null) {
for (int i = tokenScanner.getTokenOffset(); i < reg.getOffset() + reg.getLength(); i++) {
scanner.read();
}
return ModulaPartitionTokens.DISABLED_CODE;
}
}
}
return Token.UNDEFINED;
}
/***
* Add some partition rules common to C-style languages.
* All rules are optional, if an id is null, the rule will not be added.
*/
protected static void addStandardRules(ArrayList2<IPredicateRule> rules,
String lineCommentId, String blockCommentId,
String docLineCommentId, String docBlockCommentId,
String stringId) {
if(docLineCommentId != null) {
rules.add(new PatternRule("///", null, new Token(docLineCommentId), NO_ESCAPE_CHAR, true, true));
}
if(docBlockCommentId != null) {
rules.add(new PatternRule("/**", "*/", new Token(docBlockCommentId), NO_ESCAPE_CHAR, false, true));
}
if(lineCommentId != null) {
rules.add(new PatternRule("//", null, new Token(lineCommentId), NO_ESCAPE_CHAR, true, true));
}
if(blockCommentId != null) {
rules.add(new PatternRule("/*", "*/", new Token(blockCommentId), NO_ESCAPE_CHAR, false, true));
}
if(stringId != null) {
rules.add(new PatternRule("\"", "\"", new Token(stringId), '\\', false, true));
}
}
@Override
protected List<IRule> createRules() {
List<IRule> list= new ArrayList<IRule>();
Token defaultToken= getToken(fDefaultTokenProperty);
List<WordMatcher> matchers= createMatchers();
if (matchers.size() > 0) {
CombinedWordRule combinedWordRule= new CombinedWordRule(new AtJavaIdentifierDetector(), defaultToken);
for (int i= 0, n= matchers.size(); i < n; i++)
combinedWordRule.addWordMatcher(matchers.get(i));
list.add(combinedWordRule);
}
setDefaultReturnToken(defaultToken);
return list;
}
/**
* Creates the partitioner and sets up the appropriate rules.
*/
public SamplePartitionScanner() {
IToken tkString = new Token(LANG_STRING);
IToken tkRawString = new Token(LANG_RAW_STRING);
IToken tkCharacter = new Token(LANG_CHARACTER);
IToken tkSingleComment = new Token(LANG_SINGLE_COMMENT);
IToken tkMultiComment = new Token(LANG_MULTI_COMMENT);
List<IPredicateRule> rules = new ArrayList<IPredicateRule>();
rules.add(new MultiLineRule("`", "`", tkRawString, NO_ESCAPE, true));
rules.add(new MultiLineRule("\"", "\"", tkString, '\\', true));
rules.add(new SingleLineRule("'", "'", tkCharacter, '\\', true));
rules.add(new EndOfLineRule("//", tkSingleComment, NO_ESCAPE));
rules.add(new MultiLineRule("/*", "*/", tkMultiComment, NO_ESCAPE, true));
setPredicateRules(rules.toArray(new IPredicateRule[rules.size()]));
}
/**
* HTMLSubPartitionScanner
*/
public HTMLSubPartitionScanner()
{
// @formatter:off
super(
new ISubPartitionScanner[] {
new SubPartitionScanner(
HTMLSourceConfiguration.getDefault().getPartitioningRules(),
HTMLSourceConfiguration.CONTENT_TYPES,
new Token(HTMLSourceConfiguration.DEFAULT)
),
JSSourceConfiguration.getDefault().createSubPartitionScanner(),
CSSSourceConfiguration.getDefault().createSubPartitionScanner(),
SVGSourceConfiguration.getDefault().createSubPartitionScanner()
},
new IPartitionScannerSwitchStrategy[] {
new PartitionScannerSwitchStrategy(JS_SWITCH_SEQUENCES),
new PartitionScannerSwitchStrategy(CSS_SWITCH_SEQUENCES),
new PartitionScannerSwitchStrategy(SVG_SWITCH_SEQUENCES)
}
);
// @formatter:on
}
protected IToken createToken() {
String attributeId = delimiter
? HighlightingConfiguration.TEMPLATE_DELIMITER_ID
: HighlightingConfiguration.TEMPLATE_ID;
Token token = new Token(getAttribute(attributeId));
return token;
}
private RuleBasedScanner getCommentScanner(){
if (commentScanner == null) {
commentScanner = new HCommentScanner();
commentScanner.setDefaultReturnToken(new Token(new TextAttribute(ResourceUtils.getColor(Styles.COMMENT))));
}
return commentScanner;
}
private RuleBasedScanner getValueScanner(){
if (valueScanner == null) {
valueScanner = new HValueScanner();
valueScanner.setDefaultReturnToken(new Token(new TextAttribute(ResourceUtils.getColor(Styles.STRING))));
}
return valueScanner;
}
protected void setupPresentationReconciler(PresentationReconciler reconciler, ISourceViewer sourceViewer) {
// Must be called from UI thread
assertTrue(Display.getCurrent() != null);
// Create a token registry for given sourceViewer
TokenRegistry tokenRegistry = new TokenRegistry(colorManager, stylingPrefs) {
@Override
protected void handleTokenModified(Token token) {
sourceViewer.invalidateTextPresentation();
}
};
addConfigurationScopedOwned(sourceViewer, tokenRegistry);
ArrayList2<AbstractLangScanner> scanners = new ArrayList2<>();
for(LangPartitionTypes partitionType : getPartitionTypes()) {
String contentType = partitionType.getId();
AbstractLangScanner scanner = createScannerFor(Display.getCurrent(), partitionType, tokenRegistry);
scanners.add(scanner);
DefaultDamagerRepairer dr = getDamagerRepairer(scanner, contentType);
reconciler.setDamager(dr, contentType);
reconciler.setRepairer(dr, contentType);
}
}
protected CDataScanner getCDataScanner(){
if( cdataScanner == null) {
cdataScanner = new CDataScanner(colorManager);
cdataScanner.setDefaultReturnToken(new Token(new TextAttribute(
colorManager.getColor(IXMLColorConstants.CDATA_TEXT))));
}
return cdataScanner;
}
public JavaRegExpRule(LogToolRuleDesc ruleDesc) {
int flags = 0;
find = false;
if (ruleDesc.isCaseInsensitive())
flags = java.util.regex.Pattern.CASE_INSENSITIVE | java.util.regex.Pattern.UNICODE_CASE;
if (ruleDesc.getMatchMode().startsWith("find"))
find = true;
regexp = Pattern.compile(ruleDesc.getRuleValue(),flags);
priority = ruleDesc.getPriority();
successToken = new Token(new TokenData(TextAttributeFactory.getTextAttribute(ruleDesc),priority));
}
public XMLScanner( ColorManager manager) {
IToken procInstr = new Token(new TextAttribute(manager.getColor(IXMLColorConstants.PROC_INSTR)));
IToken docType = new Token(new TextAttribute(manager.getColor(IXMLColorConstants.DOCTYPE)));
IRule[] rules = new IRule[3];
// Add rule for processing instructions and doctype
rules[0] = new MultiLineRule("<?", "?>", procInstr);
rules[1] = new MultiLineRule("<!DOCTYPE", ">", docType);
// Add generic whitespace rule.
rules[2] = new WhitespaceRule(new XMLWhitespaceDetector());
setRules(rules);
}
private IToken computeReturn(CSSTokenType tokenData, boolean isWhitespace)
{
if (isWhitespace)
{
return tokenOnWhitespace;
}
// Grab data again, because we may have changed the token above...
if (tokenData != null)
{
String scope = tokenData.getScope();
int contextLen = fContextToAppend.length();
if (contextLen == 0)
{
return new Token(scope);
}
int scopeLen = scope.length();
// Note: optimized creating string with context+scope as this was a bottleneck when profiling.
char[] array = new char[contextLen + scopeLen];
fContextToAppend.getChars(0, contextLen, array, 0);
scope.getChars(0, scopeLen, array, contextLen);
return new Token(new String(array));
}
else
{
return new Token(CSSTokenType.UNDEFINED.getScope());
}
}
private static IPartitionTokenScanner createDefaultScanner() {
final IToken string = new Token(PatternExpressionViewer.GROOVY_EXPRESSION_CONTENT_TYPE);
final RuleBasedPartitionScanner scanner = new RuleBasedPartitionScanner();
scanner.setPredicateRules(new IPredicateRule[] {
new MultiLineRule(GROOVY_START_TAG, GROOVY_END_TAG, string)
});
return scanner;
}
protected XMLTagScanner getXMLTagScanner() {
if (tagScanner == null) {
tagScanner = new XMLTagScanner(colorManager);
tagScanner.setDefaultReturnToken(
new Token(
new TextAttribute(
colorManager.getColor(IXMLColorConstants.TAG))));
}
return tagScanner;
}
private ITokenScanner getStartEndTokenScanner() {
if (startEndTokenScanner == null) {
RuleBasedScanner ts = new RuleBasedScanner();
IToken seqToken = new Token(getStartEndTokenType());
List<IRule> rules = new ArrayList<IRule>();
for (String[] pair : getPartitionerSwitchStrategy().getSwitchTagPairs()) {
rules.add(new SingleTagRule(pair[0], seqToken));
rules.add(new SingleTagRule(pair[1], seqToken));
}
ts.setRules(rules.toArray(new IRule[rules.size()]));
ts.setDefaultReturnToken(new Token("text")); //$NON-NLS-1$
startEndTokenScanner = ts;
}
return startEndTokenScanner;
}
public void createPresentation(TextPresentation presentation, ITypedRegion region) {
if (fScanner == null) {
// will be removed if deprecated constructor will be removed
addRange(presentation, region.getOffset(), region.getLength(), fDefaultTextStyle);
return;
}
int lastStart = region.getOffset();
int length = 0;
boolean firstToken = true;
IToken lastToken = Token.UNDEFINED;
TextStyle lastTextStyle = getTokenTextStyle(lastToken);
fScanner.setRange(fDocument, lastStart, region.getLength());
while (true) {
IToken token = fScanner.nextToken();
if (token.isEOF())
break;
TextStyle textStyle = getTokenTextStyle(token);
if (lastTextStyle != null && lastTextStyle.equals(textStyle)) {
length += fScanner.getTokenLength();
firstToken = false;
} else {
if (!firstToken)
addRange(presentation, lastStart, length, lastTextStyle);
firstToken = false;
lastToken = token;
lastTextStyle = textStyle;
lastStart = fScanner.getTokenOffset();
length = fScanner.getTokenLength();
}
}
addRange(presentation, lastStart, length, lastTextStyle);
}