public static void configure(TextLayout textLayout) {
	String text = textLayout.getText();
	Document doc = new Document(text);
	ITokenScanner scanner = getRecipeScanner(doc);
	scanner.setRange(doc, 0, doc.getLength());
	IToken token;
	while ((token = scanner.nextToken()) != Token.EOF) {
		int offset = scanner.getTokenOffset();
		int length = scanner.getTokenLength();
		Object data = token.getData();
		if (data != null && data instanceof TextStyle) {
			TextStyle textStyle = (TextStyle) data;
			textLayout.setStyle(textStyle, offset, offset + length - 1);
		}
	}
	scanner = null;
	doc = null;
}
 

public static void configure(TextLayout textLayout) {
	String text = textLayout.getText();
	Document doc = new Document(text);
	ITokenScanner scanner = getRecipeScanner(doc);
	scanner.setRange(doc, 0, doc.getLength());
	IToken token;
	while ((token = scanner.nextToken()) != Token.EOF) {
		int offset = scanner.getTokenOffset();
		int length = scanner.getTokenLength();
		Object data = token.getData();
		if (data != null && data instanceof TextStyle) {
			TextStyle textStyle = (TextStyle) data;
			textLayout.setStyle(textStyle, offset, offset + length - 1);
		}
	}
}
 

public XMLPartitionScanner() {

      IToken xmlComment = new Token(XML_COMMENT);
      IToken xmlPI = new Token(XML_PI);
      IToken startTag = new Token(XML_START_TAG);
      IToken endTag = new Token(XML_END_TAG);
      IToken docType = new Token(XML_DOCTYPE);
      IToken text = new Token(XML_TEXT);

      IPredicateRule[] rules = new IPredicateRule[7];

      rules[0] = new NonMatchingRule();
      rules[1] = new MultiLineRule("<!--", "-->", xmlComment);
      rules[2] = new MultiLineRule("<?", "?>", xmlPI);
      rules[3] = new MultiLineRule("</", ">", endTag);
      rules[4] = new StartTagRule(startTag);
      rules[5] = new MultiLineRule("<!DOCTYPE", ">", docType);
      rules[6] = new XMLTextPredicateRule(text);

      setPredicateRules(rules);
   }
 

@Test public void testWholePart() throws Exception {
	PartitionTokenScanner scanner = getPartitionTokenScanner(t(2, 3),t(4,3),t(2,1),t(34,3));
	scanner.setPartialRange(null, 0, 42, "3", 0);
	
	assertEquals("3",scanner.nextToken().getData());
	assertEquals(0,scanner.getTokenOffset());
	assertEquals(6,scanner.getTokenLength());
	
	assertEquals("1",scanner.nextToken().getData());
	assertEquals(6,scanner.getTokenOffset());
	assertEquals(2,scanner.getTokenLength());
	
	assertEquals("3",scanner.nextToken().getData());
	assertEquals(8,scanner.getTokenOffset());
	assertEquals(34,scanner.getTokenLength());
	
	assertSame(Token.EOF, scanner.nextToken());
}
 

/**
 * 
 */
public CompositePartitionScanner(ISubPartitionScanner defaultPartitionScanner, ISubPartitionScanner primaryPartitionScanner,
		IPartitionerSwitchStrategy partitionerSwitchStrategy) {
	this.defaultPartitionScanner = defaultPartitionScanner;
	this.primaryPartitionScanner = primaryPartitionScanner;

	defaultPartitionScanner.initCharacterScanner(this, partitionerSwitchStrategy.getDefaultSwitchStrategy());
	primaryPartitionScanner.initCharacterScanner(this, partitionerSwitchStrategy.getPrimarySwitchStrategy());

	String[][] pairs = partitionerSwitchStrategy.getSwitchTagPairs();
	switchRules = new IPredicateRule[pairs.length][];
	for (int i = 0; i < pairs.length; ++i) {
		switchRules[i] = new IPredicateRule[] { new SingleTagRule(pairs[i][0], new Token(START_SWITCH_TAG)), new SingleTagRule(pairs[i][1], new Token(END_SWITCH_TAG)) };
	}

	currentPartitionScanner = defaultPartitionScanner;
	setDefaultReturnToken(new Token(IDocument.DEFAULT_CONTENT_TYPE));
}
 

private void adaptToStyleChange(Token token, PropertyChangeEvent event, int styleAttribute) {
	boolean eventValue = false;
	Object value = event.getNewValue();
	if (value instanceof Boolean)
		eventValue = ((Boolean) value).booleanValue();
	else if (IPreferenceStore.TRUE.equals(value))
		eventValue = true;

	Object data = token.getData();
	if (data instanceof TextAttribute) {
		TextAttribute oldAttr = (TextAttribute) data;
		boolean activeValue = (oldAttr.getStyle() & styleAttribute) == styleAttribute;
		if (activeValue != eventValue)
			token.setData(new TextAttribute(oldAttr.getForeground(), oldAttr.getBackground(),
					eventValue ? oldAttr.getStyle() | styleAttribute : oldAttr.getStyle() & ~styleAttribute));
	}
}
 

@Override
public IToken nextToken() {
	while (true) {
		lastScannedIndex++;
		if (lastScannedIndex >= concepts.size())
			return Token.EOF;

		SyntaxConcept concept = getLastConcept();
		TextAttribute attr = ClassificationFormat.getTextAttribute(concept);
		if (attr != null) {
			Logger.debug("token: '" + concept.type + "' @" + concept.line + "," + concept.column + "; value: "
					+ concept.value);
			return new Token(attr);
		} else {
			Logger.debug("ignored: '" + concept.type + "' @" + concept.line + "," + concept.column + "; value: "
					+ concept.value);
		}
	}
}
 

/**
 * createLexemeProvider
 * 
 * @param document
 * @param offset
 * @return
 */
ILexemeProvider<XMLTokenType> createLexemeProvider(IDocument document, int offset)
{
	int documentLength = document.getLength();

	// account for last position returning an empty IDocument default partition
	int lexemeProviderOffset = (offset >= documentLength) ? documentLength - 1 : offset;

	return new XMLLexemeProvider(document, lexemeProviderOffset, new XMLTagScanner()
	{
		@Override
		protected IToken createToken(XMLTokenType type)
		{
			return new Token(type);
		}
	});
}
 

private void adaptToStyleChange(Token token, PropertyChangeEvent event, int styleAttribute) {
	boolean eventValue= false;
	Object value= event.getNewValue();
	if (value instanceof Boolean)
		eventValue= ((Boolean) value).booleanValue();
	else if (IPreferenceStore.TRUE.equals(value))
		eventValue= true;

	Object data= token.getData();
	if (data instanceof TextAttribute) {
		TextAttribute oldAttr= (TextAttribute) data;
		boolean activeValue= (oldAttr.getStyle() & styleAttribute) == styleAttribute;
		if (activeValue != eventValue)
			token.setData(new TextAttribute(oldAttr.getForeground(), oldAttr.getBackground(), eventValue ? oldAttr.getStyle() | styleAttribute : oldAttr.getStyle() & ~styleAttribute));
	}
}
 

public IToken evaluate(ICharacterScanner scanner, boolean resume) {
    if (resume) {
        boolean inString = false;
        do {
            int c = scanner.read();
            if (((char) c) == ',' && !inString)
                break;
            else if (((char) c) == '@') {
                scanner.unread();
                return Token.UNDEFINED;
            } else if (((char) c) == '"' && !inString)
                inString = true;
            else if (((char) c) == '"' && inString)
                inString = false;
            else if (c == ICharacterScanner.EOF)
                return Token.UNDEFINED;
        } while (true);
    }
    return doEvaluate(scanner, 1);
}
 

@Override
public IToken evaluate(ICharacterScanner scanner) {
	int ch = scanner.read();
	if (ch == '<') {
		ch = scanner.read();
		if (ch == '/') {
			return token;
		}
		scanner.unread();
		return token;
	} else if (ch == '>') {
		ch = scanner.read();
		if (ch == '/') {
			return token;
		}
		scanner.unread();
		return token;
	}
	scanner.unread();
	return Token.UNDEFINED;
}
 

/**
 * createScannerSpecificRules
 * 
 * @return
 */
protected Collection<? extends IRule> createScannerSpecificRules()
{
	List<IRule> rules = new ArrayList<IRule>();
	WordRule wordRule = new WordRule(new KeywordIdentifierDetector(), Token.UNDEFINED);
	wordRule.addWord("em", createToken(CSSTokenType.EMS));
	wordRule.addWord("ex", createToken(CSSTokenType.EXS));
	wordRule.addWord("px", createToken(CSSTokenType.LENGTH));
	wordRule.addWord("cm", createToken(CSSTokenType.LENGTH));
	wordRule.addWord("mm", createToken(CSSTokenType.LENGTH));
	wordRule.addWord("in", createToken(CSSTokenType.LENGTH));
	wordRule.addWord("pt", createToken(CSSTokenType.LENGTH));
	wordRule.addWord("pc", createToken(CSSTokenType.LENGTH));
	wordRule.addWord("deg", createToken(CSSTokenType.ANGLE));
	wordRule.addWord("rad", createToken(CSSTokenType.ANGLE));
	wordRule.addWord("grad", createToken(CSSTokenType.ANGLE));
	wordRule.addWord("ms", createToken(CSSTokenType.TIME));
	wordRule.addWord("s", createToken(CSSTokenType.TIME));
	wordRule.addWord("hz", createToken(CSSTokenType.FREQUENCY));
	wordRule.addWord("khz", createToken(CSSTokenType.FREQUENCY));
	wordRule.addWord("Hz", createToken(CSSTokenType.FREQUENCY));
	wordRule.addWord("kHz", createToken(CSSTokenType.FREQUENCY));
	addWordsToRule(wordRule, FUNCTIONS, CSSTokenType.FUNCTION);
	rules.add(wordRule);
	return rules;
}
 

/**
 * A default constructor.
 * @param manager
 */
public TexArgScanner(ColorManager manager) {
    IToken commentToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.COMMENT),
            null,
            manager.getStyle(ColorManager.COMMENT_STYLE)));

    //Commands are colored in argument color with command styles 
    IToken commandToken = new Token(
            new TextAttribute(
                    manager.getColor(ColorManager.CURLY_BRACKETS),
                    null,
                    manager.getStyle(ColorManager.COMMAND_STYLE)));

    List<IRule> rules = new ArrayList<IRule>();
    rules.add(new EndOfLineRule("%", commentToken, '\\'));
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    rules.add(new WordRule(new TexWord(), commandToken));

    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}
 

@Override
public IToken evaluate(ICharacterScanner scanner, boolean resume) {
	if (scanner instanceof ITokenScanner) {
		if (isShowInactiveCode && !CollectionUtils.isEmpty(inactiveCodeRegions)) {
			ITokenScanner tokenScanner = (ITokenScanner) scanner;
			ITextRegion reg = findInactiveCodeRegion(tokenScanner.getTokenOffset());
			if (reg != null) {
				for (int i = tokenScanner.getTokenOffset(); i < reg.getOffset() + reg.getLength(); i++) {
					scanner.read();
				}
				return ModulaPartitionTokens.DISABLED_CODE;
			}
		}
	}
	return Token.UNDEFINED;
}
 

/***
 * Add some partition rules common to C-style languages.
 * All rules are optional, if an id is null, the rule will not be added.
 */
protected static void addStandardRules(ArrayList2<IPredicateRule> rules, 
		String lineCommentId, String blockCommentId, 
		String docLineCommentId, String docBlockCommentId,
		String stringId) {
	if(docLineCommentId != null) {
		rules.add(new PatternRule("///", null, new Token(docLineCommentId), NO_ESCAPE_CHAR, true, true));
	}
	if(docBlockCommentId != null) {
		rules.add(new PatternRule("/**", "*/", new Token(docBlockCommentId), NO_ESCAPE_CHAR, false, true));
	}
	
	if(lineCommentId != null) {
		rules.add(new PatternRule("//", null, new Token(lineCommentId), NO_ESCAPE_CHAR, true, true));
	}
	if(blockCommentId != null) {
		rules.add(new PatternRule("/*", "*/", new Token(blockCommentId), NO_ESCAPE_CHAR, false, true));
	}
	
	if(stringId != null) {
		rules.add(new PatternRule("\"", "\"", new Token(stringId), '\\', false, true));
	}
	
}
 

@Override
protected List<IRule> createRules() {
	List<IRule> list= new ArrayList<IRule>();
	Token defaultToken= getToken(fDefaultTokenProperty);

	List<WordMatcher> matchers= createMatchers();
	if (matchers.size() > 0) {
		CombinedWordRule combinedWordRule= new CombinedWordRule(new AtJavaIdentifierDetector(), defaultToken);
		for (int i= 0, n= matchers.size(); i < n; i++)
			combinedWordRule.addWordMatcher(matchers.get(i));
		list.add(combinedWordRule);
	}

	setDefaultReturnToken(defaultToken);

	return list;
}
 

/**
 * Creates the partitioner and sets up the appropriate rules.
 */
public SamplePartitionScanner() {
	IToken tkString = new Token(LANG_STRING);
	IToken tkRawString = new Token(LANG_RAW_STRING);
	IToken tkCharacter = new Token(LANG_CHARACTER);
	IToken tkSingleComment = new Token(LANG_SINGLE_COMMENT);
	IToken tkMultiComment = new Token(LANG_MULTI_COMMENT);
	
	List<IPredicateRule> rules = new ArrayList<IPredicateRule>();
	
	rules.add(new MultiLineRule("`", "`", tkRawString, NO_ESCAPE, true));
	rules.add(new MultiLineRule("\"", "\"", tkString, '\\', true));
	rules.add(new SingleLineRule("'", "'", tkCharacter, '\\', true));
	
	rules.add(new EndOfLineRule("//", tkSingleComment, NO_ESCAPE));
	
	rules.add(new MultiLineRule("/*", "*/", tkMultiComment, NO_ESCAPE, true));
	
	
	setPredicateRules(rules.toArray(new IPredicateRule[rules.size()]));
}
 

/**
 * HTMLSubPartitionScanner
 */
public HTMLSubPartitionScanner()
{
	// @formatter:off
	super(
			new ISubPartitionScanner[] {
			new SubPartitionScanner(
					HTMLSourceConfiguration.getDefault().getPartitioningRules(),
					HTMLSourceConfiguration.CONTENT_TYPES,
					new Token(HTMLSourceConfiguration.DEFAULT)
					),
					JSSourceConfiguration.getDefault().createSubPartitionScanner(),
					CSSSourceConfiguration.getDefault().createSubPartitionScanner(),
					SVGSourceConfiguration.getDefault().createSubPartitionScanner()
			},
			new IPartitionScannerSwitchStrategy[] {
			new PartitionScannerSwitchStrategy(JS_SWITCH_SEQUENCES),
					new PartitionScannerSwitchStrategy(CSS_SWITCH_SEQUENCES),
					new PartitionScannerSwitchStrategy(SVG_SWITCH_SEQUENCES)
			}
	);
	// @formatter:on
}
 

protected IToken createToken() {
	String attributeId = delimiter
			? HighlightingConfiguration.TEMPLATE_DELIMITER_ID
			: HighlightingConfiguration.TEMPLATE_ID;
	Token token = new Token(getAttribute(attributeId));
	return token;
}
 

private RuleBasedScanner getCommentScanner(){
   if (commentScanner == null) {
       commentScanner = new HCommentScanner();
       commentScanner.setDefaultReturnToken(new Token(new TextAttribute(ResourceUtils.getColor(Styles.COMMENT))));
   }
   return commentScanner;
}
 

private RuleBasedScanner getValueScanner(){
   if (valueScanner == null) {
       valueScanner = new HValueScanner();
       valueScanner.setDefaultReturnToken(new Token(new TextAttribute(ResourceUtils.getColor(Styles.STRING))));
   }
   return valueScanner;
}
 

protected void setupPresentationReconciler(PresentationReconciler reconciler, ISourceViewer sourceViewer) {
	// Must be called from UI thread
	assertTrue(Display.getCurrent() != null);
	
	// Create a token registry for given sourceViewer
	TokenRegistry tokenRegistry = new TokenRegistry(colorManager, stylingPrefs) {
		@Override
		protected void handleTokenModified(Token token) {
			sourceViewer.invalidateTextPresentation();
		}
	};
	addConfigurationScopedOwned(sourceViewer, tokenRegistry);
	
	ArrayList2<AbstractLangScanner> scanners = new ArrayList2<>();
	
	for(LangPartitionTypes partitionType : getPartitionTypes()) {
		
		String contentType = partitionType.getId();
		AbstractLangScanner scanner = createScannerFor(Display.getCurrent(), partitionType, tokenRegistry);
		scanners.add(scanner);
		
		DefaultDamagerRepairer dr = getDamagerRepairer(scanner, contentType);
		reconciler.setDamager(dr, contentType);
		reconciler.setRepairer(dr, contentType);
	}
	
}
 

protected CDataScanner getCDataScanner(){
    if( cdataScanner == null) {
        cdataScanner = new CDataScanner(colorManager);
        cdataScanner.setDefaultReturnToken(new Token(new TextAttribute(
                colorManager.getColor(IXMLColorConstants.CDATA_TEXT))));
    }
    return cdataScanner;
}
 

public JavaRegExpRule(LogToolRuleDesc ruleDesc) {
	int flags = 0;
	find = false;
	if (ruleDesc.isCaseInsensitive())
		flags = java.util.regex.Pattern.CASE_INSENSITIVE | java.util.regex.Pattern.UNICODE_CASE;
	if (ruleDesc.getMatchMode().startsWith("find"))
		find = true;
	regexp = Pattern.compile(ruleDesc.getRuleValue(),flags);
	priority = ruleDesc.getPriority();
	successToken = new Token(new TokenData(TextAttributeFactory.getTextAttribute(ruleDesc),priority));
}
 

public XMLScanner( ColorManager manager) {
   IToken procInstr = new Token(new TextAttribute(manager.getColor(IXMLColorConstants.PROC_INSTR)));
   IToken docType = new Token(new TextAttribute(manager.getColor(IXMLColorConstants.DOCTYPE)));

   IRule[] rules = new IRule[3];
   // Add rule for processing instructions and doctype
   rules[0] = new MultiLineRule("<?", "?>", procInstr);
   rules[1] = new MultiLineRule("<!DOCTYPE", ">", docType);
   // Add generic whitespace rule.
   rules[2] = new WhitespaceRule(new XMLWhitespaceDetector());

   setRules(rules);
}
 

private IToken computeReturn(CSSTokenType tokenData, boolean isWhitespace)
{
	if (isWhitespace)
	{
		return tokenOnWhitespace;
	}

	// Grab data again, because we may have changed the token above...
	if (tokenData != null)
	{
		String scope = tokenData.getScope();
		int contextLen = fContextToAppend.length();
		if (contextLen == 0)
		{
			return new Token(scope);
		}
		int scopeLen = scope.length();

		// Note: optimized creating string with context+scope as this was a bottleneck when profiling.
		char[] array = new char[contextLen + scopeLen];
		fContextToAppend.getChars(0, contextLen, array, 0);
		scope.getChars(0, scopeLen, array, contextLen);

		return new Token(new String(array));
	}
	else
	{
		return new Token(CSSTokenType.UNDEFINED.getScope());
	}
}
 

private static IPartitionTokenScanner createDefaultScanner() {
    final IToken string = new Token(PatternExpressionViewer.GROOVY_EXPRESSION_CONTENT_TYPE);
    final RuleBasedPartitionScanner scanner = new RuleBasedPartitionScanner();
    scanner.setPredicateRules(new IPredicateRule[] {
            new MultiLineRule(GROOVY_START_TAG, GROOVY_END_TAG, string)
    });
    return scanner;
}
 

protected XMLTagScanner getXMLTagScanner() {
	if (tagScanner == null) {
		tagScanner = new XMLTagScanner(colorManager);
		tagScanner.setDefaultReturnToken(
			new Token(
				new TextAttribute(
					colorManager.getColor(IXMLColorConstants.TAG))));
	}
	return tagScanner;
}
 

private ITokenScanner getStartEndTokenScanner() {
	if (startEndTokenScanner == null) {
		RuleBasedScanner ts = new RuleBasedScanner();
		IToken seqToken = new Token(getStartEndTokenType());
		List<IRule> rules = new ArrayList<IRule>();
		for (String[] pair : getPartitionerSwitchStrategy().getSwitchTagPairs()) {
			rules.add(new SingleTagRule(pair[0], seqToken));
			rules.add(new SingleTagRule(pair[1], seqToken));
		}
		ts.setRules(rules.toArray(new IRule[rules.size()]));
		ts.setDefaultReturnToken(new Token("text")); //$NON-NLS-1$
		startEndTokenScanner = ts;
	}
	return startEndTokenScanner;
}
 

public void createPresentation(TextPresentation presentation, ITypedRegion region) {
	if (fScanner == null) {
		// will be removed if deprecated constructor will be removed
		addRange(presentation, region.getOffset(), region.getLength(), fDefaultTextStyle);
		return;
	}

	int lastStart = region.getOffset();
	int length = 0;
	boolean firstToken = true;
	IToken lastToken = Token.UNDEFINED;
	TextStyle lastTextStyle = getTokenTextStyle(lastToken);

	fScanner.setRange(fDocument, lastStart, region.getLength());

	while (true) {
		IToken token = fScanner.nextToken();
		if (token.isEOF())
			break;

		TextStyle textStyle = getTokenTextStyle(token);
		if (lastTextStyle != null && lastTextStyle.equals(textStyle)) {
			length += fScanner.getTokenLength();
			firstToken = false;
		} else {
			if (!firstToken)
				addRange(presentation, lastStart, length, lastTextStyle);
			firstToken = false;
			lastToken = token;
			lastTextStyle = textStyle;
			lastStart = fScanner.getTokenOffset();
			length = fScanner.getTokenLength();
		}
	}

	addRange(presentation, lastStart, length, lastTextStyle);
}