public ModulaRuleBasedPartitionScanner() {
	super();
	
       // Create the list of rules that produce tokens
	inactiveCodeRule = new InactiveCodeRule();
	IPredicateRule[] rules = new IPredicateRule[] {
		inactiveCodeRule,
	    new EndOfLineRule("--", END_OF_LINE_COMMENT_TOKEN), //$NON-NLS-1$
	    new ModulaCommentRule(BLOCK_COMMENT_TOKEN),
                            
           new SingleLineRule("\"", "\"", DOUBLE_QUOTE_STRING_TOKEN), //$NON-NLS-1$ //$NON-NLS-2$
           new SingleLineRule("'", "'",   SINGLE_QUOTE_STRING_TOKEN), //$NON-NLS-1$ //$NON-NLS-2$

	    new MultiLineRule("<*", "*>", PRAGMA_TOKEN, (char)0, true) //$NON-NLS-1$ //$NON-NLS-2$
	};

	setPredicateRules(rules);
       setDefaultReturnToken(DEFAULT_TOKEN);
}
 

@Override
protected List<IRule> createRules() {
	List<IRule> rules = new ArrayList<IRule>();

	Token tagBorder = getToken(IJSXColorConstants.TAG_BORDER);
	Token tagName = getToken(IJSXColorConstants.TAG_NAME);
	Token tagAttributeName = getToken(IJSXColorConstants.TAG_ATTRIBUTE_NAME);
	Token tagAttributeEquals = getToken(IJSXColorConstants.TAG_ATTRIBUTE_EQUALS);
	Token tagAttributeValue = getToken(IJSXColorConstants.TAG_ATTRIBUTE_VALUE);

	rules.add(new SingleLineRule("\"", "\"", tagAttributeValue, '\\'));
	rules.add(new SingleLineRule("'", "'", tagAttributeValue, '\\'));
	rules.add(new SingleLineRule("{", "}", tagAttributeValue, '\\'));
	rules.add(new JSXTagRule(tagName, tagBorder));
	rules.add(new WordRule(new NameDetector(), tagAttributeName));

	// setDefaultReturnToken(token);
	return rules;
}
 

/**
 * Create a new javadoc scanner for the given color provider.
 */
public JSDocScanner()
{
	super();

	List<IRule> list = new ArrayList<IRule>();

	// Add rule for tags.
	list.add(new SingleLineRule("<", ">", getToken("text.html.basic"))); //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-1$

	// Add rule for links.
	list.add(new SingleLineRule("{", "}", getToken("markup.underline.link"))); //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-1$

	// Add word rule for keywords.
	IToken keyword = getToken("meta.tag.documentation.js"); //$NON-NLS-1$
	WordRule wordRule = new WordRule(new JSDocWordDetector());
	for (String word : KEYWORDS)
	{
		wordRule.addWord(word, keyword);
	}
	list.add(wordRule);

	setDefaultReturnToken(getToken("comment.block.documentation.js")); //$NON-NLS-1$
	setRules(list.toArray(new IRule[list.size()]));
}
 

/**
 * Creates the partitioner and sets up the appropriate rules.
 */
public SamplePartitionScanner() {
	IToken tkString = new Token(LANG_STRING);
	IToken tkRawString = new Token(LANG_RAW_STRING);
	IToken tkCharacter = new Token(LANG_CHARACTER);
	IToken tkSingleComment = new Token(LANG_SINGLE_COMMENT);
	IToken tkMultiComment = new Token(LANG_MULTI_COMMENT);
	
	List<IPredicateRule> rules = new ArrayList<IPredicateRule>();
	
	rules.add(new MultiLineRule("`", "`", tkRawString, NO_ESCAPE, true));
	rules.add(new MultiLineRule("\"", "\"", tkString, '\\', true));
	rules.add(new SingleLineRule("'", "'", tkCharacter, '\\', true));
	
	rules.add(new EndOfLineRule("//", tkSingleComment, NO_ESCAPE));
	
	rules.add(new MultiLineRule("/*", "*/", tkMultiComment, NO_ESCAPE, true));
	
	
	setPredicateRules(rules.toArray(new IPredicateRule[rules.size()]));
}
 

public HPartitionScanner() {
   // IToken key= new Token(IDocument.DEFAULT_CONTENT_TYPE);
   IToken comment = new Token(COMMENT);
   IToken propertyValue = new Token(PROPERTY_VALUE);

   setPredicateRules(new IPredicateRule[] { 
         new SingleLineRule(AssistConstants.PARAM_DELIM_EQ, null, propertyValue, '\\', true, true), 
         new SingleLineRule("#", null, comment, (char) 0, true, true), });
}
 

public XMLTagScanner( ColorManager manager) {
   IToken string = new Token(new TextAttribute(manager.getColor(IXMLColorConstants.STRING)));

   IRule[] rules = new IRule[3];

   // Add rule for double quotes
   rules[0] = new SingleLineRule("\"", "\"", string, '\\');
   // Add a rule for single quotes
   rules[1] = new SingleLineRule("'", "'", string, '\\');
   // Add generic whitespace rule.
   rules[2] = new WhitespaceRule(new XMLWhitespaceDetector());

   setRules(rules);
}
 

/**
 * createStringRules
 * 
 * @return
 */
private List<IRule> createStringRules()
{
	List<IRule> rules = new ArrayList<IRule>();

	rules.add(new SingleLineRule("\"", "\"", createToken(CSSTokenType.DOUBLE_QUOTED_STRING), '\\'));
	rules.add(new SingleLineRule("\'", "\'", createToken(CSSTokenType.SINGLE_QUOTED_STRING), '\\'));

	return rules;
}
 

/**
 * Creates the partitioner and sets up the appropriate rules.
 */
public PropertiesFilePartitionScanner() {
	super();

	IToken comment= new Token(COMMENT);
	IToken propertyValue= new Token(PROPERTY_VALUE);
	IToken key= new Token(IDocument.DEFAULT_CONTENT_TYPE);

	List<IPredicateRule> rules= new ArrayList<IPredicateRule>();

	// Add rule for leading white space.
	rules.add(new LeadingWhitespacePredicateRule(key, "\t")); //$NON-NLS-1$
	rules.add(new LeadingWhitespacePredicateRule(key, " ")); //$NON-NLS-1$

	// Add rules for comments.
	rules.add(new EndOfLineRule("#", comment, (char) 0, true)); //$NON-NLS-1$
	rules.add(new EndOfLineRule("!", comment, (char) 0, true)); //$NON-NLS-1$

	// Add rules for property values.
	rules.add(new SingleLineRule("=", null, propertyValue, '\\', true, true)); //$NON-NLS-1$
	rules.add(new SingleLineRule(":", null, propertyValue, '\\', true, true)); //$NON-NLS-1$
	rules.add(new SingleLineRule(" ", null, propertyValue, '\\', true, true)); //$NON-NLS-1$
	rules.add(new SingleLineRule("\t", null, propertyValue, '\\', true, true)); //$NON-NLS-1$

	// Add special case word rule.
	EmptyCommentRule wordRule= new EmptyCommentRule(comment);
	rules.add(wordRule);

	IPredicateRule[] result= new IPredicateRule[rules.size()];
	rules.toArray(result);
	setPredicateRules(result);
}
 

/**
 * Creates the partitioner and sets up the appropriate rules.
 */
public JavaPartitionScanner() {
	super();

	IToken string= new Token(JAVA_STRING);
	IToken character= new Token(JAVA_CHARACTER);
	IToken javaDoc= new Token(JAVA_DOC);
	IToken multiLineComment= new Token(JAVA_MULTI_LINE_COMMENT);
	IToken singleLineComment= new Token(JAVA_SINGLE_LINE_COMMENT);

	List<IPredicateRule> rules= new ArrayList<IPredicateRule>();

	// Add rule for single line comments.
	rules.add(new EndOfLineRule("//", singleLineComment)); //$NON-NLS-1$

	// Add rule for strings.
	rules.add(new SingleLineRule("\"", "\"", string, '\\')); //$NON-NLS-2$ //$NON-NLS-1$

	// Add rule for character constants.
	rules.add(new SingleLineRule("'", "'", character, '\\')); //$NON-NLS-2$ //$NON-NLS-1$

	// Add special case word rule.
	EmptyCommentRule wordRule= new EmptyCommentRule(multiLineComment);
	rules.add(wordRule);

	// Add rules for multi-line comments and javadoc.
	rules.add(new MultiLineRule("/**", "*/", javaDoc)); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(new MultiLineRule("/*", "*/", multiLineComment)); //$NON-NLS-1$ //$NON-NLS-2$

	IPredicateRule[] result= new IPredicateRule[rules.size()];
	rules.toArray(result);
	setPredicateRules(result);
}
 

/**
 * Creates the partitioner and sets up the appropriate rules.
 */
public EditorConfigPartitionScanner() {
	super();

	IToken comment = new Token(COMMENT);
	IToken sectionName = new Token(SECTION);
	IToken propertyValue = new Token(PROPERTY_VALUE);
	IToken key = new Token(IDocument.DEFAULT_CONTENT_TYPE);

	List<IPredicateRule> rules = new ArrayList<IPredicateRule>();

	// Add rule for leading white space.
	rules.add(new LeadingWhitespacePredicateRule(key, "\t")); //$NON-NLS-1$
	rules.add(new LeadingWhitespacePredicateRule(key, " ")); //$NON-NLS-1$

	// Add rules for comments.
	rules.add(new EndOfLineRule("#", comment, (char) 0, true)); //$NON-NLS-1$
	// rules.add(new EndOfLineRule("!", comment, (char) 0, true));
	// //$NON-NLS-1$

	// Add rules for sections.
	rules.add(new SingleLineRule("[", "]", sectionName, '\\', true, true)); //$NON-NLS-1$

	// Add rules for property values.
	rules.add(new SingleLineRule("=", null, propertyValue, '\\', true, true)); //$NON-NLS-1$
	rules.add(new SingleLineRule(":", null, propertyValue, '\\', true, true)); //$NON-NLS-1$
	rules.add(new SingleLineRule(" ", null, propertyValue, '\\', true, true)); //$NON-NLS-1$
	rules.add(new SingleLineRule("\t", null, propertyValue, '\\', true, true)); //$NON-NLS-1$

	// Add special case word rule.
	EmptyCommentRule wordRule = new EmptyCommentRule(comment);
	rules.add(wordRule);

	IPredicateRule[] result = new IPredicateRule[rules.size()];
	rules.toArray(result);
	setPredicateRules(result);
}
 

public CalcitePartitionScanner() {
	IToken comment = new Token(SQL_COMMENT);
	IToken string = new Token(SQL_STRING);
	IToken quotedIdentifier = new Token(SQL_QUOTED_IDENTIFIER);

	setPredicateRules(new IPredicateRule[] {
			new EndOfLineRule("//", comment),
			new EndOfLineRule("--", comment),
			new MultiLineRule("/*", "*/", comment),
			new SingleLineRule("\"", "\"", quotedIdentifier),
			new MultiLineRule("'", "'", string),
	});
}
 

public StringScanner() {
	List<IRule> rules = new ArrayList<IRule>();

	Token stringToken = new Token(new TextAttribute(new Color(
			Display.getCurrent(), new RGB(42, 0, 255))));
	rules.add(new SingleLineRule("'", "'", stringToken));
	rules.add(new SingleLineRule("\"", "\"", stringToken));

	setRules(rules.toArray(new IRule[rules.size()]));
}
 

protected void initialize(String[] keywords) {
    ColorRegistry registry = JFaceResources.getColorRegistry();

    IToken keyword = new Token(new TextAttribute(registry.get(KEYWORD_COLOR), null, SWT.BOLD));
    IToken string = new Token(new TextAttribute(registry.get(STRING_COLOR)));
    IToken number = new Token(new TextAttribute(registry.get(NUMBER_COLOR)));
    IToken annotation = new Token(new TextAttribute(registry.get(ANNOTATION_COLOR)));
    IToken defaultToken = new Token(new TextAttribute(registry.get(DEFAULT_COLOR)));

    List<IRule> rules = new ArrayList<IRule>();

    // strings
    rules.add(new SingleLineRule("\"", "\"", string, '\\'));

    // annotations
    rules.add(new MultiLineRule("[", "]", annotation));

    // numbers
    rules.add(new NumberRule(number));

    // keywords and normal (default) text
    WordRule wordRule = new WordRule(new WordDetector(), defaultToken);
    for (int i = 0; i < keywords.length; i++) {
        wordRule.addWord(keywords[i], keyword);
    }
    rules.add(wordRule);

    setRules(rules.toArray(new IRule[rules.size()]));
}
 

@Override
protected void initPredicateRules(ArrayList2<IPredicateRule> rules) {
	addStandardRules(rules, 
		LangPartitionTypes.LINE_COMMENT.getId(), 
		LangPartitionTypes.BLOCK_COMMENT.getId(), 
		null, 
		null, 
		null
	);
	
	rules.add(new SingleLineRule("'", "'", new Token(LangPartitionTypes.CHARACTER.getId()), '\\'));
	rules.add(new MultiLineRule("`", "`", new Token(LangPartitionTypes.MULTILINE_STRING.getId())));
	rules.add(new SingleLineRule("\"", "\"", new Token(LangPartitionTypes.STRING.getId()), '\\'));
}
 

protected List createRules() {

		List rules= new ArrayList();

		// Add rule for character constants.
		Token token= getToken(IJavaScriptColorConstants.JAVA_STRING);
		rules.add(new SingleLineRule("'", "'", token, '\\')); //$NON-NLS-2$ //$NON-NLS-1$


		// Add generic whitespace rule.
		rules.add(new WhitespaceRule(new JavaWhitespaceDetector()));

		String version= getPreferenceStore().getString(SOURCE_VERSION);

		// Add JLS3 rule for /@\s*interface/ and /@\s*\w+/
		token = getToken(ITypeScriptColorConstants.DECORATOR);
		DecoratorRule decoratorRule = new DecoratorRule(token,
				"", version);
		rules.add(decoratorRule);
		
		// Add word rule for new keywords, 4077
		JavaWordDetector wordDetector= new JavaWordDetector();
		token= getToken(IJavaScriptColorConstants.JAVA_DEFAULT);
		CombinedWordRule combinedWordRule= new CombinedWordRule(wordDetector, token);
		
		// Add rule for operators
		token= getToken(IJavaScriptColorConstants.JAVA_OPERATOR);
		rules.add(new OperatorRule(token));

		// Add rule for brackets
		token= getToken(IJavaScriptColorConstants.JAVA_BRACKET);
		rules.add(new BracketRule(token));

		// Add word rule for keyword 'return'.
		CombinedWordRule.WordMatcher returnWordRule= new CombinedWordRule.WordMatcher();
		token= getToken(IJavaScriptColorConstants.JAVA_KEYWORD_RETURN);
		returnWordRule.addWord(RETURN, token);  
		combinedWordRule.addWordMatcher(returnWordRule);

		// Add word rule for keywords, types, and constants.
		CombinedWordRule.WordMatcher wordRule= new CombinedWordRule.WordMatcher();
		token= getToken(IJavaScriptColorConstants.JAVA_KEYWORD);
		for (int i=0; i<fgKeywords.length; i++)
			wordRule.addWord(fgKeywords[i], token);
		for (int i=0; i<fgConstants.length; i++)
			wordRule.addWord(fgConstants[i], token);
		// TypeScript key words
		for (int i=0; i<fgTypeScriptKeywords.length; i++)
			wordRule.addWord(fgTypeScriptKeywords[i], token);

		combinedWordRule.addWordMatcher(wordRule);

		rules.add(combinedWordRule);

		setDefaultReturnToken(getToken(IJavaScriptColorConstants.JAVA_DEFAULT));
		return rules;
	}
 

public JsniScanner(IColorManager colorManager) {
  // TODO: get these from GWT preference store instead of Java's

  // TODO: need to refresh colorManager when prefs change
  //
  IToken jsniJavaRef = new Token(new TextAttribute(
      colorManager.getColor(JsniColorConstants.JSNI_JAVA_REF)));

  IToken jsniComment = new Token(new TextAttribute(
      colorManager.getColor(JsniColorConstants.JSNI_COMMENT)));

  IToken jsniKeyword = new Token(new TextAttribute(
      colorManager.getColor(JsniColorConstants.JSNI_KEYWORD)));

  IToken jsniDefault = new Token(new TextAttribute(
      colorManager.getColor(JsniColorConstants.JSNI_DEFAULT)));

  IToken jsniString = new Token(new TextAttribute(
      colorManager.getColor(JsniColorConstants.JSNI_STRING)));

  ArrayList<IRule> rules = new ArrayList<IRule>();

  // Java references
  rules.add(new JavaRefRule(jsniJavaRef));

  // single line comments
  rules.add(new EndOfLineRule("//", jsniComment));

  // JS keywords
  WordRule keywordRules = new WordRule(new JsWordDetector(), jsniDefault);
  for (String keyword : JS_KEYWORDS) {
    keywordRules.addWord(keyword, jsniKeyword);
  }
  rules.add(keywordRules);

  // Strings
  rules.add(new SingleLineRule("\"", "\"", jsniString, '\\', true));
  rules.add(new SingleLineRule("'", "'", jsniString, '\\', true));

  // Add generic whitespace rule.
  rules.add(new WhitespaceRule(new WhitespaceDetector()));

  // convert to array and set them for RuleBasedScanner to use
  setRules(rules.toArray(new IRule[rules.size()]));
}
 

@Override
protected List<IRule> createRules() {

	List<IRule> rules= new ArrayList<IRule>();

	// Add rule for character constants.
	Token token= getToken(IJavaColorConstants.JAVA_STRING);
	rules.add(new SingleLineRule("'", "'", token, '\\')); //$NON-NLS-2$ //$NON-NLS-1$


	Token defaultToken= getToken(IJavaColorConstants.JAVA_DEFAULT);
	
	// Add generic whitespace rule.
	rules.add(new WhitespaceRule(new JavaWhitespaceDetector(), defaultToken));

	String version= getPreferenceStore().getString(SOURCE_VERSION);

	// Add JLS3 rule for /@\s*interface/ and /@\s*\w+/
	token= getToken(ANNOTATION_COLOR_KEY);
	AnnotationRule atInterfaceRule= new AnnotationRule(getToken(IJavaColorConstants.JAVA_KEYWORD), token, JavaCore.VERSION_1_5, version);
	rules.add(atInterfaceRule);
	fVersionDependentRules.add(atInterfaceRule);

	// Add word rule for new keywords, see bug 4077
	JavaWordDetector wordDetector= new JavaWordDetector();
	CombinedWordRule combinedWordRule= new CombinedWordRule(wordDetector, defaultToken);

	VersionedWordMatcher j14Matcher= new VersionedWordMatcher(defaultToken, JavaCore.VERSION_1_4, version);

	token= getToken(IJavaColorConstants.JAVA_KEYWORD);
	for (int i=0; i<fgJava14Keywords.length; i++)
		j14Matcher.addWord(fgJava14Keywords[i], token);

	combinedWordRule.addWordMatcher(j14Matcher);
	fVersionDependentRules.add(j14Matcher);

	VersionedWordMatcher j15Matcher= new VersionedWordMatcher(defaultToken, JavaCore.VERSION_1_5, version);
	
	token= getToken(IJavaColorConstants.JAVA_KEYWORD);
	for (int i=0; i<fgJava15Keywords.length; i++)
		j15Matcher.addWord(fgJava15Keywords[i], token);

	combinedWordRule.addWordMatcher(j15Matcher);
	fVersionDependentRules.add(j15Matcher);

	// Add rule for operators
	token= getToken(IJavaColorConstants.JAVA_OPERATOR);
	rules.add(new OperatorRule(token));

	// Add rule for brackets
	token= getToken(IJavaColorConstants.JAVA_BRACKET);
	rules.add(new BracketRule(token));

	// Add word rule for keyword 'return'.
	CombinedWordRule.WordMatcher returnWordRule= new CombinedWordRule.WordMatcher();
	token= getToken(IJavaColorConstants.JAVA_KEYWORD_RETURN);
	returnWordRule.addWord(RETURN, token);
	combinedWordRule.addWordMatcher(returnWordRule);

	// Add word rule for keywords, types, and constants.
	CombinedWordRule.WordMatcher wordRule= new CombinedWordRule.WordMatcher();
	token= getToken(IJavaColorConstants.JAVA_KEYWORD);
	for (int i=0; i<fgKeywords.length; i++)
		wordRule.addWord(fgKeywords[i], token);
	for (int i=0; i<fgTypes.length; i++)
		wordRule.addWord(fgTypes[i], token);
	for (int i=0; i<fgConstants.length; i++)
		wordRule.addWord(fgConstants[i], token);

	combinedWordRule.addWordMatcher(wordRule);

	rules.add(combinedWordRule);

	setDefaultReturnToken(defaultToken);
	return rules;
}