下面列出了怎么用org.antlr.runtime.Token的API类实例代码及写法，或者点击链接到github查看源代码。
public void testList2() throws Exception {
// Add child ^(nil 101 102 103) to root 5
// should pull 101 102 103 directly to become 5's child list
CommonTree root = new CommonTree(new CommonToken(5));
// child tree
CommonTree r0 = new CommonTree((Token)null);
CommonTree c0, c1, c2;
r0.addChild(c0=new CommonTree(new CommonToken(101)));
r0.addChild(c1=new CommonTree(new CommonToken(102)));
r0.addChild(c2=new CommonTree(new CommonToken(103)));
root.addChild(r0);
assertNull(root.parent);
assertEquals(-1, root.childIndex);
// check children of root all point at root
assertEquals(root, c0.parent);
assertEquals(0, c0.childIndex);
assertEquals(root, c0.parent);
assertEquals(1, c1.childIndex);
assertEquals(root, c0.parent);
assertEquals(2, c2.childIndex);
}
@Override
protected void createContextsForLastCompleteNode(EObject previousModel, boolean strict){
String currentNodePrefix = getPrefix(currentNode);
if (!Strings.isEmpty(currentNodePrefix) && !currentNode.getText().equals(currentNodePrefix)) {
InternalFlexer flexer = flexerFactory.createFlexer(new StringReader(currentNodePrefix));
try {
int token = flexer.advance();
if (token == Token.EOF) { // error case - nothing could be parsed
return;
}
while(token != Token.EOF) {
if (isErrorToken(token))
return;
token = flexer.advance();
}
} catch (IOException e) {
log.error(e.getMessage(), e);
return;
}
}
String prefix = "";
Collection<FollowElement> followElements = parseFollowElements(completionOffset, strict);
// String completeInput = viewer.getDocument().get(0, completionOffset);
// Collection<FollowElement> followElements = parser.getFollowElements(completeInput, strict);
doCreateContexts(lastCompleteNode, currentNode, prefix, previousModel, followElements);
}
Token COMMENT() {
match('!');
while ( !(c=='!' && input.LA(2)==delimiterStopChar) ) {
if ( c==EOF ) {
RecognitionException re = new MismatchedTokenException((int)'!', input);
re.line = input.getLine();
re.charPositionInLine = input.getCharPositionInLine();
errMgr.lexerError(input.getSourceName(), "Nonterminated comment starting at "+startLine+":"+startCharPositionInLine+": '!"+delimiterStopChar+"' missing", templateToken, re);
break;
}
consume();
}
consume();
consume(); // grab !>
return newToken(COMMENT);
}
public void rawDefineTemplate(String name, CompiledST code, Token defT) {
CompiledST prev = rawGetTemplate(name);
if ( prev!=null ) {
if ( !prev.isRegion ) {
errMgr.compileTimeError(ErrorType.TEMPLATE_REDEFINITION, null, defT);
return;
}
else {
if ( code.regionDefType!= ST.RegionType.IMPLICIT && prev.regionDefType== ST.RegionType.EMBEDDED ) {
errMgr.compileTimeError(ErrorType.EMBEDDED_REGION_REDEFINITION, null, defT, getUnMangledTemplateName(name));
return;
}
else if ( code.regionDefType== ST.RegionType.IMPLICIT || prev.regionDefType== ST.RegionType.EXPLICIT ) {
errMgr.compileTimeError(ErrorType.REGION_REDEFINITION,
null,
defT,
getUnMangledTemplateName(name));
return;
}
}
}
code.nativeGroup = this;
code.templateDefStartToken = defT;
templates.put(name, code);
}
public CompiledST defineTemplate(String fullyQualifiedTemplateName,
Token nameT,
List<FormalArgument> args,
String template, Token templateToken) {
if ( verbose ) System.out.println("defineTemplate("+fullyQualifiedTemplateName+")");
if ( fullyQualifiedTemplateName==null || fullyQualifiedTemplateName.length()==0 ) {
throw new IllegalArgumentException("empty template name");
}
if ( fullyQualifiedTemplateName.indexOf('.')>=0 ) {
throw new IllegalArgumentException("cannot have '.' in template names");
}
template = Misc.trimOneStartingNewline(template);
template = Misc.trimOneTrailingNewline(template);
// compile, passing in templateName as enclosing name for any embedded regions
CompiledST code = compile(getFileName(),
fullyQualifiedTemplateName,
args,
template,
templateToken);
code.name = fullyQualifiedTemplateName;
rawDefineTemplate(fullyQualifiedTemplateName, code, nameT);
code.defineArgDefaultValueTemplates(this);
code.defineImplicitlyDefinedTemplates(this); // define any anonymous subtemplates
return code;
}
@Override
protected boolean isSameTokenSequence(String originalText, String newText, int expectedLength) {
try {
InternalFlexer originalSequence = flexerFactory.createFlexer(new StringReader(originalText));
InternalFlexer newSequence = flexerFactory.createFlexer(new StringReader(newText));
int token = originalSequence.advance();
int newLength = 0;
while(token != Token.EOF) {
if (token != newSequence.advance()) {
return false;
}
newLength += newSequence.getTokenLength();
token = originalSequence.advance();
}
return newLength == expectedLength;
} catch(IOException e) {
throw new RuntimeException(e);
}
}
Token COMMENT() {
match('!');
while ( !(c=='!' && input.LA(2)==delimiterStopChar) ) {
if ( c==EOF ) {
RecognitionException re = new MismatchedTokenException((int)'!', input);
re.line = input.getLine();
re.charPositionInLine = input.getCharPositionInLine();
errMgr.lexerError(input.getSourceName(),
"Nonterminated comment starting at " +
startLine +":"+startCharPositionInLine +
": '!" +
delimiterStopChar +
"' missing", templateToken, re);
break;
}
consume();
}
consume();
consume(); // grab !>
return newToken(COMMENT);
}
public CompiledST defineRegion(String enclosingTemplateName, Token regionT, String template, Token templateToken) {
String name = regionT.getText();
template = Misc.trimOneStartingNewline(template);
template = Misc.trimOneTrailingNewline(template);
CompiledST code = compile(getFileName(), enclosingTemplateName, null, template, templateToken);
String mangled = getMangledRegionName(enclosingTemplateName, name);
if ( lookupTemplate(mangled)==null ) {
errMgr.compileTimeError(ErrorType.NO_SUCH_REGION, templateToken, regionT, enclosingTemplateName, name);
return new CompiledST();
}
code.name = mangled;
code.isRegion = true;
code.regionDefType = ST.RegionType.EXPLICIT;
code.templateDefStartToken = regionT;
rawDefineTemplate(mangled, code, regionT);
code.defineArgDefaultValueTemplates(this);
code.defineImplicitlyDefinedTemplates(this);
return code;
}
public CompiledST defineRegion(String enclosingTemplateName, Token regionT, String template, Token templateToken) {
String name = regionT.getText();
template = Misc.trimOneStartingNewline(template);
template = Misc.trimOneTrailingNewline(template);
CompiledST code = compile(getFileName(), enclosingTemplateName, null, template, templateToken);
String mangled = getMangledRegionName(enclosingTemplateName, name);
if ( lookupTemplate(mangled)==null ) {
errMgr.compileTimeError(ErrorType.NO_SUCH_REGION, templateToken, regionT, enclosingTemplateName, name);
return new CompiledST();
}
code.name = mangled;
code.isRegion = true;
code.regionDefType = ST.RegionType.EXPLICIT;
code.templateDefStartToken = regionT;
rawDefineTemplate(mangled, code, regionT);
code.defineArgDefaultValueTemplates(this);
code.defineImplicitlyDefinedTemplates(this);
return code;
}
public CompiledST defineRegion(String enclosingTemplateName, Token regionT, String template, Token templateToken) {
String name = regionT.getText();
template = Misc.trimOneStartingNewline(template);
template = Misc.trimOneTrailingNewline(template);
CompiledST code = compile(getFileName(), enclosingTemplateName, null, template, templateToken);
String mangled = getMangledRegionName(enclosingTemplateName, name);
if ( lookupTemplate(mangled)==null ) {
errMgr.compileTimeError(ErrorType.NO_SUCH_REGION, templateToken, regionT, enclosingTemplateName, name);
return new CompiledST();
}
code.name = mangled;
code.isRegion = true;
code.regionDefType = ST.RegionType.EXPLICIT;
code.templateDefStartToken = regionT;
rawDefineTemplate(mangled, code, regionT);
code.defineArgDefaultValueTemplates(this);
code.defineImplicitlyDefinedTemplates(this);
return code;
}
private ObservableXtextTokenStream toTokenStream(FollowElement element,
ITokenDefProvider tokenDefProvider) {
Iterator<LookAheadTerminal> iter = element.getLookAheadTerminals().iterator();
return new ObservableXtextTokenStream(new TokenSource() {
@Override
public Token nextToken() {
if (iter.hasNext()) {
LookAheadTerminal lookAhead = iter.next();
return lookAhead.getToken();
}
return Token.EOF_TOKEN;
}
@Override
public String getSourceName() {
return "LookAheadTerminalTokenSource";
}
}, tokenDefProvider);
}
public CompiledST defineRegion(String enclosingTemplateName, Token regionT, String template, Token templateToken) {
String name = regionT.getText();
template = Misc.trimOneStartingNewline(template);
template = Misc.trimOneTrailingNewline(template);
CompiledST code = compile(getFileName(), enclosingTemplateName, null, template, templateToken);
String mangled = getMangledRegionName(enclosingTemplateName, name);
if ( lookupTemplate(mangled) ==null ) {
errMgr.compileTimeError(ErrorType.NO_SUCH_REGION,
templateToken,
regionT,
enclosingTemplateName,
name);
return new CompiledST();
}
code.name = mangled;
code.isRegion = true;
code.regionDefType = ST.RegionType.EXPLICIT;
code.templateDefStartToken = regionT;
rawDefineTemplate(mangled, code, regionT);
code.defineArgDefaultValueTemplates(this);
code.defineImplicitlyDefinedTemplates(this);
return code;
}
public void testList() throws Exception {
// ^(nil 101 102 103)
CommonTree r0 = new CommonTree((Token)null);
CommonTree c0, c1, c2;
r0.addChild(c0=new CommonTree(new CommonToken(101)));
r0.addChild(c1=new CommonTree(new CommonToken(102)));
r0.addChild(c2=new CommonTree(new CommonToken(103)));
assertNull(r0.parent);
assertEquals(-1, r0.childIndex);
assertEquals(r0, c0.parent);
assertEquals(0, c0.childIndex);
assertEquals(r0, c1.parent);
assertEquals(1, c1.childIndex);
assertEquals(r0, c2.parent);
assertEquals(2, c2.childIndex);
}
public static List<ActionChunk> translateActionChunk(OutputModelFactory factory,
RuleFunction rf,
String action,
ActionAST node)
{
Token tokenWithinAction = node.token;
ActionTranslator translator = new ActionTranslator(factory, node);
translator.rf = rf;
factory.getGrammar().tool.log("action-translator", "translate " + action);
String altLabel = node.getAltLabel();
if ( rf!=null ) {
translator.nodeContext = rf.ruleCtx;
if ( altLabel!=null ) translator.nodeContext = rf.altLabelCtxs.get(altLabel);
}
ANTLRStringStream in = new ANTLRStringStream(action);
in.setLine(tokenWithinAction.getLine());
in.setCharPositionInLine(tokenWithinAction.getCharPositionInLine());
ActionSplitter trigger = new ActionSplitter(in, translator);
// forces eval, triggers listener methods
trigger.getActionTokens();
return translator.chunks;
}
static XTree getParseTree(CharStream sparqlFile)
throws RecognitionException {
IbmSparqlLexer lex = new IbmSparqlLexer(sparqlFile);
CommonTokenStream tokens = new CommonTokenStream(lex);
IbmSparqlParser parser = new IbmSparqlParser(tokens);
parser.setTreeAdaptor(new CommonTreeAdaptor() {
@Override
public Object create(Token t) {
return new XTree(t);
}
});
IbmSparqlParser.queryUnit_return ret = parser.queryUnit();
XTree ast = (XTree) ret.getTree();
return ast;
}
void checkInvalidRuleRef(Token ruleID) {
String fileName = ruleID.getInputStream().getSourceName();
if ( g.isLexer() && Character.isLowerCase(ruleID.getText().charAt(0)) ) {
g.tool.errMgr.grammarError(ErrorType.PARSER_RULE_REF_IN_LEXER_RULE,
fileName, ruleID, ruleID.getText(), currentRuleName);
}
}
public STLexer(ErrorManager errMgr,
CharStream input,
Token templateToken,
char delimiterStartChar, char delimiterStopChar) {
this.errMgr = errMgr;
this.input = input;
c = (char)input.LA(1); // prime lookahead
this.templateToken = templateToken;
this.delimiterStartChar = delimiterStartChar;
this.delimiterStopChar = delimiterStopChar;
}
@Override
public void importTemplates(Token fileNameToken) {
String msg = "import illegal in group files embedded in STGroupDirs; "+"import " +
fileNameToken.getText() +
" in STGroupDir " +
this.getName();
throw new UnsupportedOperationException(msg);
}
protected void newLeafNode(Token token, EObject grammarElement) {
if (token == null)
return;
final int tokenIndex = token.getTokenIndex();
if (tokenIndex > lastConsumedIndex) {
for (int x = lastConsumedIndex + 1; x < tokenIndex; x++) {
Token hidden = input.get(x);
createLeafNode(hidden, null);
}
lastConsumedIndex = tokenIndex;
lastConsumedNode = createLeafNode(token, grammarElement);
}
}
public void defineTemplateOrRegion(String fullyQualifiedTemplateName, String regionSurroundingTemplateName, Token templateToken, String template, Token nameToken, List<FormalArgument> args) {
try {
if ( regionSurroundingTemplateName!=null ) {
defineRegion(regionSurroundingTemplateName, nameToken, template, templateToken);
}
else {
defineTemplate(fullyQualifiedTemplateName, nameToken, args, template, templateToken);
}
}
catch (STException e) {
// after getting syntax error in a template, we emit msg
// and throw exception to blast all the way out to here.
}
}
private void tokenize(String content) {
CharStream cs = new ANTLRStringStream(content);
CTFLexer lexer = new CTFLexer(cs);
tokens.clear();
for (;;) {
Token token = lexer.nextToken();
if (token == Token.EOF_TOKEN) {
return;
}
tokens.add(token);
}
}
@Override
protected void doSplitToken(Token token, ITokenAcceptor result) {
if (token.getType() == Token.EOF) {
doSplitEofToken(token, result);
} else {
doSplitTokenImpl(token, result);
}
}
@Override
public void importTemplates(Token fileNameToken) {
String msg = "import illegal in group files embedded in STGroupDirs; "+"import " +
fileNameToken.getText() +
" in STGroupDir " +
this.getName();
throw new UnsupportedOperationException(msg);
}
public STLexer(ErrorManager errMgr, CharStream input, Token templateToken, char delimiterStartChar, char delimiterStopChar) {
this.errMgr = errMgr;
this.input = input;
c = (char)input.LA(1); // prime lookahead
this.templateToken = templateToken;
this.delimiterStartChar = delimiterStartChar;
this.delimiterStopChar = delimiterStopChar;
}
public PigParserNode(Token t, String fileName, int lineOffset) {
super(t);
if (t != null && lineOffset > 0) {
t.setLine(t.getLine() + lineOffset);
}
this.fileName = fileName;
}
/**
* defaultConsequence := THEN chunk
* @param rule
*/
public void defaultConsequence(RuleDescrBuilder rule) {
try {
int first = input.index();
Token t = match(input,
DRL6Lexer.ID,
DroolsSoftKeywords.THEN,
null,
DroolsEditorType.KEYWORD);
if (state.failed)
return;
if (state.backtracking == 0) {
rule.getDescr().setConsequenceLocation(t.getLine(),
t.getCharPositionInLine());
helper.emit(Location.LOCATION_RHS);
}
String chunk = getConsequenceCode(first);
// remove the "then" keyword and any subsequent spaces and line breaks
// keep indentation of 1st non-blank line
chunk = chunk.replaceFirst("^then\\s*\\r?\\n?",
"");
rule.rhs(chunk);
} catch (RecognitionException re) {
reportError(re);
}
}
public STLexer(ErrorManager errMgr, CharStream input, Token templateToken, char delimiterStartChar, char delimiterStopChar) {
this.errMgr = errMgr;
this.input = input;
c = (char)input.LA(1); // prime lookahead
this.templateToken = templateToken;
this.delimiterStartChar = delimiterStartChar;
this.delimiterStopChar = delimiterStopChar;
}
/**
* Constructor that preserves the char offset
*
* @param oldToken
*/
public DroolsToken(Token oldToken) {
super(oldToken);
if (null != oldToken
&& (oldToken.getClass().equals(CommonToken.class) || oldToken
.getClass().equals(DroolsToken.class))) {
start = ((CommonToken) oldToken).getStartIndex();
stop = ((CommonToken) oldToken).getStopIndex();
}
}
Token mTEXT() {
boolean modifiedText = false;
StringBuilder buf = new StringBuilder();
while ( c != EOF && c != delimiterStartChar ) {
if ( c=='\r' || c=='\n') break;
if ( c=='}' && subtemplateDepth>0 ) break;
if ( c=='\\' ) {
if ( input.LA(2)=='\\' ) { // convert \\ to \
consume(); consume(); buf.append('\\');
modifiedText = true;
continue;
}
if ( input.LA(2)==delimiterStartChar ||
input.LA(2)=='}' )
{
modifiedText = true;
consume(); // toss out \ char
buf.append(c); consume();
}
else {
buf.append(c);
consume();
}
continue;
}
buf.append(c);
consume();
}
if ( modifiedText ) return newToken(TEXT, buf.toString());
else return newToken(TEXT);
}
protected void assertLexing(String input, Pair<String,String>... expectedTokens) {
Lexer lexer = new InternalXtendLexer(null);
CharStream stream = new ANTLRStringStream(input);
lexer.setCharStream(stream);
XtextTokenStream tokenStream = new XtextTokenStream(lexer, tokenDefProvider);
List<?> tokens = tokenStream.getTokens();
assertEquals(input + " / " + tokens, expectedTokens.length, tokens.size());
for(int i = 0;i < tokens.size(); i++) {
Token token = (Token) tokens.get(i);
assertEquals(token.toString(), expectedTokens[i].getFirst(), token.getText());
final String expected = expectedTokens[i].getSecond();
String actual = tokenDefProvider.getTokenDefMap().get(token.getType());
assertEquals("expected "+expected+" but was "+actual, expected, actual);
}
}