下面列出了怎么用org.antlr.runtime.CommonToken的API类实例代码及写法，或者点击链接到github查看源代码。
@Override
protected void announce(Token start, Token stop, AbstractElement element) {
if (start != null && start != Token.EOF_TOKEN) {
if (start == stop) {
announce(start, element);
} else {
CommonToken castedStart = (CommonToken) start;
if (stop == null) { // possible error condition
if (start.getTokenIndex() == state.lastErrorIndex) {
return;
}
}
CommonToken castedEnd = (CommonToken) stop;
Integer newType = rewriter.rewrite(castedStart, element);
if (newType != null && castedEnd != null && castedEnd != Token.EOF_TOKEN) {
LazyTokenStream castedInput = (LazyTokenStream) this.input;
for (int i = castedStart.getTokenIndex() + 1; i < castedEnd.getTokenIndex(); i++) {
Token token = castedInput.get(i);
if (token.getChannel() != Token.HIDDEN_CHANNEL)
token.setType(newType);
}
castedEnd.setType(newType);
}
}
}
}
public void testDupTree() throws Exception {
// ^(101 ^(102 103 ^(106 107) ) 104 105)
CommonTree r0 = new CommonTree(new CommonToken(101));
CommonTree r1 = new CommonTree(new CommonToken(102));
r0.addChild(r1);
r1.addChild(new CommonTree(new CommonToken(103)));
Tree r2 = new CommonTree(new CommonToken(106));
r2.addChild(new CommonTree(new CommonToken(107)));
r1.addChild(r2);
r0.addChild(new CommonTree(new CommonToken(104)));
r0.addChild(new CommonTree(new CommonToken(105)));
CommonTree dup = (CommonTree)(new CommonTreeAdaptor()).dupTree(r0);
assertNull(dup.parent);
assertEquals(-1, dup.childIndex);
dup.sanityCheckParentAndChildIndexes();
}
public void testList2() throws Exception {
// Add child ^(nil 101 102 103) to root 5
// should pull 101 102 103 directly to become 5's child list
CommonTree root = new CommonTree(new CommonToken(5));
// child tree
CommonTree r0 = new CommonTree((Token)null);
CommonTree c0, c1, c2;
r0.addChild(c0=new CommonTree(new CommonToken(101)));
r0.addChild(c1=new CommonTree(new CommonToken(102)));
r0.addChild(c2=new CommonTree(new CommonToken(103)));
root.addChild(r0);
assertNull(root.parent);
assertEquals(-1, root.childIndex);
// check children of root all point at root
assertEquals(root, c0.parent);
assertEquals(0, c0.childIndex);
assertEquals(root, c0.parent);
assertEquals(1, c1.childIndex);
assertEquals(root, c0.parent);
assertEquals(2, c2.childIndex);
}
public void setCursorToGrammarElement(Project project, PreviewState previewState, int offset) {
Token tokenUnderCursor = ParsingUtils.getTokenUnderCursor(previewState, offset);
if ( tokenUnderCursor==null ) {
return;
}
PreviewParser parser = (PreviewParser) previewState.parsingResult.parser;
Integer atnState = parser.inputTokenToStateMap.get(tokenUnderCursor);
if ( atnState==null ) { // likely an error token
//LOG.error("no ATN state for input token " + tokenUnderCursor);
return;
}
Interval region = previewState.g.getStateToGrammarRegion(atnState);
CommonToken token =
(CommonToken) previewState.g.tokenStream.get(region.a);
jumpToGrammarPosition(project, token.getStartIndex());
}
/** for testing */
public CompiledST defineTemplate(String templateName, String template) {
if ( templateName.charAt(0) !='/' ) templateName = "/"+templateName;
try {
CompiledST impl = defineTemplate(templateName,
new CommonToken(GroupParser.ID, templateName),
null,
template,
null);
return impl;
}
catch (STException se) {
// we have reported the error; the exception just blasts us
// out of parsing this template
}
return null;
}
public void setCursorToGrammarRule(Project project, PreviewState previewState, int offset) {
Token tokenUnderCursor = ParsingUtils.getTokenUnderCursor(previewState, offset);
if ( tokenUnderCursor==null ) {
return;
}
ParseTree tree = previewState.parsingResult.tree;
TerminalNode nodeWithToken =
(TerminalNode) ParsingUtils.getParseTreeNodeWithToken(tree, tokenUnderCursor);
if ( nodeWithToken==null ) {
// hidden token
return;
}
ParserRuleContext parent = (ParserRuleContext) nodeWithToken.getParent();
int ruleIndex = parent.getRuleIndex();
Rule rule = previewState.g.getRule(ruleIndex);
GrammarAST ruleNameNode = (GrammarAST) rule.ast.getChild(0);
int start = ((CommonToken) ruleNameNode.getToken()).getStartIndex();
jumpToGrammarPosition(project, start);
}
/** for testing */
public CompiledST defineTemplate(String templateName, String template) {
if ( templateName.charAt(0) !='/' ) templateName = "/"+templateName;
try {
CompiledST impl = defineTemplate(templateName,
new CommonToken(GroupParser.ID, templateName),
null,
template,
null);
return impl;
}
catch (STException se) {
// we have reported the error; the exception just blasts us
// out of parsing this template
}
return null;
}
/**
* Create a new token from the given prototype. Any argument besides the prototype is optional and
* will be ignored if its value is <code>null</code>.
*/
protected CommonToken createToken(CommonToken prototype, String text,
Integer charPosInLine, Integer channel, Integer start, Integer stop, Integer type) {
if (prototype == null)
throw new IllegalArgumentException("Prototype may not be null.");
CommonToken result = new CommonToken(prototype);
if (text != null)
result.setText(text);
if (charPosInLine != null)
result.setCharPositionInLine(charPosInLine.intValue());
if (channel != null)
result.setChannel(channel.intValue());
if (start != null)
result.setStartIndex(start.intValue());
if (stop != null)
result.setStopIndex(stop.intValue());
if (type != null)
result.setType(type.intValue());
return result;
}
public void testAoverB() throws Exception {
Tree t = new CommonTree(new CommonToken(101));
t.addChild(new CommonTree(new CommonToken(102)));
TreeNodeStream stream = newStream(t);
String expecting = " 101 102";
String found = toNodesOnlyString(stream);
assertEquals(expecting, found);
expecting = " 101 2 102 3";
found = stream.toString();
assertEquals(expecting, found);
}
/**
* Constructor that preserves the char offset
*
* @param oldToken
*/
public DroolsToken(Token oldToken) {
super(oldToken);
if (null != oldToken
&& (oldToken.getClass().equals(CommonToken.class) || oldToken
.getClass().equals(DroolsToken.class))) {
start = ((CommonToken) oldToken).getStartIndex();
stop = ((CommonToken) oldToken).getStopIndex();
}
}
SemanticException(IntStream input, Token token, String errorMessage, Object... messageArguments) {
super();
this.input = input;
this.token = token;
this.index = ((CommonToken)token).getStartIndex();
this.line = token.getLine();
this.charPositionInLine = token.getCharPositionInLine();
this.errorMessage = String.format(errorMessage, messageArguments);
}
void setStart( BaseDescr descr,
Token first ) {
if ( descr != null && first != null ) {
descr.setLocation( first.getLine(),
first.getCharPositionInLine() );
descr.setStartCharacter( ((CommonToken) first).getStartIndex() );
}
}
@Override
public CompiledST loadTemplateFile(String prefix, String unqualifiedFileName, CharStream templateStream) {
String template = templateStream.substring(0, templateStream.size()- 1);
String templateName = Misc.getFileNameNoSuffix(unqualifiedFileName);
String fullyQualifiedTemplateName = prefix+templateName;
CompiledST impl = new Compiler(this).compile(fullyQualifiedTemplateName, template);
CommonToken nameT = new CommonToken(STLexer.SEMI); // Seems like a hack, best I could come up with.
nameT.setInputStream(templateStream);
rawDefineTemplate(fullyQualifiedTemplateName, impl, nameT);
impl.defineImplicitlyDefinedTemplates(this);
return impl;
}
void setEnd( DescrBuilder< ? , ? > db ) {
Token last = input.LT( -1 );
if ( db != null && last != null ) {
int endLocation = last.getText() != null ? last.getCharPositionInLine() + last.getText().length() - 1 : last.getCharPositionInLine();
db.endCharacter( ((CommonToken) last).getStopIndex() + 1 ).endLocation( last.getLine(),
endLocation );
}
}
SemanticException(IntStream input, Token token, String errorMessage, Object... messageArguments) {
super();
this.input = input;
this.token = token;
this.index = ((CommonToken)token).getStartIndex();
this.line = token.getLine();
this.charPositionInLine = token.getCharPositionInLine();
this.errorMessage = String.format(errorMessage, messageArguments);
}
public void testTwoChildrenOfNilRoot() throws Exception {
CommonTree root_0 = (CommonTree)adaptor.nil();
CommonTree t = new V(101, 2);
CommonTree u = new V(new CommonToken(102,"102"));
adaptor.addChild(root_0, t);
adaptor.addChild(root_0, u);
assertNull(root_0.parent);
assertEquals(-1, root_0.childIndex);
assertEquals(0, t.childIndex);
assertEquals(1, u.childIndex);
}
@Override
public Token nextToken() {
Token result = super.nextToken();
if (result instanceof CommonToken) {
nextOffset = ((CommonToken) result).getStopIndex() + 1;
} else {
throw new IllegalArgumentException(String.valueOf(result));
}
return result;
}
public void testMarkRewindNested() throws Exception {
// ^(101 ^(102 103 ^(106 107) ) 104 105)
// stream has 7 real + 6 nav nodes
// Sequence of types: 101 DN 102 DN 103 106 DN 107 UP UP 104 105 UP EOF
Tree r0 = new CommonTree(new CommonToken(101));
Tree r1 = new CommonTree(new CommonToken(102));
r0.addChild(r1);
r1.addChild(new CommonTree(new CommonToken(103)));
Tree r2 = new CommonTree(new CommonToken(106));
r2.addChild(new CommonTree(new CommonToken(107)));
r1.addChild(r2);
r0.addChild(new CommonTree(new CommonToken(104)));
r0.addChild(new CommonTree(new CommonToken(105)));
CommonTreeNodeStream stream = new CommonTreeNodeStream(r0);
int m = stream.mark(); // MARK at start
stream.consume(); // consume 101
stream.consume(); // consume DN
int m2 = stream.mark(); // MARK on 102
stream.consume(); // consume 102
stream.consume(); // consume DN
stream.consume(); // consume 103
stream.consume(); // consume 106
stream.rewind(m2); // REWIND to 102
assertEquals(102, ((Tree)stream.LT(1)).getType());
stream.consume();
assertEquals(Token.DOWN, ((Tree)stream.LT(1)).getType());
stream.consume();
// stop at 103 and rewind to start
stream.rewind(m); // REWIND to 101
assertEquals(101, ((Tree)stream.LT(1)).getType());
stream.consume();
assertEquals(Token.DOWN, ((Tree)stream.LT(1)).getType());
stream.consume();
assertEquals(102, ((Tree)stream.LT(1)).getType());
stream.consume();
assertEquals(Token.DOWN, ((Tree)stream.LT(1)).getType());
}
/**
* JSDoc comments are identified by the lexer as normal ML comments to simplify the ASI code. If a comment starts
* with the sequence @çode{/**} it is remapped to a JS Doc token.
*/
@Override
protected TokenInfo createTokenInfo(CommonToken token) {
if (token.getType() == InternalN4JSParser.RULE_ML_COMMENT) {
String text = token.getText();
if (text.length() > 4 && text.startsWith("/**") && text.charAt(3) != '*') {
CommonToken jsDoc = new CommonToken(token);
jsDoc.setType(JS_DOC_TOKEN);
return super.createTokenInfo(jsDoc);
}
}
return super.createTokenInfo(token);
}
@Override
public CompiledST loadTemplateFile(String prefix, String unqualifiedFileName, CharStream templateStream) {
String template = templateStream.substring(0, templateStream.size()- 1);
String templateName = Misc.getFileNameNoSuffix(unqualifiedFileName);
String fullyQualifiedTemplateName = prefix+templateName;
CompiledST impl = new Compiler(this).compile(fullyQualifiedTemplateName, template);
CommonToken nameT = new CommonToken(STLexer.SEMI); // Seems like a hack, best I could come up with.
nameT.setInputStream(templateStream);
rawDefineTemplate(fullyQualifiedTemplateName, impl, nameT);
impl.defineImplicitlyDefinedTemplates(this);
return impl;
}
/**
* The second pass over the given input. If the input ends with an ASI, the semicolon is removed and it is parsed
* again since the production may have skipped the ASI if more input was present. Same applies for the opposite
* direction, e.g if it does not end with an ASI but the prev token suggests that there may have been a semicolon,
* it is inserted.
*/
private void adjustASIAndCollectFollowElements(CustomInternalN4JSParser previousParser,
String ruleName,
boolean strict,
Set<FollowElement> result) {
ObservableXtextTokenStream tokens = (ObservableXtextTokenStream) previousParser.getTokenStream();
int lastTokenIndex = tokens.size() - 1;
if (lastTokenIndex >= 0) {
CommonToken lastToken = (CommonToken) tokens.LT(-1);
@SuppressWarnings("unchecked")
List<Token> tokenList = tokens.getTokens();
if (lastToken == null) {
return; // TODO ask Sebastian why and how this can happen... it happens!
}
if (shouldSkipASI(lastToken)) {
// Some statements may not span multiple lines, e.g between the return keyword
// and the returned expression, there may not be an ASI. Filter these locations.
if (maySkipASI(lastToken, tokens)) {
tokenList.remove(lastTokenIndex);
result.addAll(resetAndGetFollowElements(tokens, ruleName, strict));
// If a postfix operator sneaked into the result, remove it since
// we'd have produce an ASI before that
removePostfixOperator(result);
}
} else if (shouldAddSyntheticSemicolon(previousParser, lastTokenIndex, lastToken)) {
CommonToken token = new CommonToken(semi);
tokenList.add(token);
result.addAll(resetAndGetFollowElements(tokens, ruleName, strict));
// Same here, if we had added an ASI, the postfix operator would be rendered
// invalid, remove it.
removePostfixOperator(result);
}
}
}
/** Dup token too since we overwrite during LR rule transform */
@Override
public RuleRefAST dupNode() {
RuleRefAST r = new RuleRefAST(this);
// In LR transform, we alter original token stream to make e -> e[n]
// Since we will be altering the dup, we need dup to have the
// original token. We can set this tree (the original) to have
// a new token.
r.token = this.token;
this.token = new CommonToken(r.token);
return r;
}
/**
* Returns true if the ASI at the end should be skipped for a second pass of the CA parser. Otherwise false.
*/
private boolean shouldSkipASI(CommonToken lastToken) {
if (lastToken.getType() == eol) {
return true;
}
if (lastToken.getType() == semi && lastToken.getText() != null && !";".equals(lastToken.getText())) {
return true;
}
return false;
}
@Test public void testLexerError_01() throws Exception {
String model = "a /* incomplete comment *";
InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
lexer.setCharStream(new ANTLRStringStream(model));
CommonTokenStream stream = new CommonTokenStream(lexer);
@SuppressWarnings("unchecked")
List<CommonToken> tokens = stream.getTokens();
assertEquals(tokens.toString(), 3, tokens.size());
assertEquals("a", tokens.get(0).getText());
assertEquals(" ", tokens.get(1).getText());
assertEquals("/* incomplete comment *", tokens.get(2).getText());
assertEquals(0, tokens.get(2).getType());
}
/**
* Skips the given leaf as it's hidden. If it was the last token to be returned, a hidden token may be syntesized if
* would affect the semicolon insertion.
*/
private Token processHiddenToken(ILeafNode leaf) {
Token result = nextToken();
if (result == Token.EOF_TOKEN && Strings.countLineBreaks(leaf.getText()) > 0) {
next = result;
CommonToken hidden = new CommonToken(tokenTypeMapper.getInternalTokenType(leaf), leaf.getText());
hidden.setChannel(Token.HIDDEN_CHANNEL);
return hidden;
}
return result;
}
/**
* Produce an Antlr token for the prefix of the given leaf that overlaps the requested region
*
* @see #endOffset
*/
private Token toPrefixToken(ILeafNode leaf) {
Lexer lexer = new InternalN4JSLexer();
String text = leaf.getText();
String prefix = text.substring(0, endOffset - leaf.getTotalOffset());
ANTLRStringStream stream = new ANTLRStringStream(prefix);
lexer.setCharStream(stream);
Token nextToken = lexer.nextToken();
// copy to get rid of the reference to the stream again
return new CommonToken(nextToken.getType(), nextToken.getText());
}
/** for testing */
public CompiledST defineTemplate(String name, String argsS, String template) {
if ( name.charAt(0)!='/' ) name = "/"+name;
String[] args = argsS.split(",");
List<FormalArgument> a = new ArrayList<FormalArgument>();
for (String arg : args) {
a.add(new FormalArgument(arg));
}
return defineTemplate(name, new CommonToken(GroupParser.ID, name), a, template, null);
}
public void testReplaceInMiddle() throws Exception {
CommonTree t = new CommonTree(new CommonToken(99, "a"));
t.addChild(new CommonTree(new CommonToken(99, "b")));
t.addChild(new CommonTree(new CommonToken(99, "c"))); // index 1
t.addChild(new CommonTree(new CommonToken(99, "d")));
CommonTree newChild = new CommonTree(new CommonToken(99,"x"));
t.replaceChildren(1, 1, newChild);
String expecting = "(a b x d)";
assertEquals(expecting, t.toStringTree());
t.sanityCheckParentAndChildIndexes();
}
/** for testing */
public CompiledST defineTemplate(String templateName, String template) {
if ( templateName.charAt(0)!='/' ) templateName = "/"+templateName;
try {
CompiledST impl = defineTemplate(templateName, new CommonToken(GroupParser.ID, templateName), null, template, null);
return impl;
}
catch (STException se) {
// we have reported the error; the exception just blasts us
// out of parsing this template
}
return null;
}
@Override
public void consumeToken(Token token) {
if (backtracking > 0 || resync) {
return;
}
if (debug_tokens) {
CommonToken ct = (CommonToken) token;
int[] ctr = CommonTokenUtil.getCommonTokenOffsetRange(ct);
System.out.println(token + "(" + ctr[0] + "-" + ctr[1] + ")");
}
//ignore the closing EOF token, we do not want it
//it the parse tree
if (token.getType() == Css3Lexer.EOF) {
return;
}
//also ignore error tokens - they are added as children of ErrorNode-s in the recognitionException(...) method
if (token.getType() == Token.INVALID_TOKEN_TYPE) {
return;
}
lastConsumedToken = (CommonToken) token;
RuleNode ruleNode = callStack.peek();
TokenNode elementNode = new TokenNode(source, (CommonToken) token);
elementNode.hiddenTokens = this.hiddenTokens;
hiddenTokens.clear();
ruleNode.addChild(elementNode);
updateFirstTokens(ruleNode, lastConsumedToken);
}