下面列出了怎么用org.antlr.runtime.Lexer的API类实例代码及写法，或者点击链接到github查看源代码。
public void testSimpleSupportsStatementTokens() throws Exception {
String source = "@supports not (text-align: center) {} ";
Lexer lexer = createLexer(source);
assertANTLRToken("@supports", Css3Lexer.SUPPORTS_SYM, lexer.nextToken());
assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
assertANTLRToken("not" ,Css3Lexer.NOT, lexer.nextToken());
assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
assertANTLRToken("(" ,Css3Lexer.LPAREN, lexer.nextToken());
assertANTLRToken("text-align" ,Css3Lexer.IDENT, lexer.nextToken());
assertANTLRToken(":", Css3Lexer.COLON, lexer.nextToken());
assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
assertANTLRToken("center" ,Css3Lexer.IDENT, lexer.nextToken());
assertANTLRToken(")" ,Css3Lexer.RPAREN, lexer.nextToken());
assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
assertANTLRToken("{" ,Css3Lexer.LBRACE, lexer.nextToken());
assertANTLRToken("}" ,Css3Lexer.RBRACE, lexer.nextToken());
}
private String lexerResult(Lexer lexer, Map<Integer, String> tokenNames, CharSequence text) {
lexer.setCharStream(new ANTLRStringStream(text.toString()));
List<String> result = new ArrayList<>();
while (true) {
Token token = lexer.nextToken();
if (token == Token.EOF_TOKEN) {
return Joiner.on(System.lineSeparator()).join(result);
}
Object nameOrType = null;
String tokenName = tokenNames.get(token.getType());
if (tokenName != null) {
nameOrType = tokenName;
} else {
nameOrType = Integer.valueOf(token.getType());
}
result.add(nameOrType + " \'" + token.getText() + "\'");
}
}
public void testCaseInsensivityOfSomeAtTokens() throws Exception {
String source = "@FONT-face @charset @CHARSET @charSeT ";
Lexer lexer = createLexer(source);
assertANTLRToken("@FONT-face" ,Css3Lexer.FONT_FACE_SYM, lexer.nextToken());
assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
assertANTLRToken("@charset" ,Css3Lexer.CHARSET_SYM, lexer.nextToken());
assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
assertANTLRToken("@CHARSET" ,Css3Lexer.CHARSET_SYM, lexer.nextToken());
assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
assertANTLRToken("@charSeT" ,Css3Lexer.CHARSET_SYM, lexer.nextToken());
}
public void testExtendOnlySelector2() throws Exception {
String source = "#context a%extreme {";
Lexer lexer = createLexer(source);
assertANTLRToken(null ,Css3Lexer.HASH, lexer.nextToken());
assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
assertANTLRToken(null ,Css3Lexer.IDENT, lexer.nextToken());
assertANTLRToken(null ,Css3Lexer.SASS_EXTEND_ONLY_SELECTOR, lexer.nextToken());
assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
assertANTLRToken(null ,Css3Lexer.LBRACE, lexer.nextToken());
}
public void testSASS_Else() throws Exception {
String source = "@else cau";
Lexer lexer = createLexer(source);
assertANTLRToken(null ,Css3Lexer.SASS_ELSE, lexer.nextToken());
assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
assertANTLRToken(null ,Css3Lexer.IDENT, lexer.nextToken());
}
public void testLineComment() throws Exception {
String source = "//comment\na";
Lexer lexer = createLexer(source);
assertANTLRToken("//comment" ,Css3Lexer.LINE_COMMENT, lexer.nextToken());
assertANTLRToken(null ,Css3Lexer.NL, lexer.nextToken());
assertANTLRToken("a",Css3Lexer.IDENT, lexer.nextToken());
}
public void testLexingOfPercentageWithoutNumberPrefix() throws Exception {
String source = "font: %/20 ";
Lexer lexer = createLexer(source);
assertANTLRToken("font" ,Css3Lexer.IDENT, lexer.nextToken());
assertANTLRToken(":" ,Css3Lexer.COLON, lexer.nextToken());
assertANTLRToken(" " ,Css3Lexer.WS, lexer.nextToken());
assertANTLRToken("%" , Css3Lexer.PERCENTAGE_SYMBOL, lexer.nextToken());
assertANTLRToken("/" ,Css3Lexer.SOLIDUS, lexer.nextToken());
assertANTLRToken("20" , Css3Lexer.NUMBER, lexer.nextToken());
assertANTLRToken(" " , Css3Lexer.WS, lexer.nextToken());
}
public Block(Architecture arch) {
this.architecture = arch;
Lexer lex = null;
Assembler assembler = null;
switch (architecture) {
// no parsing necessary if we're writing bytes to memory
case raw:
return;
// arm architectures trickle down
case armv7:
case armv9:
// lex = new armLexer(new ANTLRStringStream(program));
// assembler = new armAssembler(new CommonTokenStream(lex));
break;
case x64:
lex = new x64Lexer(new ANTLRStringStream(program));
assembler = new x64Assembler(new CommonTokenStream(lex));
break;
case x86:
break;
default:
break;
}
try {
if (assembler != null) {
assembler.start();
this.instructions = assembler.getMachineCode();
} else {
throw new JssemblyException("Assembler not found for architecture: " + this.architecture.name());
}
} catch (RecognitionException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public Lexer[] getDelegates() {
return new Lexer[] {};
}
public Lexer[] getDelegates() {
return new Lexer[] {};
}
public Lexer[] getDelegates() {
return new Lexer[] {};
}
@Override
public Lexer get() {
return new InternalN4JSLexer(null);
}
@Override
public Lexer get() {
return new InternalN4JSLexer(null);
}
public void testLexingURLToken() throws Exception {
String source = "url(http://fonts.googleapis.com/css?family=Syncopate) ";
Lexer lexer = createLexer(source);
assertANTLRToken(null ,Css3Lexer.URI, lexer.nextToken());
}
public void testSassVar() throws Exception {
String source = "$var ";
Lexer lexer = createLexer(source);
assertANTLRToken(null ,Css3Lexer.SASS_VAR, lexer.nextToken());
}
public void testCPLineComment() throws Exception {
String source = "//line comment\n";
Lexer lexer = createLexer(source);
assertANTLRToken(null ,Css3Lexer.LINE_COMMENT, lexer.nextToken());
}
public void testExtendOnlySelector() throws Exception {
String source = "body%my";
Lexer lexer = createLexer(source);
assertANTLRToken(null ,Css3Lexer.IDENT, lexer.nextToken());
assertANTLRToken(null ,Css3Lexer.SASS_EXTEND_ONLY_SELECTOR, lexer.nextToken());
}
public void testLineCommentAtTheFileEnd() throws Exception {
String source = "//comment";
Lexer lexer = createLexer(source);
assertANTLRToken("//comment" ,Css3Lexer.LINE_COMMENT, lexer.nextToken());
}
public void testLESS_JS_STRING() throws Exception {
String source = "`\"hello\".toUpperCase() + '!'`;";
Lexer lexer = createLexer(source);
assertANTLRToken("`\"hello\".toUpperCase() + '!'`" ,Css3Lexer.LESS_JS_STRING, lexer.nextToken());
assertANTLRToken(";" ,Css3Lexer.SEMI, lexer.nextToken());
}
public void testURLWithAtSign() throws Exception {
String source = "url([email protected])";
Lexer lexer = createLexer(source);
assertANTLRToken("url([email protected])" ,Css3Lexer.URI, lexer.nextToken());
}
public void testIssue236649() throws Exception {
String source = "url(http://fonts.googleapis.com/css?family=Josefin+Sans|Sigmar+One|Maven+Pro)";
Lexer lexer = createLexer(source);
assertANTLRToken("url(http://fonts.googleapis.com/css?family=Josefin+Sans|Sigmar+One|Maven+Pro)", Css3Lexer.URI, lexer.nextToken());
}
protected String lex(Lexer lexer, InputStream tokensStream, CharSequence text) throws Exception {
InputStreamReader _inputStreamReader = new InputStreamReader(tokensStream);
LinkedHashMap<Integer, String> tokenNames = tokenNames(_inputStreamReader);
return lexerResult(lexer, tokenNames, text);
}
public String getLexerErrorMessage(Token invalidToken) {
if (tokenSource instanceof org.eclipse.xtext.parser.antlr.Lexer) {
return ((org.eclipse.xtext.parser.antlr.Lexer) tokenSource).getErrorMessage(invalidToken);
}
return (invalidToken.getType() == Token.INVALID_TOKEN_TYPE) ? "Invalid token " + invalidToken.getText() : null;
}
public static <T extends Lexer> LexerProvider<T> create(Class<T> clazz) {
return new LexerProvider<T>(clazz);
}
public Lexer[] getDelegates() {
return new Lexer[] {};
}
public Lexer[] getDelegates() {
return new Lexer[] {};
}
public void testSupportsToken() throws Exception {
String source = "@supports ";
Lexer lexer = createLexer(source);
assertANTLRToken("@supports", Css3Lexer.SUPPORTS_SYM, lexer.nextToken());
}
public void testRemUnit() throws Exception {
String source = "10rad 20rem ";
Lexer lexer = createLexer(source);
assertANTLRToken(null ,Css3Lexer.ANGLE, lexer.nextToken());
assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
assertANTLRToken(null ,Css3Lexer.REM, lexer.nextToken());
assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
}
protected abstract Lexer lexer();