下面列出了怎么用org.antlr.runtime.CommonTokenStream的API类实例代码及写法,或者点击链接到github查看源代码。
/** Parse an expression from text */
public ConstraintConnectiveDescr parse( final String text ) {
ConstraintConnectiveDescr constraint = null;
try {
DRLLexer lexer = DRLFactory.getDRLLexer(new ANTLRStringStream(text), languageLevel);
CommonTokenStream input = new CommonTokenStream( lexer );
RecognizerSharedState state = new RecognizerSharedState();
helper = new ParserHelper( input, state, languageLevel );
DRLExpressions parser = DRLFactory.getDRLExpressions(input, state, helper, languageLevel);
parser.setBuildDescr( true );
parser.setLeftMostExpr( null ); // setting initial value just in case
BaseDescr expr = parser.conditionalOrExpression();
if ( expr != null && !parser.hasErrors() ) {
constraint = ConstraintConnectiveDescr.newAnd();
constraint.addOrMerge( expr );
}
} catch ( RecognitionException e ) {
helper.reportError( e );
}
return constraint;
}
/**
* Parses a snippet of maple code and returns a list of CFG nodes that implement the returned maple expressions.
*
* @param graph The control flow graph the new nodes should be created in.
* @param mapleCode The code returned by Maple.
* @return A list of control flow nodes modeling the returned code.
*/
List<AssignmentNode> parseMapleCode(ControlFlowGraph graph, String mapleCode) {
oldMinVal = new HashMap<String, String>();
oldMaxVal = new HashMap<String, String>();
/* fill the Maps with the min and maxvalues from the nodes */
for (Variable v : graph.getInputVariables()) {
if (v.getMinValue() != null)
oldMinVal.put(v.getName(), v.getMinValue());
if (v.getMaxValue() != null)
oldMaxVal.put(v.getName(), v.getMaxValue());
}
MapleLexer lexer = new MapleLexer(new ANTLRStringStream(mapleCode));
MapleParser parser = new MapleParser(new CommonTokenStream(lexer));
try {
MapleParser.program_return result = parser.program();
MapleTransformer transformer = new MapleTransformer(new CommonTreeNodeStream(result.getTree()));
return transformer.script(graph, oldMinVal, oldMaxVal);
} catch (RecognitionException e) {
throw new RuntimeException(e);
}
}
public void generateDependencies(String text, EGTask task) throws Exception {
try {
this.task = task;
DependenciesLexer lex = new DependenciesLexer(new ANTLRStringStream(text));
CommonTokenStream tokens = new CommonTokenStream(lex);
DependenciesParser g = new DependenciesParser(tokens);
try {
g.setGenerator(this);
g.prog();
} catch (RecognitionException ex) {
logger.error("Unable to load mapping task: " + ex.getMessage());
throw new ParserException(ex);
}
} catch (Exception e) {
e.printStackTrace();
logger.error(e.getLocalizedMessage());
throw new ParserException(e);
}
}
/**
* Apply a lexer to a source and check that the token stream produced
* is as expected.
* @param source the source code
* @param expected the expected token stream
*/
public void lexAndCheck(final String source, final String expected) {
try {
CommonTokenStream ts = lex(source);
ts.fill();
StringBuilder sb = new StringBuilder();
for (Object token : ts.getTokens()) {
sb.append(toString((Token) token));
}
CobolStructureLexerImpl lexer = (CobolStructureLexerImpl) ts.getTokenSource();
if (lexer.getErrorHandler().getErrorMessages().size() > 0) {
throw new RecognizerException(
lexer.getErrorHandler().getErrorMessages().get(0));
}
assertEquals(expected, sb.toString());
} catch (RecognizerException e) {
e.printStackTrace();
fail(e.getMessage());
}
}
private LogicalExpression parseExpression(String expr) throws RecognitionException, IOException{
ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr));
CommonTokenStream tokens = new CommonTokenStream(lexer);
// tokens.fill();
// for(Token t : (List<Token>) tokens.getTokens()){
// System.out.println(t + "" + t.getType());
// }
// tokens.rewind();
ExprParser parser = new ExprParser(tokens);
parse_return ret = parser.parse();
return ret.e;
}
@Override
public Operand createOperand(String operand) throws MoqlException {
// TODO Auto-generated method stub
Validate.notEmpty(operand, "Parameter 'operand' is empty!");
Operand pseudoOperand = createPseudoOperand(operand);
if (pseudoOperand != null)
return pseudoOperand;
try {
ANTLRInputStream is = new ANTLRInputStream(new ByteArrayInputStream(operand.getBytes()));
OperandLexer lexer = new OperandLexer(is);
CommonTokenStream tokens = new CommonTokenStream(lexer);
OperandParser parser = new OperandParser(tokens);
parser.setFunctionFactory(functionFactory);
return parser.operand();
} catch (Exception e) {
// TODO Auto-generated catch block
throw new MoqlException(
StringFormater.format("Create operand '{}' failed!", operand), e);
}
}
/**
*
* @param moql
* @return SelectorMetadata or SetlectorMetadata
*/
public static SelectorDefinition parseMoql(String moql) throws MoqlException {
Validate.notEmpty(moql, "Parameter 'moql' is empty!");
try {
ANTLRInputStream is = new ANTLRInputStream(new ByteArrayInputStream(
moql.getBytes()));
SelectorLexer lexer = new SelectorLexer(is);
CommonTokenStream tokens = new CommonTokenStream(lexer);
SelectorParser parser = new SelectorParser(tokens);
return parser.selector();
} catch (Exception e) {
// TODO Auto-generated catch block
throw new MoqlException(StringFormater.format("Parse moql '{}' failed!",
moql), e);
}
}
/** Load a group file with full path {@code fileName}; it's relative to root by {@code prefix}. */
public void loadGroupFile(String prefix, String fileName) {
if ( verbose ) System.out.println(this.getClass().getSimpleName()+".loadGroupFile(group-file-prefix="+prefix+", fileName="+fileName+")");
GroupParser parser;
try {
URL f = new URL(fileName);
ANTLRInputStream fs = new ANTLRInputStream(f.openStream(), encoding);
GroupLexer lexer = new GroupLexer(fs);
fs.name = fileName;
CommonTokenStream tokens = new CommonTokenStream(lexer);
parser = new GroupParser(tokens);
parser.group(this, prefix);
}
catch (Exception e) {
errMgr.IOError(null, ErrorType.CANT_LOAD_GROUP_FILE, e, fileName);
}
}
@Override
public void load() {
if ( alreadyLoaded ) return;
alreadyLoaded = true;
GroupParser parser;
try {
ANTLRStringStream fs = new ANTLRStringStream(text);
fs.name = sourceName;
GroupLexer lexer = new GroupLexer(fs);
CommonTokenStream tokens = new CommonTokenStream(lexer);
parser = new GroupParser(tokens);
// no prefix since this group file is the entire group, nothing lives
// beneath it.
parser.group(this, "/");
}
catch (Exception e) {
errMgr.IOError(null, ErrorType.CANT_LOAD_GROUP_FILE, e, "<string>");
}
}
/** Load a group file with full path {@code fileName}; it's relative to root by {@code prefix}. */
public void loadGroupFile(String prefix, String fileName) {
if ( verbose ) System.out.println(this.getClass().getSimpleName()+".loadGroupFile(group-file-prefix="+prefix+", fileName="+fileName+")");
GroupParser parser;
try {
URL f = new URL(fileName);
ANTLRInputStream fs = new ANTLRInputStream(f.openStream(), encoding);
GroupLexer lexer = new GroupLexer(fs);
fs.name = fileName;
CommonTokenStream tokens = new CommonTokenStream(lexer);
parser = new GroupParser(tokens);
parser.group(this, prefix);
}
catch (Exception e) {
errMgr.IOError(null, ErrorType.CANT_LOAD_GROUP_FILE, e, fileName);
}
}
/** Load a group file with full path {@code fileName}; it's relative to root by {@code prefix}. */
public void loadGroupFile(String prefix, String fileName) {
if ( verbose ) System.out.println(this.getClass().getSimpleName()+".loadGroupFile(group-file-prefix="+prefix+", fileName="+fileName+")");
GroupParser parser;
try {
URL f = new URL(fileName);
ANTLRInputStream fs = new ANTLRInputStream(f.openStream(), encoding);
GroupLexer lexer = new GroupLexer(fs);
fs.name = fileName;
CommonTokenStream tokens = new CommonTokenStream(lexer);
parser = new GroupParser(tokens);
parser.group(this, prefix);
}
catch (Exception e) {
errMgr.IOError(null, ErrorType.CANT_LOAD_GROUP_FILE, e, fileName);
}
}
/** Load template stream into this group. {@code unqualifiedFileName} is
* {@code "a.st"}. The {@code prefix} is path from group root to
* {@code unqualifiedFileName} like {@code "/subdir"} if file is in
* {@code /subdir/a.st}.
*/
public CompiledST loadTemplateFile(String prefix, String unqualifiedFileName, CharStream templateStream) {
GroupLexer lexer = new GroupLexer(templateStream);
CommonTokenStream tokens = new CommonTokenStream(lexer);
GroupParser parser = new GroupParser(tokens);
parser.group = this;
lexer.group = this;
try {
parser.templateDef(prefix);
}
catch (RecognitionException re) {
errMgr.groupSyntaxError(ErrorType.SYNTAX_ERROR, unqualifiedFileName, re, re.getMessage());
}
String templateName = Misc.getFileNameNoSuffix(unqualifiedFileName);
if ( prefix!=null && prefix.length()>0 ) templateName = prefix+templateName;
CompiledST impl = rawGetTemplate(templateName);
impl.prefix = prefix;
return impl;
}
@Override
public void load() {
if ( alreadyLoaded ) return;
alreadyLoaded = true;
GroupParser parser;
try {
ANTLRStringStream fs = new ANTLRStringStream(text);
fs.name = sourceName;
GroupLexer lexer = new GroupLexer(fs);
CommonTokenStream tokens = new CommonTokenStream(lexer);
parser = new GroupParser(tokens);
// no prefix since this group file is the entire group, nothing lives
// beneath it.
parser.group(this, "/");
}
catch (Exception e) {
errMgr.IOError(null, ErrorType.CANT_LOAD_GROUP_FILE, e, "<string>");
}
}
/** Load a group file with full path {@code fileName}; it's relative to root by {@code prefix}. */
public void loadGroupFile(String prefix, String fileName) {
if ( verbose ) System.out.println(this.getClass().getSimpleName()+".loadGroupFile(group-file-prefix="+prefix+", fileName="+fileName+")");
GroupParser parser;
try {
URL f = new URL(fileName);
ANTLRInputStream fs = new ANTLRInputStream(f.openStream(), encoding);
GroupLexer lexer = new GroupLexer(fs);
fs.name = fileName;
CommonTokenStream tokens = new CommonTokenStream(lexer);
parser = new GroupParser(tokens);
parser.group(this, prefix);
}
catch (Exception e) {
errMgr.IOError(null, ErrorType.CANT_LOAD_GROUP_FILE, e, fileName);
}
}
@Override
public void load() {
if ( alreadyLoaded ) return;
alreadyLoaded = true;
GroupParser parser;
try {
ANTLRStringStream fs = new ANTLRStringStream(text);
fs.name = sourceName;
GroupLexer lexer = new GroupLexer(fs);
CommonTokenStream tokens = new CommonTokenStream(lexer);
parser = new GroupParser(tokens);
// no prefix since this group file is the entire group, nothing lives
// beneath it.
parser.group(this, "/");
}
catch (Exception e) {
errMgr.IOError(null, ErrorType.CANT_LOAD_GROUP_FILE, e, "<string>");
}
}
/** Load a group file with full path {@code fileName}; it's relative to root by {@code prefix}. */
public void loadGroupFile(String prefix, String fileName) {
if ( verbose ) System.out.println(this.getClass().getSimpleName()+".loadGroupFile(group-file-prefix="+prefix+", fileName="+fileName+")");
GroupParser parser;
try {
URL f = new URL(fileName);
ANTLRInputStream fs = new ANTLRInputStream(f.openStream(), encoding);
GroupLexer lexer = new GroupLexer(fs);
fs.name = fileName;
CommonTokenStream tokens = new CommonTokenStream(lexer);
parser = new GroupParser(tokens);
parser.group(this, prefix);
}
catch (Exception e) {
errMgr.IOError(null, ErrorType.CANT_LOAD_GROUP_FILE, e, fileName);
}
}
@Override
public void load() {
if ( alreadyLoaded ) return;
alreadyLoaded = true;
GroupParser parser;
try {
ANTLRStringStream fs = new ANTLRStringStream(text);
fs.name = sourceName;
GroupLexer lexer = new GroupLexer(fs);
CommonTokenStream tokens = new CommonTokenStream(lexer);
parser = new GroupParser(tokens);
// no prefix since this group file is the entire group, nothing lives
// beneath it.
parser.group(this, "/");
}
catch (Exception e) {
errMgr.IOError(null, ErrorType.CANT_LOAD_GROUP_FILE, e, "<string>");
}
}
/** Load a group file with full path {@code fileName}; it's relative to root by {@code prefix}. */
public void loadGroupFile(String prefix, String fileName) {
if ( verbose ) System.out.println(this.getClass().getSimpleName()+".loadGroupFile(group-file-prefix="+prefix+", fileName="+fileName+")");
GroupParser parser;
try {
URL f = new URL(fileName);
ANTLRInputStream fs = new ANTLRInputStream(f.openStream(), encoding);
GroupLexer lexer = new GroupLexer(fs);
fs.name = fileName;
CommonTokenStream tokens = new CommonTokenStream(lexer);
parser = new GroupParser(tokens);
parser.group(this, prefix);
}
catch (Exception e) {
errMgr.IOError(null, ErrorType.CANT_LOAD_GROUP_FILE, e, fileName);
}
}
/** Load template stream into this group. {@code unqualifiedFileName} is
* {@code "a.st"}. The {@code prefix} is path from group root to
* {@code unqualifiedFileName} like {@code "/subdir"} if file is in
* {@code /subdir/a.st}.
*/
public CompiledST loadTemplateFile(String prefix, String unqualifiedFileName, CharStream templateStream) {
GroupLexer lexer = new GroupLexer(templateStream);
CommonTokenStream tokens = new CommonTokenStream(lexer);
GroupParser parser = new GroupParser(tokens);
parser.group = this;
lexer.group = this;
try {
parser.templateDef(prefix);
}
catch (RecognitionException re) {
errMgr.groupSyntaxError(ErrorType.SYNTAX_ERROR, unqualifiedFileName, re, re.getMessage());
}
String templateName = Misc.getFileNameNoSuffix(unqualifiedFileName);
if ( prefix!=null && prefix.length()>0 ) templateName = prefix+templateName;
CompiledST impl = rawGetTemplate(templateName);
impl.prefix = prefix;
return impl;
}
@Override
public void load() {
if ( alreadyLoaded ) return;
alreadyLoaded = true;
GroupParser parser;
try {
ANTLRStringStream fs = new ANTLRStringStream(text);
fs.name = sourceName;
GroupLexer lexer = new GroupLexer(fs);
CommonTokenStream tokens = new CommonTokenStream(lexer);
parser = new GroupParser(tokens);
// no prefix since this group file is the entire group, nothing lives
// beneath it.
parser.group(this, "/");
}
catch (Exception e) {
errMgr.IOError(null, ErrorType.CANT_LOAD_GROUP_FILE, e, "<string>");
}
}
/** Load a group file with full path {@code fileName}; it's relative to root by {@code prefix}. */
public void loadGroupFile(String prefix, String fileName) {
if ( verbose ) System.out.println(this.getClass().getSimpleName()+".loadGroupFile(group-file-prefix="+prefix+", fileName="+fileName+")");
GroupParser parser;
try {
URL f = new URL(fileName);
ANTLRInputStream fs = new ANTLRInputStream(f.openStream(), encoding);
GroupLexer lexer = new GroupLexer(fs);
fs.name = fileName;
CommonTokenStream tokens = new CommonTokenStream(lexer);
parser = new GroupParser(tokens);
parser.group(this, prefix);
}
catch (Exception e) {
errMgr.IOError(null, ErrorType.CANT_LOAD_GROUP_FILE, e, fileName);
}
}
/** Load template stream into this group. {@code unqualifiedFileName} is
* {@code "a.st"}. The {@code prefix} is path from group root to
* {@code unqualifiedFileName} like {@code "/subdir"} if file is in
* {@code /subdir/a.st}.
*/
public CompiledST loadTemplateFile(String prefix, String unqualifiedFileName, CharStream templateStream) {
GroupLexer lexer = new GroupLexer(templateStream);
CommonTokenStream tokens = new CommonTokenStream(lexer);
GroupParser parser = new GroupParser(tokens);
parser.group = this;
lexer.group = this;
try {
parser.templateDef(prefix);
}
catch (RecognitionException re) {
errMgr.groupSyntaxError(ErrorType.SYNTAX_ERROR, unqualifiedFileName, re, re.getMessage());
}
String templateName = Misc.getFileNameNoSuffix(unqualifiedFileName);
if ( prefix!=null && prefix.length()>0 ) templateName = prefix+templateName;
CompiledST impl = rawGetTemplate(templateName);
impl.prefix = prefix;
return impl;
}
/** Load a group file with full path {@code fileName}; it's relative to root by {@code prefix}. */
public void loadGroupFile(String prefix, String fileName) {
if ( verbose ) System.out.println(this.getClass().getSimpleName()+".loadGroupFile(group-file-prefix="+prefix+", fileName="+fileName+")");
GroupParser parser;
try {
URL f = new URL(fileName);
ANTLRInputStream fs = new ANTLRInputStream(f.openStream(), encoding);
GroupLexer lexer = new GroupLexer(fs);
fs.name = fileName;
CommonTokenStream tokens = new CommonTokenStream(lexer);
parser = new GroupParser(tokens);
parser.group(this, prefix);
}
catch (Exception e) {
errMgr.IOError(null, ErrorType.CANT_LOAD_GROUP_FILE, e, fileName);
}
}
/** Load a group file with full path {@code fileName}; it's relative to root by {@code prefix}. */
public void loadGroupFile(String prefix, String fileName) {
if ( verbose ) System.out.println(this.getClass().getSimpleName()+".loadGroupFile(group-file-prefix="+prefix +
", fileName=" +
fileName +
")");
GroupParser parser;
try {
URL f = new URL(fileName);
ANTLRInputStream fs = new ANTLRInputStream(f.openStream(), encoding);
GroupLexer lexer = new GroupLexer(fs);
fs.name = fileName;
CommonTokenStream tokens = new CommonTokenStream(lexer);
parser = new GroupParser(tokens);
parser.group(this, prefix);
}
catch (Exception e) {
errMgr.IOError(null, ErrorType.CANT_LOAD_GROUP_FILE, e, fileName);
}
}
/** Load template stream into this group. {@code unqualifiedFileName} is
* {@code "a.st"}. The {@code prefix} is path from group root to
* {@code unqualifiedFileName} like {@code "/subdir"} if file is in
* {@code /subdir/a.st}.
*/
public CompiledST loadTemplateFile(String prefix, String unqualifiedFileName, CharStream templateStream) {
GroupLexer lexer = new GroupLexer(templateStream);
CommonTokenStream tokens = new CommonTokenStream(lexer);
GroupParser parser = new GroupParser(tokens);
parser.group = this;
lexer.group = this;
try {
parser.templateDef(prefix);
}
catch (RecognitionException re) {
errMgr.groupSyntaxError(ErrorType.SYNTAX_ERROR, unqualifiedFileName, re, re.getMessage());
}
String templateName = Misc.getFileNameNoSuffix(unqualifiedFileName);
if ( prefix !=null && prefix.length()>0 ) templateName = prefix+templateName;
CompiledST impl = rawGetTemplate(templateName);
impl.prefix = prefix;
return impl;
}
@Override
public void load() {
if ( alreadyLoaded ) return;
alreadyLoaded = true;
GroupParser parser;
try {
ANTLRStringStream fs = new ANTLRStringStream(text);
fs.name = sourceName;
GroupLexer lexer = new GroupLexer(fs);
CommonTokenStream tokens = new CommonTokenStream(lexer);
parser = new GroupParser(tokens);
// no prefix since this group file is the entire group, nothing lives
// beneath it.
parser.group(this, "/");
}
catch (Exception e) {
errMgr.IOError(null, ErrorType.CANT_LOAD_GROUP_FILE, e, "<string>");
}
}
/** Load a group file with full path {@code fileName}; it's relative to root by {@code prefix}. */
public void loadGroupFile(String prefix, String fileName) {
if ( verbose ) System.out.println(this.getClass().getSimpleName()+".loadGroupFile(group-file-prefix="+prefix +
", fileName=" +
fileName +
")");
GroupParser parser;
try {
URL f = new URL(fileName);
ANTLRInputStream fs = new ANTLRInputStream(f.openStream(), encoding);
GroupLexer lexer = new GroupLexer(fs);
fs.name = fileName;
CommonTokenStream tokens = new CommonTokenStream(lexer);
parser = new GroupParser(tokens);
parser.group(this, prefix);
}
catch (Exception e) {
errMgr.IOError(null, ErrorType.CANT_LOAD_GROUP_FILE, e, fileName);
}
}
/** Load template stream into this group. {@code unqualifiedFileName} is
* {@code "a.st"}. The {@code prefix} is path from group root to
* {@code unqualifiedFileName} like {@code "/subdir"} if file is in
* {@code /subdir/a.st}.
*/
public CompiledST loadTemplateFile(String prefix, String unqualifiedFileName, CharStream templateStream) {
GroupLexer lexer = new GroupLexer(templateStream);
CommonTokenStream tokens = new CommonTokenStream(lexer);
GroupParser parser = new GroupParser(tokens);
parser.group = this;
lexer.group = this;
try {
parser.templateDef(prefix);
}
catch (RecognitionException re) {
errMgr.groupSyntaxError(ErrorType.SYNTAX_ERROR, unqualifiedFileName, re, re.getMessage());
}
String templateName = Misc.getFileNameNoSuffix(unqualifiedFileName);
if ( prefix !=null && prefix.length()>0 ) templateName = prefix+templateName;
CompiledST impl = rawGetTemplate(templateName);
impl.prefix = prefix;
return impl;
}
/** Load template stream into this group. {@code unqualifiedFileName} is
* {@code "a.st"}. The {@code prefix} is path from group root to
* {@code unqualifiedFileName} like {@code "/subdir"} if file is in
* {@code /subdir/a.st}.
*/
public CompiledST loadTemplateFile(String prefix, String unqualifiedFileName, CharStream templateStream) {
GroupLexer lexer = new GroupLexer(templateStream);
CommonTokenStream tokens = new CommonTokenStream(lexer);
GroupParser parser = new GroupParser(tokens);
parser.group = this;
lexer.group = this;
try {
parser.templateDef(prefix);
}
catch (RecognitionException re) {
errMgr.groupSyntaxError(ErrorType.SYNTAX_ERROR, unqualifiedFileName, re, re.getMessage());
}
String templateName = Misc.getFileNameNoSuffix(unqualifiedFileName);
if ( prefix!=null && prefix.length()>0 ) templateName = prefix+templateName;
CompiledST impl = rawGetTemplate(templateName);
impl.prefix = prefix;
return impl;
}
/** Load template stream into this group. {@code unqualifiedFileName} is
* {@code "a.st"}. The {@code prefix} is path from group root to
* {@code unqualifiedFileName} like {@code "/subdir"} if file is in
* {@code /subdir/a.st}.
*/
public CompiledST loadTemplateFile(String prefix, String unqualifiedFileName, CharStream templateStream) {
GroupLexer lexer = new GroupLexer(templateStream);
CommonTokenStream tokens = new CommonTokenStream(lexer);
GroupParser parser = new GroupParser(tokens);
parser.group = this;
lexer.group = this;
try {
parser.templateDef(prefix);
}
catch (RecognitionException re) {
errMgr.groupSyntaxError(ErrorType.SYNTAX_ERROR, unqualifiedFileName, re, re.getMessage());
}
String templateName = Misc.getFileNameNoSuffix(unqualifiedFileName);
if ( prefix !=null && prefix.length()>0 ) templateName = prefix+templateName;
CompiledST impl = rawGetTemplate(templateName);
impl.prefix = prefix;
return impl;
}