下面列出了java.io.StreamTokenizer#nextToken ( ) 实例代码，或者点击链接到github查看源代码，也可以在右侧发表评论。
/**
* Read an int from the command file
* Negative numbers are preceded by "-"
*/
private int getInt(StreamTokenizer tkn) throws IOException
{
int mult = 1;
int val = tkn.nextToken();
if (tkn.sval != null && tkn.sval.equals("-"))
{
mult = -1;
val = tkn.nextToken();
}
if (val != StreamTokenizer.TT_NUMBER)
{
assertNotNull("Invalid string on line " + ln(tkn), tkn.sval);
String str = tkn.sval.toLowerCase(Locale.ENGLISH);
if (!str.startsWith("0x")) {
fail("Expecting number, got " + tkn.sval + " on line " +
ln(tkn));
} else {
return convertHex(str, ln(tkn));
}
}
return (new Double(tkn.nval).intValue() * mult);
}
String[] parseBenchArgs(StreamTokenizer tokens)
throws IOException, ConfigFormatException
{
Vector vec = new Vector();
for (;;) {
switch (tokens.ttype) {
case StreamTokenizer.TT_EOF:
case StreamTokenizer.TT_EOL:
return (String[]) vec.toArray(new String[vec.size()]);
case StreamTokenizer.TT_WORD:
case '"':
vec.add(tokens.sval);
tokens.nextToken();
break;
default:
throw new ConfigFormatException("unrecognized arg token " +
"on line " + tokens.lineno());
}
}
}
String[] parseBenchArgs(StreamTokenizer tokens)
throws IOException, ConfigFormatException
{
Vector vec = new Vector();
for (;;) {
switch (tokens.ttype) {
case StreamTokenizer.TT_EOF:
case StreamTokenizer.TT_EOL:
return (String[]) vec.toArray(new String[vec.size()]);
case StreamTokenizer.TT_WORD:
case '"':
vec.add(tokens.sval);
tokens.nextToken();
break;
default:
throw new ConfigFormatException("unrecognized arg token " +
"on line " + tokens.lineno());
}
}
}
/**
* Parses a option line likes
* -genkaypair -dname "CN=Me"
* and add the results into a list
* @param list the list to fill into
* @param s the line
*/
private static void parseArgsLine(List<String> list, String s)
throws IOException, PropertyExpander.ExpandException {
StreamTokenizer st = new StreamTokenizer(new StringReader(s));
st.resetSyntax();
st.whitespaceChars(0x00, 0x20);
st.wordChars(0x21, 0xFF);
// Everything is a word char except for quotation and apostrophe
st.quoteChar('"');
st.quoteChar('\'');
while (true) {
if (st.nextToken() == StreamTokenizer.TT_EOF) {
break;
}
list.add(PropertyExpander.expand(st.sval));
}
}
Benchmark parseBenchClass(StreamTokenizer tokens)
throws IOException, ConfigFormatException
{
Benchmark bench;
switch (tokens.ttype) {
case StreamTokenizer.TT_WORD:
case '"':
try {
Class cls = Class.forName(tokens.sval);
bench = (Benchmark) cls.newInstance();
} catch (Exception e) {
throw new ConfigFormatException("unable to instantiate " +
"benchmark \"" + tokens.sval + "\" on line " +
tokens.lineno());
}
tokens.nextToken();
return bench;
default:
throw new ConfigFormatException("missing benchmark class " +
"name on line " + tokens.lineno());
}
}
private String getUnquotedString(String str) {
if ((str == null) || !(str.startsWith("\"") && str.endsWith("\""))) return str;
new StreamTokenizer(new StringReader(str));
StreamTokenizer parser = new StreamTokenizer(new StringReader(str));
String result;
try {
parser.nextToken();
if (parser.ttype == '"') {
result = parser.sval;
} else {
result = "ERROR!";
}
} catch (IOException e) {
result = null;
LoggerHelper.info("Encountered an IOException while parsing the input string", e);
}
return result;
}
private static List<String> getTokens(StreamTokenizer st) {
final List<String> tokens = new LinkedList<>();
try {
while (st.nextToken() != StreamTokenizer.TT_EOF) {
if (st.ttype == StreamTokenizer.TT_NUMBER) {
tokens.add(String.valueOf(st.nval));
} else if (st.ttype == StreamTokenizer.TT_WORD) {
tokens.add(st.sval.trim());
} else if (st.sval != null) {
if (((char) st.ttype) == '"' || ((char) st.ttype) == '\'') {
tokens.add('"' + st.sval + '"');
} else {
tokens.add(st.sval.trim());
}
} else if ((char) st.ttype != ' ') {
tokens.add(Character.toString((char) st.ttype));
}
}
} catch (IOException e) {
logger.warn("Swallowing IOException", e); // TODO: Why are we swallowing this?
}
return tokens;
}
static Expression element(StreamTokenizer st) throws SyntaxError {
Expression result = null;
try {
switch (st.nextToken()) {
case StreamTokenizer.TT_NUMBER :
result = new ConstantExpression(st.nval);
break;
case StreamTokenizer.TT_WORD :
result = new VariableExpression(st.sval);
break;
case '(' :
result = expression(st);
st.nextToken();
if (st.ttype != ')') {
st.pushBack();
throw new SyntaxError("Mismatched parenthesis.");
}
break;
default:
st.pushBack();
throw new SyntaxError("Unexpected symbol on input.");
}
} catch (IOException ioe) {
throw new SyntaxError("Caught an I/O exception.");
}
return result;
}
/** Delivers the next token and skip empty lines. */
public static void getFirstToken(StreamTokenizer tokenizer) throws IOException {
while (tokenizer.nextToken() == StreamTokenizer.TT_EOL) {
// skip empty lines
}
if (tokenizer.ttype == '\'' || tokenizer.ttype == '"') {
tokenizer.ttype = StreamTokenizer.TT_WORD;
} else if (tokenizer.ttype == StreamTokenizer.TT_WORD && tokenizer.sval.equals("?")) {
tokenizer.ttype = '?';
}
}
private void parseAttributes(StreamTokenizer st) throws IOException, ParseException
{
int token = st.nextToken();
boolean wantAttribute = true;
while(token != StreamTokenizer.TT_EOL)
{
if (wantAttribute)
{
if (token != StreamTokenizer.TT_WORD)
{
throw new ParseException("expected an attribute name on line "+st.lineno(), st.lineno());
}
this.getParser().getCurrentParsedData().addAttribute(st.sval, st.lineno());
}
else
{
if (token != ',')
{
throw new ParseException("Expected a comma on line "+st.lineno(), st.lineno());
}
}
wantAttribute = !wantAttribute;
token = st.nextToken();
}
if (wantAttribute)
{
throw new ParseException("extra comma at the end of line "+st.lineno(), st.lineno());
}
}
/** Skips all tokens before next end of line (EOL). */
public static void waitForEOL(StreamTokenizer tokenizer) throws IOException {
while (tokenizer.nextToken() != StreamTokenizer.TT_EOL) {
// skip everything until EOL
}
tokenizer.pushBack();
}
/** Delivers the next token and checks for an unexpected end of line or file. */
public static void getNextToken(StreamTokenizer tokenizer) throws IOException {
if (tokenizer.nextToken() == StreamTokenizer.TT_EOL) {
throw new IOException("unexpected end of line " + tokenizer.lineno());
}
if (tokenizer.ttype == StreamTokenizer.TT_EOF) {
throw new IOException("unexpected end of file in line " + tokenizer.lineno());
} else if (tokenizer.ttype == '\'' || tokenizer.ttype == '"') {
tokenizer.ttype = StreamTokenizer.TT_WORD;
} else if (tokenizer.ttype == StreamTokenizer.TT_WORD && tokenizer.sval.equals("?")) {
tokenizer.ttype = '?';
}
}
BenchInfo parseBenchInfo(StreamTokenizer tokens)
throws IOException, ConfigFormatException
{
float weight = parseBenchWeight(tokens);
String name = parseBenchName(tokens);
Benchmark bench = parseBenchClass(tokens);
String[] args = parseBenchArgs(tokens);
if (tokens.ttype == StreamTokenizer.TT_EOL)
tokens.nextToken();
return new BenchInfo(bench, name, weight, args);
}
BenchInfo parseBenchInfo(StreamTokenizer tokens)
throws IOException, ConfigFormatException
{
float weight = parseBenchWeight(tokens);
String name = parseBenchName(tokens);
Benchmark bench = parseBenchClass(tokens);
String[] args = parseBenchArgs(tokens);
if (tokens.ttype == StreamTokenizer.TT_EOL)
tokens.nextToken();
return new BenchInfo(bench, name, weight, args);
}
void parseCommand(StreamTokenizer in) throws IOException {
if (in.nextToken() != StreamTokenizer.TT_WORD) {
throw new IOException("Unexpected token: " + in.ttype);
}
String word;
String[] aliasBuf = null;
int aliasPos;
if (aliases != null
&& (aliasBuf = (String[]) aliases.get(in.sval)) != null) {
word = aliasBuf[0];
aliasPos = 1;
} else {
word = in.sval;
aliasPos = 0;
}
if ("".equals(groupName) || word.startsWith("/")) {
if (word.startsWith("/"))
word = word.substring(1);
commandGroupRef = matchCommandGroup(bc, word);
word = null;
} else {
if (SessionCommandGroup.NAME.equals(groupName)) {
commandGroupRef = null;
} else {
Collection<ServiceReference<CommandGroup>> refs = null;
try {
refs = bc.getServiceReferences(CommandGroup.class,
"(groupName=" + groupName + ")");
} catch (InvalidSyntaxException ignore) {
}
if (refs.isEmpty()) {
throw new IOException("No such command group: " + groupName);
}
commandGroupRef = refs.iterator().next();
}
}
ArrayList<String> vargs = new ArrayList<String>();
boolean done = false;
while (!done) {
if (word != null) {
vargs.add(word);
word = null;
} else if (aliasPos > 0) {
if (aliasPos < aliasBuf.length) {
word = aliasBuf[aliasPos++];
} else {
aliasPos = 0;
}
} else {
switch (in.nextToken()) {
case '|':
out = new Pipe();
isPiped = true;
done = true;
break;
case '&':
isBackground = true;
case ';':
done = true;
break;
case '<':
if (in.nextToken() != StreamTokenizer.TT_WORD) {
throw new IOException("Empty input redirect");
}
// Set input to file
break;
case '>':
if (in.nextToken() != StreamTokenizer.TT_WORD) {
throw new IOException("Empty output redirect");
}
// Set output to file
break;
case StreamTokenizer.TT_EOL:
case StreamTokenizer.TT_EOF:
in.pushBack();
done = true;
break;
case '"':
case '\'':
case StreamTokenizer.TT_WORD:
word = in.sval;
break;
default:
throw new IOException("Unknown token: " + in.ttype);
}
}
}
args = vargs.toArray(new String[vargs.size()]);
}
@Override
public ComputedAttributeParserState parse(StreamTokenizer st) throws IOException, ParseException
{
ComputedAttributeParserState nextState = null;
while(nextState == null && st.ttype != StreamTokenizer.TT_EOF)
{
int nextToken = st.nextToken();
if (nextToken != StreamTokenizer.TT_EOL && nextToken != StreamTokenizer.TT_EOF)
{
ArrayList<Expression> stack = this.getParser().getStateStack();
CaseExpression caseExpression = (CaseExpression) stack.get(stack.size() - 1);
switch(nextToken)
{
case StreamTokenizer.TT_NUMBER:
caseExpression.addNumberKey(st.nval);
nextState = new CaseSelectorMiddleParserState(this.getParser());
break;
case StreamTokenizer.TT_WORD:
if (st.sval.equals("null"))
{
caseExpression.addNullKey();
stack.add(new NullExpression());
}
else if (st.sval.equals("true") || st.sval.equals("false"))
{
caseExpression.addBooleanKey(Boolean.valueOf(st.sval));
stack.add(new NullExpression());
}
else if (st.sval.equals("default"))
{
caseExpression.addDefaultKey();
stack.add(new NullExpression());
}
else
{
throw new ParseException("unexpected word "+st.sval+" in expression "+this.getParser().getFormula()+" in "+this.getParser().getDiagnosticMessage());
}
nextState = new CaseSelectorMiddleParserState(this.getParser());
break;
case '"':
caseExpression.addStringConstantKey(st.sval);
nextState = new CaseSelectorMiddleParserState(this.getParser());
break;
default:
char ch = (char)st.ttype;
throw createUnexpectedCharacterException(ch, "<number>|<null>|<boolean>|<default>|<string>");
}
}
}
return nextState;
}
/** Create a Chemical model by parsing an input stream */
XYZChemModel(InputStream is) throws Exception {
this();
StreamTokenizer st = new StreamTokenizer(
new BufferedReader(new InputStreamReader(is, "UTF-8")));
st.eolIsSignificant(true);
st.commentChar('#');
try {
scan:
while (true) {
switch (st.nextToken()) {
case StreamTokenizer.TT_EOF:
break scan;
default:
break;
case StreamTokenizer.TT_WORD:
String name = st.sval;
double x = 0,
y = 0,
z = 0;
if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
x = st.nval;
if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
y = st.nval;
if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
z = st.nval;
}
}
}
addVert(name, (float) x, (float) y, (float) z);
while (st.ttype != StreamTokenizer.TT_EOL
&& st.ttype != StreamTokenizer.TT_EOF) {
st.nextToken();
}
} // end Switch
} // end while
is.close();
} // end Try
catch (IOException e) {
}
if (st.ttype != StreamTokenizer.TT_EOF) {
throw new Exception(st.toString());
}
}
/** Function to read the options from the execution file and assign
* the values to the parameters.
*
* @param options The StreamTokenizer that reads the parameters file.
*
* @throws Exception If the format of the file is not correct.
*/
protected void setOptions(StreamTokenizer options) throws Exception {
options.nextToken();
/* Checks that the file starts with the token algorithm */
if (options.sval.equalsIgnoreCase("algorithm")) {
options.nextToken();
options.nextToken();
/* Check algorithm name
if (!options.sval.equalsIgnoreCase("CART")) {
throw new Exception("The name of the algorithm is not correct.");
}
*/
options.nextToken();
options.nextToken();
/* Reads the names of the input files*/
if (options.sval.equalsIgnoreCase("inputData")) {
options.nextToken();
options.nextToken();
modelFileName = options.sval;
if (options.nextToken() != StreamTokenizer.TT_EOL) {
trainFileName = options.sval;
options.nextToken();
testFileName = options.sval;
if (options.nextToken() != StreamTokenizer.TT_EOL) {
trainFileName = modelFileName;
options.nextToken();
}
}
} else {
throw new Exception("No file test provided.");
}
/* Reads the names of the output files*/
while (true) {
if (options.nextToken() == StreamTokenizer.TT_EOF) {
throw new Exception("No output file provided.");
}
if (options.sval == null) {
continue;
} else if (options.sval.equalsIgnoreCase("outputData")) {
break;
}
}
options.nextToken();
options.nextToken();
trainOutputFileName = options.sval;
options.nextToken();
testOutputFileName = options.sval;
options.nextToken();
resultFileName = options.sval;
if (!getNextToken(options)) {
throw new Exception("No instances provided.");
}
if (options.ttype == StreamTokenizer.TT_EOF) {
return;
}
for (int k = 0; k < nParam; k++) {
/* Reads the maxDepth parameter */
if (options.sval.equalsIgnoreCase("maxDepth")) {
options.nextToken();
options.nextToken();
if (Integer.parseInt(options.sval) > 0) {
maxDepth = Integer.parseInt(options.sval);
}
if (!getNextToken(options)) {
return;
} else {
continue;
}
}
/* Any other parameter should be added here */
} // end for
} else {
throw new Exception("The file must start with the word " +
"algorithm followed of the name of the algorithm.");
}
}
/** Create a Chemical model by parsing an input stream */
XYZChemModel(InputStream is) throws Exception {
this();
StreamTokenizer st = new StreamTokenizer(
new BufferedReader(new InputStreamReader(is, "UTF-8")));
st.eolIsSignificant(true);
st.commentChar('#');
try {
scan:
while (true) {
switch (st.nextToken()) {
case StreamTokenizer.TT_EOF:
break scan;
default:
break;
case StreamTokenizer.TT_WORD:
String name = st.sval;
double x = 0,
y = 0,
z = 0;
if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
x = st.nval;
if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
y = st.nval;
if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
z = st.nval;
}
}
}
addVert(name, (float) x, (float) y, (float) z);
while (st.ttype != StreamTokenizer.TT_EOL
&& st.ttype != StreamTokenizer.TT_EOF) {
st.nextToken();
}
} // end Switch
} // end while
is.close();
} // end Try
catch (IOException e) {
}
if (st.ttype != StreamTokenizer.TT_EOF) {
throw new Exception(st.toString());
}
}
/**
* Create new benchmark harness with given configuration and reporter.
* Throws ConfigFormatException if there was an error parsing the config
* file.
* <p>
* <b>Config file syntax:</b>
* <p>
* '#' marks the beginning of a comment. Blank lines are ignored. All
* other lines should adhere to the following format:
* <pre>
* <weight> <name> <class> [<args>]
* </pre>
* <weight> is a floating point value which is multiplied times the
* benchmark's execution time to determine its weighted score. The
* total score of the benchmark suite is the sum of all weighted scores
* of its benchmarks.
* <p>
* <name> is a name used to identify the benchmark on the benchmark
* report. If the name contains whitespace, the quote character '"' should
* be used as a delimiter.
* <p>
* <class> is the full name (including the package) of the class
* containing the benchmark implementation. This class must implement
* bench.Benchmark.
* <p>
* [<args>] is a variable-length list of runtime arguments to pass to
* the benchmark. Arguments containing whitespace should use the quote
* character '"' as a delimiter.
* <p>
* <b>Example:</b>
* <pre>
* 3.5 "My benchmark" bench.serial.Test first second "third arg"
* </pre>
*/
public Harness(InputStream in) throws IOException, ConfigFormatException {
Vector bvec = new Vector();
StreamTokenizer tokens = new StreamTokenizer(new InputStreamReader(in));
tokens.resetSyntax();
tokens.wordChars(0, 255);
tokens.whitespaceChars(0, ' ');
tokens.commentChar('#');
tokens.quoteChar('"');
tokens.eolIsSignificant(true);
tokens.nextToken();
while (tokens.ttype != StreamTokenizer.TT_EOF) {
switch (tokens.ttype) {
case StreamTokenizer.TT_WORD:
case '"': // parse line
bvec.add(parseBenchInfo(tokens));
break;
default: // ignore
tokens.nextToken();
break;
}
}
binfo = (BenchInfo[]) bvec.toArray(new BenchInfo[bvec.size()]);
}