下面列出了怎么用java.io.StreamTokenizer的API类实例代码及写法，或者点击链接到github查看源代码。
protected Object parseR(StreamTokenizer tokens) throws IOException {
int token = tokens.nextToken();
switch (token) {
case TT_EOF:
return null;
case TT_WORD:
return parseAtomic(tokens);
case LBRACE:
return parseMap(tokens);
case LBRACKET:
return parseArray(tokens);
case QUOTE:
return parseAtomic(tokens);
default:
throw new IOException("Unexpected token:" + (char) token);
}
}
float parseBenchWeight(StreamTokenizer tokens)
throws IOException, ConfigFormatException
{
float weight;
switch (tokens.ttype) {
case StreamTokenizer.TT_WORD:
case '"':
try {
weight = Float.parseFloat(tokens.sval);
} catch (NumberFormatException e) {
throw new ConfigFormatException("illegal weight value \"" +
tokens.sval + "\" on line " + tokens.lineno());
}
tokens.nextToken();
return weight;
default:
throw new ConfigFormatException("missing weight value on line "
+ tokens.lineno());
}
}
String[] parseBenchArgs(StreamTokenizer tokens)
throws IOException, ConfigFormatException
{
Vector vec = new Vector();
for (;;) {
switch (tokens.ttype) {
case StreamTokenizer.TT_EOF:
case StreamTokenizer.TT_EOL:
return (String[]) vec.toArray(new String[vec.size()]);
case StreamTokenizer.TT_WORD:
case '"':
vec.add(tokens.sval);
tokens.nextToken();
break;
default:
throw new ConfigFormatException("unrecognized arg token " +
"on line " + tokens.lineno());
}
}
}
private String getUnquotedString(String str) {
if ((str == null) || !(str.startsWith("\"") && str.endsWith("\""))) return str;
new StreamTokenizer(new StringReader(str));
StreamTokenizer parser = new StreamTokenizer(new StringReader(str));
String result;
try {
parser.nextToken();
if (parser.ttype == '"') {
result = parser.sval;
} else {
result = "ERROR!";
}
} catch (IOException e) {
result = null;
LoggerHelper.info("Encountered an IOException while parsing the input string", e);
}
return result;
}
Benchmark parseBenchClass(StreamTokenizer tokens)
throws IOException, ConfigFormatException
{
Benchmark bench;
switch (tokens.ttype) {
case StreamTokenizer.TT_WORD:
case '"':
try {
Class cls = Class.forName(tokens.sval);
bench = (Benchmark) cls.newInstance();
} catch (Exception e) {
throw new ConfigFormatException("unable to instantiate " +
"benchmark \"" + tokens.sval + "\" on line " +
tokens.lineno());
}
tokens.nextToken();
return bench;
default:
throw new ConfigFormatException("missing benchmark class " +
"name on line " + tokens.lineno());
}
}
float parseBenchWeight(StreamTokenizer tokens)
throws IOException, ConfigFormatException
{
float weight;
switch (tokens.ttype) {
case StreamTokenizer.TT_WORD:
case '"':
try {
weight = Float.parseFloat(tokens.sval);
} catch (NumberFormatException e) {
throw new ConfigFormatException("illegal weight value \"" +
tokens.sval + "\" on line " + tokens.lineno());
}
tokens.nextToken();
return weight;
default:
throw new ConfigFormatException("missing weight value on line "
+ tokens.lineno());
}
}
String parseBenchName(StreamTokenizer tokens)
throws IOException, ConfigFormatException
{
String name;
switch (tokens.ttype) {
case StreamTokenizer.TT_WORD:
case '"':
name = tokens.sval;
tokens.nextToken();
return name;
default:
throw new ConfigFormatException("missing benchmark name on " +
"line " + tokens.lineno());
}
}
/**
* Read an int or codepoint - codepoint is given as a string
*/
private int getIntOrCP() throws IOException
{
int val = tkn.nextToken();
if (val == StreamTokenizer.TT_NUMBER)
{
return new Double(tkn.nval).intValue();
}
else if (val == StreamTokenizer.TT_WORD)
{
return decodeCP(tkn.sval);
}
else
{
fail("Expecting number, got " + tkn.sval + " on line "
+ tkn.lineno());
System.exit(1);
}
return 0;
}
String[] parseBenchArgs(StreamTokenizer tokens)
throws IOException, ConfigFormatException
{
Vector vec = new Vector();
for (;;) {
switch (tokens.ttype) {
case StreamTokenizer.TT_EOF:
case StreamTokenizer.TT_EOL:
return (String[]) vec.toArray(new String[vec.size()]);
case StreamTokenizer.TT_WORD:
case '"':
vec.add(tokens.sval);
tokens.nextToken();
break;
default:
throw new ConfigFormatException("unrecognized arg token " +
"on line " + tokens.lineno());
}
}
}
public String toMessage() {
switch(ttype) {
case StreamTokenizer.TT_EOL:
return "\"EOL\"";
case StreamTokenizer.TT_EOF:
return "\"EOF\"";
case StreamTokenizer.TT_NUMBER:
return "NUMBER";
case StreamTokenizer.TT_WORD:
if (sval == null) {
return "IDENTIFIER";
} else {
return "IDENTIFIER " + sval;
}
default:
if (ttype == (int)'"') {
String msg = "QUOTED STRING";
if (sval != null)
msg = msg + " \"" + sval + "\"";
return msg;
} else {
return "CHARACTER \'" + (char)ttype + "\'";
}
}
}
Benchmark parseBenchClass(StreamTokenizer tokens)
throws IOException, ConfigFormatException
{
Benchmark bench;
switch (tokens.ttype) {
case StreamTokenizer.TT_WORD:
case '"':
try {
Class cls = Class.forName(tokens.sval);
bench = (Benchmark) cls.newInstance();
} catch (Exception e) {
throw new ConfigFormatException("unable to instantiate " +
"benchmark \"" + tokens.sval + "\" on line " +
tokens.lineno());
}
tokens.nextToken();
return bench;
default:
throw new ConfigFormatException("missing benchmark class " +
"name on line " + tokens.lineno());
}
}
private static void loadCmdFile(String name, List<String> args)
throws IOException
{
Reader r = new BufferedReader(new FileReader(name));
StreamTokenizer st = new StreamTokenizer(r);
st.resetSyntax();
st.wordChars(' ', 255);
st.whitespaceChars(0, ' ');
st.commentChar('#');
st.quoteChar('"');
st.quoteChar('\'');
while (st.nextToken() != StreamTokenizer.TT_EOF) {
args.add(st.sval);
}
r.close();
}
/**
* Read a word, a symbol or EOF, or generate a parse error. If this is EOF, the string "EOF" is returned.
*/
protected String getWordOrSymbolOrEof() throws IOException {
final int tok = st.nextToken();
if (tok == StreamTokenizer.TT_NUMBER || tok == QUOTE_CHAR) {
parseError("expecting a word or symbol, " + gotWhat(tok));
}
if (tok == StreamTokenizer.TT_WORD) {
return st.sval;
} else if (tok == StreamTokenizer.TT_EOF) {
return "EOF";
} else {
return Character.toString((char) tok);
}
}
String parseBenchName(StreamTokenizer tokens)
throws IOException, ConfigFormatException
{
String name;
switch (tokens.ttype) {
case StreamTokenizer.TT_WORD:
case '"':
name = tokens.sval;
tokens.nextToken();
return name;
default:
throw new ConfigFormatException("missing benchmark name on " +
"line " + tokens.lineno());
}
}
private static void loadCmdFile(String name, List args)
throws IOException
{
Reader r = new BufferedReader(new FileReader(name));
StreamTokenizer st = new StreamTokenizer(r);
st.resetSyntax();
st.wordChars(' ', 255);
st.whitespaceChars(0, ' ');
st.commentChar('#');
st.quoteChar('"');
st.quoteChar('\'');
while (st.nextToken() != st.TT_EOF) {
args.add(st.sval);
}
r.close();
}
Benchmark parseBenchClass(StreamTokenizer tokens)
throws IOException, ConfigFormatException
{
Benchmark bench;
switch (tokens.ttype) {
case StreamTokenizer.TT_WORD:
case '"':
try {
Class cls = Class.forName(tokens.sval);
bench = (Benchmark) cls.newInstance();
} catch (Exception e) {
throw new ConfigFormatException("unable to instantiate " +
"benchmark \"" + tokens.sval + "\" on line " +
tokens.lineno());
}
tokens.nextToken();
return bench;
default:
throw new ConfigFormatException("missing benchmark class " +
"name on line " + tokens.lineno());
}
}
Benchmark parseBenchClass(StreamTokenizer tokens)
throws IOException, ConfigFormatException
{
Benchmark bench;
switch (tokens.ttype) {
case StreamTokenizer.TT_WORD:
case '"':
try {
Class cls = Class.forName(tokens.sval);
bench = (Benchmark) cls.newInstance();
} catch (Exception e) {
throw new ConfigFormatException("unable to instantiate " +
"benchmark \"" + tokens.sval + "\" on line " +
tokens.lineno());
}
tokens.nextToken();
return bench;
default:
throw new ConfigFormatException("missing benchmark class " +
"name on line " + tokens.lineno());
}
}
/**
* Reads instance. It detects if it is dense or sparse.
*
* @return the instance
*/
public Instance readInstance() {
while (streamTokenizer.ttype == StreamTokenizer.TT_EOL) {
try {
streamTokenizer.nextToken();
} catch (IOException ex) {
Logger.getLogger(ArffLoader.class.getName()).log(Level.SEVERE, null, ex);
}
}
if (streamTokenizer.ttype == '{') {
return readInstanceSparse();
// return readDenseInstanceSparse();
} else {
return readInstanceDense();
}
}
/**
* Print failure message and skip to the next test
*
* @exception IOException error reading file
*/
private void fail(String msg) throws IOException
{
System.out.println("FAILED - " + msg + " in line " + tkn.lineno());
// skip remainder of the test look for endtest or end of file
int val = tkn.nextToken();
while (val != StreamTokenizer.TT_EOF)
{
if (val == StreamTokenizer.TT_WORD && tkn.sval.toLowerCase(Locale.ENGLISH).equals("endtest"))
break;
val = tkn.nextToken();
}
failed = true;
// get ready for next test
reset();
// print out stack trace so we know where the failure occurred
Exception e = new Exception();
e.printStackTrace();
}
/**
* Push a tokenizer created from a file name on the file stack and make it current.
*
* @param file
* Name of the file used as source for the tokenizer.
*/
protected void pushTokenizer(final String file) throws IOException {
StreamTokenizer tok;
CurrentFile cur;
Reader reader;
try {
reader = createReaderFrom(file);
tok = createTokenizer(reader);
cur = new CurrentFile(file, tok, reader);
} catch (final FileNotFoundException e) {
throw new IOException("cannot read file '" + file + "', not found: " + e.getMessage());
}
configureTokenizer(tok);
tok_stack.add(cur);
st = tok;
filename = file;
}
/**
* Instantiates a new arff loader.
*
* @param reader the reader
* @param range the range
*/
public ArffLoader(Reader reader, Range range) {
this.range = range;
BufferedReader br = new BufferedReader(reader);
//Init streamTokenizer
streamTokenizer = new StreamTokenizer(br);
streamTokenizer.resetSyntax();
streamTokenizer.whitespaceChars(0, ' ');
streamTokenizer.wordChars(' ' + 1, '\u00FF');
streamTokenizer.whitespaceChars(',', ',');
streamTokenizer.commentChar('%');
streamTokenizer.quoteChar('"');
streamTokenizer.quoteChar('\'');
streamTokenizer.ordinaryChar('{');
streamTokenizer.ordinaryChar('}');
streamTokenizer.eolIsSignificant(true);
this.instanceInformation = this.getHeader();
if (range != null) { //is MultiLabel
this.instanceInformation.setRangeOutputIndices(range);
}
}
/**
* Processes the test commands in the stream.
*
* @param cmdStream command stream (see {@link #commandSequence})
* @throws IOException if accessing a file or the socket fails, or if
* Derby detects an unexpected protocol error
*/
private void processCommands(Reader cmdStream)
throws IOException {
StreamTokenizer tkn = new StreamTokenizer(cmdStream);
boolean endSignalled = false;
int val;
while ( (val = tkn.nextToken()) != StreamTokenizer.TT_EOF) {
assertFalse("End signalled, data to process left: " + tkn.sval,
endSignalled);
switch(val) {
case StreamTokenizer.TT_NUMBER:
break;
case StreamTokenizer.TT_WORD:
endSignalled = processCommand(tkn);
break;
case StreamTokenizer.TT_EOL:
break;
}
}
}
Benchmark parseBenchClass(StreamTokenizer tokens)
throws IOException, ConfigFormatException
{
Benchmark bench;
switch (tokens.ttype) {
case StreamTokenizer.TT_WORD:
case '"':
try {
Class cls = Class.forName(tokens.sval);
bench = (Benchmark) cls.newInstance();
} catch (Exception e) {
throw new ConfigFormatException("unable to instantiate " +
"benchmark \"" + tokens.sval + "\" on line " +
tokens.lineno());
}
tokens.nextToken();
return bench;
default:
throw new ConfigFormatException("missing benchmark class " +
"name on line " + tokens.lineno());
}
}
public String toMessage() {
switch(ttype) {
case StreamTokenizer.TT_EOL:
return "\"EOL\"";
case StreamTokenizer.TT_EOF:
return "\"EOF\"";
case StreamTokenizer.TT_NUMBER:
return "NUMBER";
case StreamTokenizer.TT_WORD:
if (sval == null) {
return "IDENTIFIER";
} else {
return "IDENTIFIER " + sval;
}
default:
if (ttype == (int)'"') {
String msg = "QUOTED STRING";
if (sval != null)
msg = msg + " \"" + sval + "\"";
return msg;
} else {
return "CHARACTER \'" + (char)ttype + "\'";
}
}
}
public String toMessage() {
switch(ttype) {
case StreamTokenizer.TT_EOL:
return "\"EOL\"";
case StreamTokenizer.TT_EOF:
return "\"EOF\"";
case StreamTokenizer.TT_NUMBER:
return "NUMBER";
case StreamTokenizer.TT_WORD:
if (sval == null) {
return "IDENTIFIER";
} else {
return "IDENTIFIER " + sval;
}
default:
if (ttype == (int)'"') {
String msg = "QUOTED STRING";
if (sval != null)
msg = msg + " \"" + sval + "\"";
return msg;
} else {
return "CHARACTER \'" + (char)ttype + "\'";
}
}
}
/** Function to initialize the stream tokenizer.
*
* @param tokenizer The tokenizer.
*/
protected void initTokenizer( StreamTokenizer tokenizer )
{
tokenizer.resetSyntax();
tokenizer.whitespaceChars( 0, ' ' );
tokenizer.wordChars( ' '+1,'\u00FF' );
tokenizer.whitespaceChars( ',',',' );
tokenizer.quoteChar( '"' );
tokenizer.quoteChar( '\'' );
tokenizer.ordinaryChar( '=' );
tokenizer.ordinaryChar( '{' );
tokenizer.ordinaryChar( '}' );
tokenizer.ordinaryChar( '[' );
tokenizer.ordinaryChar( ']' );
tokenizer.eolIsSignificant( true );
}
/**
* Push a tokenizer created from a stream on the file stack and make it current.
*
* @param stream
* The stream used as source for the tokenizer.
* @param name
* The name of the input stream.
*/
protected void pushTokenizer(final InputStream stream, final String name) throws IOException {
StreamTokenizer tok;
CurrentFile cur;
Reader reader;
reader = createReaderFrom(stream);
tok = createTokenizer(reader);
cur = new CurrentFile(name, tok, reader);
configureTokenizer(tok);
tok_stack.add(cur);
st = tok;
filename = name;
}
public static Object parseExpectedType(String wkt, final ShapeType shapeType) throws IOException, ParseException {
try (StringReader reader = new StringReader(wkt)) {
// setup the tokenizer; configured to read words w/o numbers
StreamTokenizer tokenizer = new StreamTokenizer(reader);
tokenizer.resetSyntax();
tokenizer.wordChars('a', 'z');
tokenizer.wordChars('A', 'Z');
tokenizer.wordChars(128 + 32, 255);
tokenizer.wordChars('0', '9');
tokenizer.wordChars('-', '-');
tokenizer.wordChars('+', '+');
tokenizer.wordChars('.', '.');
tokenizer.whitespaceChars(0, ' ');
tokenizer.commentChar('#');
Object geometry = parseGeometry(tokenizer, shapeType);
checkEOF(tokenizer);
return geometry;
}
}
protected Object parseArray(StreamTokenizer tokens) throws IOException {
assert (tokens.ttype == LBRACKET);
List<Object> array = new ArrayList<>();
loop: for (;;) {
int token = tokens.nextToken();
switch (token) {
case TT_EOL:
break; // ignore
case TT_EOF:
throw new IOException("Unexpected eof");
case RBRACKET:
break loop;
default:
tokens.pushBack();
Object o = parseR(tokens);
tokens.nextToken();
if (tokens.ttype == TT_EOF)
break;
else if (tokens.ttype == RBRACKET)
tokens.pushBack();
else if (tokens.ttype != COMMA)
throw new IOException("Missing comma in list");
array.add(o);
}
}
return array;
}
public ParserState parse(StreamTokenizer st) throws IOException, ParseException
{
if (dataClass == null)
{
throw new ParseException("no class name found before line "+st.lineno(), st.lineno());
}
MithraDataObject currentData;
currentData = this.getParser().getCurrentParsedData().createAndAddDataObject(st.lineno());
// parse the data
int currentAttribute = 0;
int token = st.ttype;
boolean wantData = true;
while(token != StreamTokenizer.TT_EOL && token != StreamTokenizer.TT_EOF)
{
if (wantData)
{
this.getParser().getCurrentParsedData().parseData(st, currentAttribute, currentData);
currentAttribute++;
}
else
{
if (token != ',')
{
throw new ParseException("Expected a comma on line "+st.lineno(), st.lineno());
}
}
wantData = !wantData;
token = st.nextToken();
}
return this.getParser().getBeginningOfLineState();
}