下面列出了java.io.StreamTokenizer#whitespaceChars ( ) 实例代码，或者点击链接到github查看源代码，也可以在右侧发表评论。
private static void loadCmdFile(String name, List args)
throws IOException
{
Reader r = new BufferedReader(new FileReader(name));
StreamTokenizer st = new StreamTokenizer(r);
st.resetSyntax();
st.wordChars(' ', 255);
st.whitespaceChars(0, ' ');
st.commentChar('#');
st.quoteChar('"');
st.quoteChar('\'');
while (st.nextToken() != st.TT_EOF) {
args.add(st.sval);
}
r.close();
}
private static void loadCmdFile(String name, List args)
throws IOException
{
Reader r = new BufferedReader(new FileReader(name));
StreamTokenizer st = new StreamTokenizer(r);
st.resetSyntax();
st.wordChars(' ', 255);
st.whitespaceChars(0, ' ');
st.commentChar('#');
st.quoteChar('"');
st.quoteChar('\'');
while (st.nextToken() != st.TT_EOF) {
args.add(st.sval);
}
r.close();
}
/** throws an exception if the parsed geometry type does not match the expected shape type */
public static ShapeBuilder parseExpectedType(XContentParser parser, final GeoShapeType shapeType,
final GeoShapeFieldMapper shapeMapper)
throws IOException, ElasticsearchParseException {
StringReader reader = new StringReader(parser.text());
try {
boolean ignoreZValue = (shapeMapper != null && shapeMapper.ignoreZValue().value() == true);
// setup the tokenizer; configured to read words w/o numbers
StreamTokenizer tokenizer = new StreamTokenizer(reader);
tokenizer.resetSyntax();
tokenizer.wordChars('a', 'z');
tokenizer.wordChars('A', 'Z');
tokenizer.wordChars(128 + 32, 255);
tokenizer.wordChars('0', '9');
tokenizer.wordChars('-', '-');
tokenizer.wordChars('+', '+');
tokenizer.wordChars('.', '.');
tokenizer.whitespaceChars(0, ' ');
tokenizer.commentChar('#');
ShapeBuilder builder = parseGeometry(tokenizer, shapeType, ignoreZValue);
checkEOF(tokenizer);
return builder;
} finally {
reader.close();
}
}
private static void loadCmdFile(String name, ListBuffer<String> args)
throws IOException
{
Reader r = new BufferedReader(new FileReader(name));
StreamTokenizer st = new StreamTokenizer(r);
st.resetSyntax();
st.wordChars(' ', 255);
st.whitespaceChars(0, ' ');
st.commentChar('#');
st.quoteChar('"');
st.quoteChar('\'');
while (st.nextToken() != StreamTokenizer.TT_EOF) {
args.append(st.sval);
}
r.close();
}
/** Function to initialize the stream tokenizer.
*
* @param tokenizer The tokenizer.
*/
protected void initTokenizer( StreamTokenizer tokenizer )
{
tokenizer.resetSyntax();
tokenizer.whitespaceChars( 0, ' ' );
tokenizer.wordChars( ' '+1,'\u00FF' );
tokenizer.whitespaceChars( ',',',' );
tokenizer.quoteChar( '"' );
tokenizer.quoteChar( '\'' );
tokenizer.ordinaryChar( '=' );
tokenizer.ordinaryChar( '{' );
tokenizer.ordinaryChar( '}' );
tokenizer.ordinaryChar( '[' );
tokenizer.ordinaryChar( ']' );
tokenizer.eolIsSignificant( true );
}
private static void loadCmdFile(String name, List<String> args)
throws IOException
{
Reader r = new BufferedReader(new FileReader(name));
StreamTokenizer st = new StreamTokenizer(r);
st.resetSyntax();
st.wordChars(' ', 255);
st.whitespaceChars(0, ' ');
st.commentChar('#');
st.quoteChar('"');
st.quoteChar('\'');
while (st.nextToken() != StreamTokenizer.TT_EOF) {
args.add(st.sval);
}
r.close();
}
private static void loadCmdFile(String name, List<String> args)
throws IOException
{
Reader r = new BufferedReader(new FileReader(name));
StreamTokenizer st = new StreamTokenizer(r);
st.resetSyntax();
st.wordChars(' ', 255);
st.whitespaceChars(0, ' ');
st.commentChar('#');
st.quoteChar('"');
st.quoteChar('\'');
while (st.nextToken() != StreamTokenizer.TT_EOF) {
args.add(st.sval);
}
r.close();
}
/**
* Initializes the StreamTokenizer used for reading the ARFF file.
*
* @param tokenizer
* the stream tokenizer
*/
protected void initTokenizer(StreamTokenizer tokenizer) {
tokenizer.resetSyntax();
tokenizer.whitespaceChars(0, ' ');
tokenizer.wordChars(' ' + 1, '\u00FF');
tokenizer.whitespaceChars(',', ',');
tokenizer.commentChar('%');
tokenizer.quoteChar('"');
tokenizer.quoteChar('\'');
tokenizer.ordinaryChar('{');
tokenizer.ordinaryChar('}');
tokenizer.eolIsSignificant(true);
}
private Map<String, String> parseHeader() {
Map<String, String> map = new HashMap<>();
try {
StreamTokenizer tok = new StreamTokenizer(new StringReader(this.fullContent));
tok.quoteChar('"');
tok.quoteChar('\'');
tok.whitespaceChars('=', '=');
tok.whitespaceChars(',', ',');
while (tok.nextToken() != StreamTokenizer.TT_EOF) {
String key = tok.sval;
if (tok.nextToken() == StreamTokenizer.TT_EOF) {
map.put(key, null);
return map;
}
String value = null;
if ("nc".equals(key)) {
//nc is a 8 length HEX number so need get it as number
value = String.valueOf(tok.nval);
if (value.indexOf('.') > 0) {
value = value.substring(0, value.indexOf('.'));
}
StringBuilder pad = new StringBuilder();
pad.append("");
for (int i = 0; i < 8 - value.length(); i++) {
pad.append('0');
}
value = pad.toString() + value;
} else {
value = tok.sval;
}
map.put(key, value);
}
} catch (IOException ex) {
//ignore can't happen for StringReader
}
return map;
}
/**
* Break a string into tokens, honoring quoted arguments and escaped spaces.
*
* @param str a {@link String} to tokenize.
* @return a {@link List} of {@link String}s representing the tokens that
* constitute the string.
*/
public static List<String> tokenizeString(final String str) {
final StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(str));
tokenizer.resetSyntax();
tokenizer.wordChars(0, 255);
tokenizer.whitespaceChars(0, ' ');
tokenizer.commentChar('#');
tokenizer.quoteChar('"');
tokenizer.quoteChar('\'');
final List<String> tokenList = new ArrayList<>();
final StringBuilder toAppend = new StringBuilder();
while (nextToken(tokenizer) != StreamTokenizer.TT_EOF) {
final String s = tokenizer.sval;
// The tokenizer understands about honoring quoted strings and recognizes
// them as one token that possibly contains multiple space-separated words.
// It does not recognize quoted spaces, though, and will split after the
// escaping \ character. This is handled here.
if (s.endsWith("\\")) {
// omit trailing \, append space instead
toAppend.append(s.substring(0, s.length() - 1)).append(' ');
} else {
tokenList.add(toAppend.append(s).toString());
toAppend.setLength(0);
}
}
if (toAppend.length() != 0) {
tokenList.add(toAppend.toString());
}
return tokenList;
}
/**
* Parses a <code>command</code>. Utilizes the {@link StreamTokenizer} which
* takes care of quoted Strings as well.
*
* @param command the command to parse
* @return the tokenized command which can be processed by the
* <code>ConsoleInterpreter</code>
*
* @see org.openhab.io.console.ConsoleInterpreter
*/
protected String[] parseCommand(String command) {
logger.trace("going to parse command '{}'", command);
// if the command starts with '>' it contains a script which needs no
// further handling here ...
if (command.startsWith(">")) {
return new String[] { ">", command.substring(1).trim() };
}
StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(command));
// treat all characters as ordinary, including digits, so we never
// have to deal with doubles
tokenizer.resetSyntax();
tokenizer.wordChars(0x23, 0xFF);
tokenizer.whitespaceChars(0x00, 0x20);
tokenizer.quoteChar('"');
List<String> tokens = new ArrayList<String>();
try {
int tokenType = 0;
while (tokenType != StreamTokenizer.TT_EOF && tokenType != StreamTokenizer.TT_EOL) {
tokenType = tokenizer.nextToken();
String token = "";
switch (tokenType) {
case StreamTokenizer.TT_WORD:
case 34 /* quoted String */:
token = tokenizer.sval;
break;
}
tokens.add(token);
logger.trace("read value {} from the given command", token);
}
} catch (IOException ioe) {
}
return tokens.toArray(new String[0]);
}
private static void loadCmdFile(String name, ListBuffer<String> args)
throws IOException {
Reader r = new BufferedReader(new FileReader(name));
StreamTokenizer st = new StreamTokenizer(r);
st.resetSyntax();
st.wordChars(' ', 255);
st.whitespaceChars(0, ' ');
st.commentChar('#');
st.quoteChar('"');
st.quoteChar('\'');
while (st.nextToken() != st.TT_EOF) {
args.append(st.sval);
}
r.close();
}
public static StreamTokenizer tokenize(String input) {
StreamTokenizer result = new StreamTokenizer(new StringReader(input));
result.resetSyntax();
result.wordChars('a', 'z');
result.wordChars('A', 'Z');
result.wordChars('0', '9');
result.whitespaceChars(0, 32);
result.quoteChar('"');
return result;
}
public FilterTreeBuilder(String filterString) throws IOException {
StreamTokenizer input = new StreamTokenizer(new StringReader(filterString));
input.resetSyntax();
// Default settings in StreamTokenizer syntax initializer.
input.wordChars('a', 'z');
input.wordChars('A', 'Z');
// Specifies that all extended ASCII characters defined in HTML 4 standard, are word constituents.
input.wordChars(128 + 32, 255);
input.whitespaceChars(0, ' ');
input.commentChar('/');
input.quoteChar('"');
input.quoteChar('\'');
// Adding other string possible values.
input.wordChars('@', '@');
input.wordChars(':', ':');
input.wordChars('_', '_');
input.wordChars('0', '9');
input.wordChars('-', '-');
input.wordChars('+', '+');
input.wordChars('.', '.');
input.wordChars('*', '*');
input.wordChars('/', '/');
tokenList = new ArrayList<>();
StringBuilder concatenatedString = new StringBuilder();
while (input.nextToken() != StreamTokenizer.TT_EOF) {
// The ttype 40 is for the '('.
if (input.ttype == 40) {
tokenList.add("(");
} else if (input.ttype == 41) {
// The ttype 41 is for the ')'.
concatenatedString = new StringBuilder(concatenatedString.toString().trim());
tokenList.add(concatenatedString.toString());
concatenatedString = new StringBuilder();
tokenList.add(")");
} else if (input.ttype == StreamTokenizer.TT_WORD) {
// Concatenate the string by adding spaces in between.
if (!(input.sval.equalsIgnoreCase(IdentityCoreConstants.Filter.AND) || input.sval.equalsIgnoreCase
(IdentityCoreConstants.Filter.OR) || input.sval.equalsIgnoreCase(IdentityCoreConstants.Filter.NOT)))
concatenatedString.append(" ").append(input.sval);
else {
concatenatedString = new StringBuilder(concatenatedString.toString().trim());
if (!concatenatedString.toString().equals("")) {
tokenList.add(concatenatedString.toString());
concatenatedString = new StringBuilder();
}
tokenList.add(input.sval);
}
} else if (input.ttype == '\"' || input.ttype == '\'') {
concatenatedString.append(" ").append(input.sval);
}
}
// Add to the list, if the filter is a simple filter.
if (!(concatenatedString.toString().equals(""))) {
tokenList.add(concatenatedString.toString());
}
}
/**
* Create new benchmark harness with given configuration and reporter.
* Throws ConfigFormatException if there was an error parsing the config
* file.
* <p>
* <b>Config file syntax:</b>
* <p>
* '#' marks the beginning of a comment. Blank lines are ignored. All
* other lines should adhere to the following format:
* <pre>
* <weight> <name> <class> [<args>]
* </pre>
* <weight> is a floating point value which is multiplied times the
* benchmark's execution time to determine its weighted score. The
* total score of the benchmark suite is the sum of all weighted scores
* of its benchmarks.
* <p>
* <name> is a name used to identify the benchmark on the benchmark
* report. If the name contains whitespace, the quote character '"' should
* be used as a delimiter.
* <p>
* <class> is the full name (including the package) of the class
* containing the benchmark implementation. This class must implement
* bench.Benchmark.
* <p>
* [<args>] is a variable-length list of runtime arguments to pass to
* the benchmark. Arguments containing whitespace should use the quote
* character '"' as a delimiter.
* <p>
* <b>Example:</b>
* <pre>
* 3.5 "My benchmark" bench.serial.Test first second "third arg"
* </pre>
*/
public Harness(InputStream in) throws IOException, ConfigFormatException {
Vector bvec = new Vector();
StreamTokenizer tokens = new StreamTokenizer(new InputStreamReader(in));
tokens.resetSyntax();
tokens.wordChars(0, 255);
tokens.whitespaceChars(0, ' ');
tokens.commentChar('#');
tokens.quoteChar('"');
tokens.eolIsSignificant(true);
tokens.nextToken();
while (tokens.ttype != StreamTokenizer.TT_EOF) {
switch (tokens.ttype) {
case StreamTokenizer.TT_WORD:
case '"': // parse line
bvec.add(parseBenchInfo(tokens));
break;
default: // ignore
tokens.nextToken();
break;
}
}
binfo = (BenchInfo[]) bvec.toArray(new BenchInfo[bvec.size()]);
}
/**
* Create new benchmark harness with given configuration and reporter.
* Throws ConfigFormatException if there was an error parsing the config
* file.
* <p>
* <b>Config file syntax:</b>
* <p>
* '#' marks the beginning of a comment. Blank lines are ignored. All
* other lines should adhere to the following format:
* <pre>
* <weight> <name> <class> [<args>]
* </pre>
* <weight> is a floating point value which is multiplied times the
* benchmark's execution time to determine its weighted score. The
* total score of the benchmark suite is the sum of all weighted scores
* of its benchmarks.
* <p>
* <name> is a name used to identify the benchmark on the benchmark
* report. If the name contains whitespace, the quote character '"' should
* be used as a delimiter.
* <p>
* <class> is the full name (including the package) of the class
* containing the benchmark implementation. This class must implement
* bench.Benchmark.
* <p>
* [<args>] is a variable-length list of runtime arguments to pass to
* the benchmark. Arguments containing whitespace should use the quote
* character '"' as a delimiter.
* <p>
* <b>Example:</b>
* <pre>
* 3.5 "My benchmark" bench.serial.Test first second "third arg"
* </pre>
*/
public Harness(InputStream in) throws IOException, ConfigFormatException {
Vector bvec = new Vector();
StreamTokenizer tokens = new StreamTokenizer(new InputStreamReader(in));
tokens.resetSyntax();
tokens.wordChars(0, 255);
tokens.whitespaceChars(0, ' ');
tokens.commentChar('#');
tokens.quoteChar('"');
tokens.eolIsSignificant(true);
tokens.nextToken();
while (tokens.ttype != StreamTokenizer.TT_EOF) {
switch (tokens.ttype) {
case StreamTokenizer.TT_WORD:
case '"': // parse line
bvec.add(parseBenchInfo(tokens));
break;
default: // ignore
tokens.nextToken();
break;
}
}
binfo = (BenchInfo[]) bvec.toArray(new BenchInfo[bvec.size()]);
}
/**
* Create new benchmark harness with given configuration and reporter.
* Throws ConfigFormatException if there was an error parsing the config
* file.
* <p>
* <b>Config file syntax:</b>
* <p>
* '#' marks the beginning of a comment. Blank lines are ignored. All
* other lines should adhere to the following format:
* <pre>
* <weight> <name> <class> [<args>]
* </pre>
* <weight> is a floating point value which is multiplied times the
* benchmark's execution time to determine its weighted score. The
* total score of the benchmark suite is the sum of all weighted scores
* of its benchmarks.
* <p>
* <name> is a name used to identify the benchmark on the benchmark
* report. If the name contains whitespace, the quote character '"' should
* be used as a delimiter.
* <p>
* <class> is the full name (including the package) of the class
* containing the benchmark implementation. This class must implement
* bench.Benchmark.
* <p>
* [<args>] is a variable-length list of runtime arguments to pass to
* the benchmark. Arguments containing whitespace should use the quote
* character '"' as a delimiter.
* <p>
* <b>Example:</b>
* <pre>
* 3.5 "My benchmark" bench.serial.Test first second "third arg"
* </pre>
*/
public Harness(InputStream in) throws IOException, ConfigFormatException {
Vector bvec = new Vector();
StreamTokenizer tokens = new StreamTokenizer(new InputStreamReader(in));
tokens.resetSyntax();
tokens.wordChars(0, 255);
tokens.whitespaceChars(0, ' ');
tokens.commentChar('#');
tokens.quoteChar('"');
tokens.eolIsSignificant(true);
tokens.nextToken();
while (tokens.ttype != StreamTokenizer.TT_EOF) {
switch (tokens.ttype) {
case StreamTokenizer.TT_WORD:
case '"': // parse line
bvec.add(parseBenchInfo(tokens));
break;
default: // ignore
tokens.nextToken();
break;
}
}
binfo = (BenchInfo[]) bvec.toArray(new BenchInfo[bvec.size()]);
}
/**
* Create new benchmark harness with given configuration and reporter.
* Throws ConfigFormatException if there was an error parsing the config
* file.
* <p>
* <b>Config file syntax:</b>
* <p>
* '#' marks the beginning of a comment. Blank lines are ignored. All
* other lines should adhere to the following format:
* <pre>
* <weight> <name> <class> [<args>]
* </pre>
* <weight> is a floating point value which is multiplied times the
* benchmark's execution time to determine its weighted score. The
* total score of the benchmark suite is the sum of all weighted scores
* of its benchmarks.
* <p>
* <name> is a name used to identify the benchmark on the benchmark
* report. If the name contains whitespace, the quote character '"' should
* be used as a delimiter.
* <p>
* <class> is the full name (including the package) of the class
* containing the benchmark implementation. This class must implement
* bench.Benchmark.
* <p>
* [<args>] is a variable-length list of runtime arguments to pass to
* the benchmark. Arguments containing whitespace should use the quote
* character '"' as a delimiter.
* <p>
* <b>Example:</b>
* <pre>
* 3.5 "My benchmark" bench.serial.Test first second "third arg"
* </pre>
*/
public Harness(InputStream in) throws IOException, ConfigFormatException {
Vector bvec = new Vector();
StreamTokenizer tokens = new StreamTokenizer(new InputStreamReader(in));
tokens.resetSyntax();
tokens.wordChars(0, 255);
tokens.whitespaceChars(0, ' ');
tokens.commentChar('#');
tokens.quoteChar('"');
tokens.eolIsSignificant(true);
tokens.nextToken();
while (tokens.ttype != StreamTokenizer.TT_EOF) {
switch (tokens.ttype) {
case StreamTokenizer.TT_WORD:
case '"': // parse line
bvec.add(parseBenchInfo(tokens));
break;
default: // ignore
tokens.nextToken();
break;
}
}
binfo = (BenchInfo[]) bvec.toArray(new BenchInfo[bvec.size()]);
}
/**
* Create new benchmark harness with given configuration and reporter.
* Throws ConfigFormatException if there was an error parsing the config
* file.
* <p>
* <b>Config file syntax:</b>
* <p>
* '#' marks the beginning of a comment. Blank lines are ignored. All
* other lines should adhere to the following format:
* <pre>
* <weight> <name> <class> [<args>]
* </pre>
* <weight> is a floating point value which is multiplied times the
* benchmark's execution time to determine its weighted score. The
* total score of the benchmark suite is the sum of all weighted scores
* of its benchmarks.
* <p>
* <name> is a name used to identify the benchmark on the benchmark
* report. If the name contains whitespace, the quote character '"' should
* be used as a delimiter.
* <p>
* <class> is the full name (including the package) of the class
* containing the benchmark implementation. This class must implement
* bench.Benchmark.
* <p>
* [<args>] is a variable-length list of runtime arguments to pass to
* the benchmark. Arguments containing whitespace should use the quote
* character '"' as a delimiter.
* <p>
* <b>Example:</b>
* <pre>
* 3.5 "My benchmark" bench.serial.Test first second "third arg"
* </pre>
*/
public Harness(InputStream in) throws IOException, ConfigFormatException {
Vector bvec = new Vector();
StreamTokenizer tokens = new StreamTokenizer(new InputStreamReader(in));
tokens.resetSyntax();
tokens.wordChars(0, 255);
tokens.whitespaceChars(0, ' ');
tokens.commentChar('#');
tokens.quoteChar('"');
tokens.eolIsSignificant(true);
tokens.nextToken();
while (tokens.ttype != StreamTokenizer.TT_EOF) {
switch (tokens.ttype) {
case StreamTokenizer.TT_WORD:
case '"': // parse line
bvec.add(parseBenchInfo(tokens));
break;
default: // ignore
tokens.nextToken();
break;
}
}
binfo = (BenchInfo[]) bvec.toArray(new BenchInfo[bvec.size()]);
}
/**
* Create new benchmark harness with given configuration and reporter.
* Throws ConfigFormatException if there was an error parsing the config
* file.
* <p>
* <b>Config file syntax:</b>
* <p>
* '#' marks the beginning of a comment. Blank lines are ignored. All
* other lines should adhere to the following format:
* <pre>
* <weight> <name> <class> [<args>]
* </pre>
* <weight> is a floating point value which is multiplied times the
* benchmark's execution time to determine its weighted score. The
* total score of the benchmark suite is the sum of all weighted scores
* of its benchmarks.
* <p>
* <name> is a name used to identify the benchmark on the benchmark
* report. If the name contains whitespace, the quote character '"' should
* be used as a delimiter.
* <p>
* <class> is the full name (including the package) of the class
* containing the benchmark implementation. This class must implement
* bench.Benchmark.
* <p>
* [<args>] is a variable-length list of runtime arguments to pass to
* the benchmark. Arguments containing whitespace should use the quote
* character '"' as a delimiter.
* <p>
* <b>Example:</b>
* <pre>
* 3.5 "My benchmark" bench.serial.Test first second "third arg"
* </pre>
*/
public Harness(InputStream in) throws IOException, ConfigFormatException {
Vector bvec = new Vector();
StreamTokenizer tokens = new StreamTokenizer(new InputStreamReader(in));
tokens.resetSyntax();
tokens.wordChars(0, 255);
tokens.whitespaceChars(0, ' ');
tokens.commentChar('#');
tokens.quoteChar('"');
tokens.eolIsSignificant(true);
tokens.nextToken();
while (tokens.ttype != StreamTokenizer.TT_EOF) {
switch (tokens.ttype) {
case StreamTokenizer.TT_WORD:
case '"': // parse line
bvec.add(parseBenchInfo(tokens));
break;
default: // ignore
tokens.nextToken();
break;
}
}
binfo = (BenchInfo[]) bvec.toArray(new BenchInfo[bvec.size()]);
}