java.io.StreamTokenizer#quoteChar ( )源码实例Demo

/**
 * Parses a option line likes
 *    -genkaypair -dname "CN=Me"
 * and add the results into a list
 * @param list the list to fill into
 * @param s the line
 */
private static void parseArgsLine(List<String> list, String s)
        throws IOException, PropertyExpander.ExpandException {
    StreamTokenizer st = new StreamTokenizer(new StringReader(s));

    st.resetSyntax();
    st.whitespaceChars(0x00, 0x20);
    st.wordChars(0x21, 0xFF);
    // Everything is a word char except for quotation and apostrophe
    st.quoteChar('"');
    st.quoteChar('\'');

    while (true) {
        if (st.nextToken() == StreamTokenizer.TT_EOF) {
            break;
        }
        list.add(PropertyExpander.expand(st.sval));
    }
}
 

private static void loadCmdFile(String name, ListBuffer<String> args)
    throws IOException
{
    Reader r = new BufferedReader(new FileReader(name));
    StreamTokenizer st = new StreamTokenizer(r);
    st.resetSyntax();
    st.wordChars(' ', 255);
    st.whitespaceChars(0, ' ');
    st.commentChar('#');
    st.quoteChar('"');
    st.quoteChar('\'');
    while (st.nextToken() != StreamTokenizer.TT_EOF) {
        args.append(st.sval);
    }
    r.close();
}
 

/**
 * Instantiates a new arff loader.
 *
 * @param reader the reader
 * @param range
 */
public ArffLoader(Reader reader, Range range) {
  this.range = range;
  BufferedReader br = new BufferedReader(reader);

  //Init streamTokenizer
  streamTokenizer = new StreamTokenizer(br);
  streamTokenizer.resetSyntax();
  streamTokenizer.whitespaceChars(0, ' ');
  streamTokenizer.wordChars(' ' + 1, '\u00FF');
  streamTokenizer.whitespaceChars(',', ',');
  streamTokenizer.commentChar('%');
  streamTokenizer.quoteChar('"');
  streamTokenizer.quoteChar('\'');
  streamTokenizer.ordinaryChar('{');
  streamTokenizer.ordinaryChar('}');
  streamTokenizer.eolIsSignificant(true);

  this.instanceInformation = this.getHeader();

  if (range != null) { //is MultiLabel
    this.instanceInformation.setRangeOutputIndices(range);
  }

}
 

private static void loadCmdFile(String name, List args)
    throws IOException
{
    Reader r = new BufferedReader(new FileReader(name));
    StreamTokenizer st = new StreamTokenizer(r);
    st.resetSyntax();
    st.wordChars(' ', 255);
    st.whitespaceChars(0, ' ');
    st.commentChar('#');
    st.quoteChar('"');
    st.quoteChar('\'');
    while (st.nextToken() != st.TT_EOF) {
        args.add(st.sval);
    }
    r.close();
}
 

/**
 * Instantiates a new arff loader.
 *
 * @param reader the reader
 * @param range the range
 */
public ArffLoader(Reader reader, Range range) {
    this.range = range;
    BufferedReader br = new BufferedReader(reader);

    //Init streamTokenizer
    streamTokenizer = new StreamTokenizer(br);
    streamTokenizer.resetSyntax();
    streamTokenizer.whitespaceChars(0, ' ');
    streamTokenizer.wordChars(' ' + 1, '\u00FF');
    streamTokenizer.whitespaceChars(',', ',');
    streamTokenizer.commentChar('%');
    streamTokenizer.quoteChar('"');
    streamTokenizer.quoteChar('\'');
    streamTokenizer.ordinaryChar('{');
    streamTokenizer.ordinaryChar('}');
    streamTokenizer.eolIsSignificant(true);

    this.instanceInformation = this.getHeader();

    if (range != null) { //is MultiLabel
        this.instanceInformation.setRangeOutputIndices(range);
    }

}
 

private static void loadCmdFile(String name, List<String> args)
    throws IOException
{
    Reader r = new BufferedReader(new FileReader(name));
    StreamTokenizer st = new StreamTokenizer(r);
    st.resetSyntax();
    st.wordChars(' ', 255);
    st.whitespaceChars(0, ' ');
    st.commentChar('#');
    st.quoteChar('"');
    st.quoteChar('\'');
    while (st.nextToken() != StreamTokenizer.TT_EOF) {
        args.add(st.sval);
    }
    r.close();
}
 

private static void loadCmdFile(String name, List<String> args)
    throws IOException
{
    Reader r = new BufferedReader(new FileReader(name));
    StreamTokenizer st = new StreamTokenizer(r);
    st.resetSyntax();
    st.wordChars(' ', 255);
    st.whitespaceChars(0, ' ');
    st.commentChar('#');
    st.quoteChar('"');
    st.quoteChar('\'');
    while (st.nextToken() != StreamTokenizer.TT_EOF) {
        args.add(st.sval);
    }
    r.close();
}
 

private static void loadCmdFile(String name, List<String> args)
    throws IOException
{
    Reader r = new BufferedReader(new FileReader(name));
    StreamTokenizer st = new StreamTokenizer(r);
    st.resetSyntax();
    st.wordChars(' ', 255);
    st.whitespaceChars(0, ' ');
    st.commentChar('#');
    st.quoteChar('"');
    st.quoteChar('\'');
    while (st.nextToken() != StreamTokenizer.TT_EOF) {
        args.add(st.sval);
    }
    r.close();
}
 

private static void loadCmdFile(String name, List args)
    throws IOException
{
    Reader r = new BufferedReader(new FileReader(name));
    StreamTokenizer st = new StreamTokenizer(r);
    st.resetSyntax();
    st.wordChars(' ', 255);
    st.whitespaceChars(0, ' ');
    st.commentChar('#');
    st.quoteChar('"');
    st.quoteChar('\'');
    while (st.nextToken() != st.TT_EOF) {
        args.add(st.sval);
    }
    r.close();
}
 

static boolean compareDescriptions(Element reportedProblem, Element expectedProblem) throws Exception {
  String expectedDescription = expectedProblem.getChildText("description");
  String reportedDescription = reportedProblem.getChildText("description");
  if (expectedDescription.equals(reportedDescription)) return true;

  StreamTokenizer tokenizer = new StreamTokenizer(new CharArrayReader(expectedDescription.toCharArray()));
  tokenizer.quoteChar('\'');

  int idx = 0;
  while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) {
    String word;
    if (tokenizer.sval != null) {
      word = tokenizer.sval;
    } else if (tokenizer.ttype == StreamTokenizer.TT_NUMBER) {
      word = Double.toString(tokenizer.nval);
    }
    else {
      continue;
    }

    idx = reportedDescription.indexOf(word, idx);
    if (idx == -1) return false;
    idx += word.length();
  }

  return true;
}
 

/**
 * Break a string into tokens, honoring quoted arguments and escaped spaces.
 *
 * @param str a {@link String} to tokenize.
 * @return a {@link List} of {@link String}s representing the tokens that
 * constitute the string.
 */
public static List<String> tokenizeString(final String str) {
    final StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(str));
    tokenizer.resetSyntax();
    tokenizer.wordChars(0, 255);
    tokenizer.whitespaceChars(0, ' ');
    tokenizer.commentChar('#');
    tokenizer.quoteChar('"');
    tokenizer.quoteChar('\'');
    final List<String> tokenList = new ArrayList<>();
    final StringBuilder toAppend = new StringBuilder();
    while (nextToken(tokenizer) != StreamTokenizer.TT_EOF) {
        final String s = tokenizer.sval;
        // The tokenizer understands about honoring quoted strings and recognizes
        // them as one token that possibly contains multiple space-separated words.
        // It does not recognize quoted spaces, though, and will split after the
        // escaping \ character. This is handled here.
        if (s.endsWith("\\")) {
            // omit trailing \, append space instead
            toAppend.append(s.substring(0, s.length() - 1)).append(' ');
        } else {
            tokenList.add(toAppend.append(s).toString());
            toAppend.setLength(0);
        }
    }
    if (toAppend.length() != 0) {
        tokenList.add(toAppend.toString());
    }
    return tokenList;
}
 

/**
 * Break a string into tokens, honoring quoted arguments and escaped spaces.
 *
 * @param str a {@link String} to tokenize.
 * @return a {@link List} of {@link String}s representing the tokens that
 * constitute the string.
 */
public static List<String> tokenizeString(final String str) {
    final StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(str));
    tokenizer.resetSyntax();
    tokenizer.wordChars(0, 255);
    tokenizer.whitespaceChars(0, ' ');
    tokenizer.commentChar('#');
    tokenizer.quoteChar('"');
    tokenizer.quoteChar('\'');
    final List<String> tokenList = new ArrayList<>();
    final StringBuilder toAppend = new StringBuilder();
    while (nextToken(tokenizer) != StreamTokenizer.TT_EOF) {
        final String s = tokenizer.sval;
        // The tokenizer understands about honoring quoted strings and recognizes
        // them as one token that possibly contains multiple space-separated words.
        // It does not recognize quoted spaces, though, and will split after the
        // escaping \ character. This is handled here.
        if (s.endsWith("\\")) {
            // omit trailing \, append space instead
            toAppend.append(s.substring(0, s.length() - 1)).append(' ');
        } else {
            tokenList.add(toAppend.append(s).toString());
            toAppend.setLength(0);
        }
    }
    if (toAppend.length() != 0) {
        tokenList.add(toAppend.toString());
    }
    return tokenList;
}
 

/**
 * Initializes the StreamTokenizer used for reading the ARFF file.
 * 
 * @param tokenizer
 *            the stream tokenizer
 */
protected void initTokenizer(StreamTokenizer tokenizer) {

	tokenizer.resetSyntax();
	tokenizer.whitespaceChars(0, ' ');
	tokenizer.wordChars(' ' + 1, '\u00FF');
	tokenizer.whitespaceChars(',', ',');
	tokenizer.commentChar('%');
	tokenizer.quoteChar('"');
	tokenizer.quoteChar('\'');
	tokenizer.ordinaryChar('{');
	tokenizer.ordinaryChar('}');
	tokenizer.eolIsSignificant(true);
}
 

/**
 * Break a string into tokens, honoring quoted arguments and escaped spaces.
 *
 * @param str a {@link String} to tokenize.
 * @return a {@link List} of {@link String}s representing the tokens that
 * constitute the string.
 * @throws IOException in case {@link StreamTokenizer#nextToken()} raises it.
 */
public static List<String> tokenizeString(final String str) throws IOException {
    final StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(str));
    tokenizer.resetSyntax();
    tokenizer.wordChars(0, 255);
    tokenizer.whitespaceChars(0, ' ');
    tokenizer.commentChar('#');
    tokenizer.quoteChar('"');
    tokenizer.quoteChar('\'');
    final List<String> tokenList = new ArrayList<>();
    final StringBuilder toAppend = new StringBuilder();
    while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) {
        final String s = tokenizer.sval;
        // The tokenizer understands about honoring quoted strings and recognizes
        // them as one token that possibly contains multiple space-separated words.
        // It does not recognize quoted spaces, though, and will split after the
        // escaping \ character. This is handled here.
        if (s.endsWith("\\")) {
            // omit trailing \, append space instead
            toAppend.append(s.substring(0, s.length() - 1)).append(' ');
        } else {
            tokenList.add(toAppend.append(s).toString());
            toAppend.setLength(0);
        }
    }
    if (toAppend.length() != 0) {
        tokenList.add(toAppend.toString());
    }
    return tokenList;
}
 

/**
 * Checks if a board file is the specified size.
 *
 * @param filepath
 *            The path to the board file.
 * @param size
 *            The dimensions of the board to test.
 * @return {@code true} if the dimensions match.
 */
public static boolean boardIsSize(final File filepath, final BoardDimensions size) {
    int boardx = 0;
    int boardy = 0;
    try {
        // make inpustream for board
        Reader r = new BufferedReader(new FileReader(filepath));
        // read board, looking for "size"
        StreamTokenizer st = new StreamTokenizer(r);
        st.eolIsSignificant(true);
        st.commentChar('#');
        st.quoteChar('"');
        st.wordChars('_', '_');
        while (st.nextToken() != StreamTokenizer.TT_EOF) {
            if ((st.ttype == StreamTokenizer.TT_WORD) && st.sval.equalsIgnoreCase("size")) {
                st.nextToken();
                boardx = (int) st.nval;
                st.nextToken();
                boardy = (int) st.nval;
                break;
            }
        }
        r.close();
    } catch (IOException ex) {
        return false;
    }

    // check and return
    return (boardx == size.width()) && (boardy == size.height());
}
 

Lexer(String s) {
  tok = new StreamTokenizer(new CharArrayReader(s.toCharArray()));
  tok.quoteChar('"');
  tok.parseNumbers();
  tok.ordinaryChar(',');
  tok.ordinaryChar('(');
  tok.ordinaryChar(')');
  tok.wordChars('$','$');
  tok.wordChars('_','_');
}
 

private List<String> parseOptions(String defaultOptions) {
    LinkedList<String> defaultOptionList = new LinkedList<>();

    StreamTokenizer st = new StreamTokenizer(new StringReader(defaultOptions));
    st.resetSyntax();
    st.wordChars(' ' + 1, 255);
    st.whitespaceChars(0, ' ');
    st.quoteChar('"');
    st.quoteChar('\'');

    boolean firstArgQuoted;
    try {
        int tt = st.nextToken();
        firstArgQuoted = tt == '\'' || tt == '"';
        if (tt != StreamTokenizer.TT_EOF) {
            do {
                if (st.sval != null) {
                    defaultOptionList.add(st.sval);
                }
                tt = st.nextToken();
            } while (tt != StreamTokenizer.TT_EOF);
        }
    } catch (IOException e) {
        throw new IllegalStateException(e);
    }
    if (defaultOptionList.size() == 1 && firstArgQuoted) {
        return parseOptions(defaultOptionList.get(0));
    }

    return defaultOptionList;
}
 

/**
 * Reads the *set file in the given directory and filename.  It looks at the
 * given image file and prints a message if the file cannot be opened and
 * if the case does not match.
 * 
 * @param dir
 * @param filename
 * @throws IOException
 */
private static void testFile(File dir, String filename) throws IOException {
    System.out.println("Listing Errors for " + filename);
    // make inpustream for board
    Reader r = new BufferedReader(new FileReader(new File(dir, filename)));
    // read board, looking for "size"
    StreamTokenizer st = new StreamTokenizer(r);
    st.eolIsSignificant(true);
    st.commentChar('#');
    st.quoteChar('"');
    st.wordChars('_', '_');
    while (st.nextToken() != StreamTokenizer.TT_EOF) {
        String name = null;
        String imageName = null;
        String entryName = null;
        if ((st.ttype == StreamTokenizer.TT_WORD)
                && st.sval.equalsIgnoreCase("include")) { //$NON-NLS-1$
            st.nextToken();
            name = st.sval;
            testFile(dir, name);
        } else if ((st.ttype == StreamTokenizer.TT_WORD)
                && st.sval.equalsIgnoreCase("chassis")) { //$NON-NLS-1$
            st.nextToken();
            name = st.sval;
            st.nextToken();
            imageName = st.sval;
            entryName = "entry: chassis " + name + " ";
            if (imageName == null) {
                System.out.println("Error with entry " + entryName + " : no image specified!");
            } else {
                testImageName(dir, imageName, entryName);
            }
        } else if ((st.ttype == StreamTokenizer.TT_WORD)
                && st.sval.equalsIgnoreCase("exact")) { //$NON-NLS-1$
            st.nextToken();
            name = st.sval;
            st.nextToken();
            imageName = st.sval;
            entryName = "entry: exact " + name + " ";
            testImageName(dir, imageName, entryName);
        }
    }
    System.out.println("\n\n");
}
 

/**
 * Reads the *set file in the given directory and filename.  It looks at the
 * given image file and prints a message if the file cannot be opened and
 * if the case does not match.
 * 
 * @param dir
 * @param filename
 * @throws IOException
 */
private static void testFile(File dir, String filename, int incDepth)
        throws IOException {
    System.out.println("Checking file: " + filename);

    // make inpustream for board
    Reader r = new BufferedReader(new FileReader(new File(dir, filename)));
    // read board, looking for "size"
    StreamTokenizer st = new StreamTokenizer(r);

    st.eolIsSignificant(true);
    st.commentChar('#');
    st.quoteChar('"');
    st.wordChars('_', '_');
    @SuppressWarnings("unused")
    int bases, supers, orthos;
    bases = supers = orthos = 0;
    while (st.nextToken() != StreamTokenizer.TT_EOF) {
        @SuppressWarnings("unused")
        int elevation = 0;
        // int levity = 0;
        String terrain = null;
        String theme = null;
        String imageName = null;
        if ((st.ttype == StreamTokenizer.TT_WORD)
                && (st.sval.equals("base") || st.sval.equals("super") || 
                    st.sval.equals("ortho"))) { //$NON-NLS-1$ //$NON-NLS-2$
            boolean bas = st.sval.equals("base"); //$NON-NLS-1$
            boolean sup = st.sval.equals("super"); //$NON-NLS-1$
            boolean ort = st.sval.equals("ortho"); //$NON-NLS-1$

            if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
                elevation = (int) st.nval;
            } else {
                elevation = ITerrain.WILDCARD;
            }
            st.nextToken();
            terrain = st.sval;
            st.nextToken();
            theme = st.sval;
            st.nextToken();
            imageName = st.sval;
            // add to list
            if (bas) {
                bases++;
            }
            if (sup) {
                supers++;
            }
            if (ort) {
                orthos++;
            }
            Vector<String> filenames = StringUtil.splitString(imageName,
                    ";"); //$NON-NLS-1$
            for (String entryFile : filenames) {
                String entryName;
                if ((theme == null) || theme.equals("")) {
                    entryName = terrain;
                } else {
                    entryName = terrain + " " +  theme;
                }
                testImageName(dir, entryFile, entryName);
            }
        } else if ((st.ttype == StreamTokenizer.TT_WORD) &&
                st.sval.equals("include")) {
            st.nextToken(); 
            incDepth++;
            if (incDepth < 100) {
                String incFile = st.sval;
                testFile(dir, incFile, incDepth);
            }
        }
    }
    r.close();
    System.out.println("\n");
    incDepth--;
}
 

/**
 * Create new benchmark harness with given configuration and reporter.
 * Throws ConfigFormatException if there was an error parsing the config
 * file.
 * <p>
 * <b>Config file syntax:</b>
 * <p>
 * '#' marks the beginning of a comment.  Blank lines are ignored.  All
 * other lines should adhere to the following format:
 * <pre>
 *     &lt;weight&gt; &lt;name&gt; &lt;class&gt; [&lt;args&gt;]
 * </pre>
 * &lt;weight&gt; is a floating point value which is multiplied times the
 * benchmark's execution time to determine its weighted score.  The
 * total score of the benchmark suite is the sum of all weighted scores
 * of its benchmarks.
 * <p>
 * &lt;name&gt; is a name used to identify the benchmark on the benchmark
 * report.  If the name contains whitespace, the quote character '"' should
 * be used as a delimiter.
 * <p>
 * &lt;class&gt; is the full name (including the package) of the class
 * containing the benchmark implementation.  This class must implement
 * bench.Benchmark.
 * <p>
 * [&lt;args&gt;] is a variable-length list of runtime arguments to pass to
 * the benchmark.  Arguments containing whitespace should use the quote
 * character '"' as a delimiter.
 * <p>
 * <b>Example:</b>
 * <pre>
 *      3.5 "My benchmark" bench.serial.Test first second "third arg"
 * </pre>
 */
public Harness(InputStream in) throws IOException, ConfigFormatException {
    Vector bvec = new Vector();
    StreamTokenizer tokens = new StreamTokenizer(new InputStreamReader(in));

    tokens.resetSyntax();
    tokens.wordChars(0, 255);
    tokens.whitespaceChars(0, ' ');
    tokens.commentChar('#');
    tokens.quoteChar('"');
    tokens.eolIsSignificant(true);

    tokens.nextToken();
    while (tokens.ttype != StreamTokenizer.TT_EOF) {
        switch (tokens.ttype) {
            case StreamTokenizer.TT_WORD:
            case '"':                       // parse line
                bvec.add(parseBenchInfo(tokens));
                break;

            default:                        // ignore
                tokens.nextToken();
                break;
        }
    }
    binfo = (BenchInfo[]) bvec.toArray(new BenchInfo[bvec.size()]);
}