| /* |
| * Copyright (c) 1994, 2020, Oracle and/or its affiliates. All rights reserved. |
| * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| * |
| * This code is free software; you can redistribute it and/or modify it |
| * under the terms of the GNU General Public License version 2 only, as |
| * published by the Free Software Foundation. Oracle designates this |
| * particular file as subject to the "Classpath" exception as provided |
| * by Oracle in the LICENSE file that accompanied this code. |
| * |
| * This code is distributed in the hope that it will be useful, but WITHOUT |
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| * version 2 for more details (a copy is included in the LICENSE file that |
| * accompanied this code). |
| * |
| * You should have received a copy of the GNU General Public License version |
| * 2 along with this work; if not, write to the Free Software Foundation, |
| * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| * |
| * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| * or visit www.oracle.com if you need additional information or have any |
| * questions. |
| */ |
| |
| package java.util; |
| |
| import java.lang.*; |
| |
| /** |
| * The string tokenizer class allows an application to break a |
| * string into tokens. The tokenization method is much simpler than |
| * the one used by the {@code StreamTokenizer} class. The |
| * {@code StringTokenizer} methods do not distinguish among |
| * identifiers, numbers, and quoted strings, nor do they recognize |
| * and skip comments. |
| * <p> |
| * The set of delimiters (the characters that separate tokens) may |
| * be specified either at creation time or on a per-token basis. |
| * <p> |
| * An instance of {@code StringTokenizer} behaves in one of two |
| * ways, depending on whether it was created with the |
| * {@code returnDelims} flag having the value {@code true} |
| * or {@code false}: |
| * <ul> |
| * <li>If the flag is {@code false}, delimiter characters serve to |
| * separate tokens. A token is a maximal sequence of consecutive |
| * characters that are not delimiters. |
| * <li>If the flag is {@code true}, delimiter characters are themselves |
| * considered to be tokens. A token is thus either one delimiter |
| * character, or a maximal sequence of consecutive characters that are |
| * not delimiters. |
| * </ul><p> |
| * A {@code StringTokenizer} object internally maintains a current |
| * position within the string to be tokenized. Some operations advance this |
| * current position past the characters processed.<p> |
| * A token is returned by taking a substring of the string that was used to |
| * create the {@code StringTokenizer} object. |
| * <p> |
| * The following is one example of the use of the tokenizer. The code: |
| * <blockquote><pre> |
| * StringTokenizer st = new StringTokenizer("this is a test"); |
| * while (st.hasMoreTokens()) { |
| * System.out.println(st.nextToken()); |
| * } |
| * </pre></blockquote> |
| * <p> |
| * prints the following output: |
| * <blockquote><pre> |
| * this |
| * is |
| * a |
| * test |
| * </pre></blockquote> |
| * |
| * <p> |
| * {@code StringTokenizer} is a legacy class that is retained for |
| * compatibility reasons although its use is discouraged in new code. It is |
| * recommended that anyone seeking this functionality use the {@code split} |
| * method of {@code String} or the java.util.regex package instead. |
| * <p> |
| * The following example illustrates how the {@code String.split} |
| * method can be used to break up a string into its basic tokens: |
| * <blockquote><pre> |
| * String[] result = "this is a test".split("\\s"); |
| * for (int x=0; x<result.length; x++) |
| * System.out.println(result[x]); |
| * </pre></blockquote> |
| * <p> |
| * prints the following output: |
| * <blockquote><pre> |
| * this |
| * is |
| * a |
| * test |
| * </pre></blockquote> |
| * |
| * @see java.io.StreamTokenizer |
| * @since 1.0 |
| */ |
| public class StringTokenizer implements Enumeration<Object> { |
| private int currentPosition; |
| private int newPosition; |
| private int maxPosition; |
| private String str; |
| private String delimiters; |
| private boolean retDelims; |
| private boolean delimsChanged; |
| |
| /** |
| * maxDelimCodePoint stores the value of the delimiter character with the |
| * highest value. It is used to optimize the detection of delimiter |
| * characters. |
| * |
| * It is unlikely to provide any optimization benefit in the |
| * hasSurrogates case because most string characters will be |
| * smaller than the limit, but we keep it so that the two code |
| * paths remain similar. |
| */ |
| private int maxDelimCodePoint; |
| |
| /** |
| * If delimiters include any surrogates (including surrogate |
| * pairs), hasSurrogates is true and the tokenizer uses the |
| * different code path. This is because String.indexOf(int) |
| * doesn't handle unpaired surrogates as a single character. |
| */ |
| private boolean hasSurrogates = false; |
| |
| /** |
| * When hasSurrogates is true, delimiters are converted to code |
| * points and isDelimiter(int) is used to determine if the given |
| * codepoint is a delimiter. |
| */ |
| private int[] delimiterCodePoints; |
| |
| /** |
| * Set maxDelimCodePoint to the highest char in the delimiter set. |
| */ |
| private void setMaxDelimCodePoint() { |
| if (delimiters == null) { |
| maxDelimCodePoint = 0; |
| return; |
| } |
| |
| int m = 0; |
| int c; |
| int count = 0; |
| for (int i = 0; i < delimiters.length(); i += Character.charCount(c)) { |
| c = delimiters.charAt(i); |
| if (c >= Character.MIN_HIGH_SURROGATE && c <= Character.MAX_LOW_SURROGATE) { |
| c = delimiters.codePointAt(i); |
| hasSurrogates = true; |
| } |
| if (m < c) |
| m = c; |
| count++; |
| } |
| maxDelimCodePoint = m; |
| |
| if (hasSurrogates) { |
| delimiterCodePoints = new int[count]; |
| for (int i = 0, j = 0; i < count; i++, j += Character.charCount(c)) { |
| c = delimiters.codePointAt(j); |
| delimiterCodePoints[i] = c; |
| } |
| } |
| } |
| |
| /** |
| * Constructs a string tokenizer for the specified string. All |
| * characters in the {@code delim} argument are the delimiters |
| * for separating tokens. |
| * <p> |
| * If the {@code returnDelims} flag is {@code true}, then |
| * the delimiter characters are also returned as tokens. Each |
| * delimiter is returned as a string of length one. If the flag is |
| * {@code false}, the delimiter characters are skipped and only |
| * serve as separators between tokens. |
| * <p> |
| * Note that if {@code delim} is {@code null}, this constructor does |
| * not throw an exception. However, trying to invoke other methods on the |
| * resulting {@code StringTokenizer} may result in a |
| * {@code NullPointerException}. |
| * |
| * @param str a string to be parsed. |
| * @param delim the delimiters. |
| * @param returnDelims flag indicating whether to return the delimiters |
| * as tokens. |
| * @throws NullPointerException if str is {@code null} |
| */ |
| public StringTokenizer(String str, String delim, boolean returnDelims) { |
| currentPosition = 0; |
| newPosition = -1; |
| delimsChanged = false; |
| this.str = str; |
| maxPosition = str.length(); |
| delimiters = delim; |
| retDelims = returnDelims; |
| setMaxDelimCodePoint(); |
| } |
| |
| /** |
| * Constructs a string tokenizer for the specified string. The |
| * characters in the {@code delim} argument are the delimiters |
| * for separating tokens. Delimiter characters themselves will not |
| * be treated as tokens. |
| * <p> |
| * Note that if {@code delim} is {@code null}, this constructor does |
| * not throw an exception. However, trying to invoke other methods on the |
| * resulting {@code StringTokenizer} may result in a |
| * {@code NullPointerException}. |
| * |
| * @param str a string to be parsed. |
| * @param delim the delimiters. |
| * @throws NullPointerException if str is {@code null} |
| */ |
| public StringTokenizer(String str, String delim) { |
| this(str, delim, false); |
| } |
| |
| /** |
| * Constructs a string tokenizer for the specified string. The |
| * tokenizer uses the default delimiter set, which is |
| * <code>" \t\n\r\f"</code>: the space character, |
| * the tab character, the newline character, the carriage-return character, |
| * and the form-feed character. Delimiter characters themselves will |
| * not be treated as tokens. |
| * |
| * @param str a string to be parsed. |
| * @throws NullPointerException if str is {@code null} |
| */ |
| public StringTokenizer(String str) { |
| this(str, " \t\n\r\f", false); |
| } |
| |
| /** |
| * Skips delimiters starting from the specified position. If retDelims |
| * is false, returns the index of the first non-delimiter character at or |
| * after startPos. If retDelims is true, startPos is returned. |
| */ |
| private int skipDelimiters(int startPos) { |
| if (delimiters == null) |
| throw new NullPointerException(); |
| |
| int position = startPos; |
| while (!retDelims && position < maxPosition) { |
| if (!hasSurrogates) { |
| char c = str.charAt(position); |
| if ((c > maxDelimCodePoint) || (delimiters.indexOf(c) < 0)) |
| break; |
| position++; |
| } else { |
| int c = str.codePointAt(position); |
| if ((c > maxDelimCodePoint) || !isDelimiter(c)) { |
| break; |
| } |
| position += Character.charCount(c); |
| } |
| } |
| return position; |
| } |
| |
| /** |
| * Skips ahead from startPos and returns the index of the next delimiter |
| * character encountered, or maxPosition if no such delimiter is found. |
| */ |
| private int scanToken(int startPos) { |
| int position = startPos; |
| while (position < maxPosition) { |
| if (!hasSurrogates) { |
| char c = str.charAt(position); |
| if ((c <= maxDelimCodePoint) && (delimiters.indexOf(c) >= 0)) |
| break; |
| position++; |
| } else { |
| int c = str.codePointAt(position); |
| if ((c <= maxDelimCodePoint) && isDelimiter(c)) |
| break; |
| position += Character.charCount(c); |
| } |
| } |
| if (retDelims && (startPos == position)) { |
| if (!hasSurrogates) { |
| char c = str.charAt(position); |
| if ((c <= maxDelimCodePoint) && (delimiters.indexOf(c) >= 0)) |
| position++; |
| } else { |
| int c = str.codePointAt(position); |
| if ((c <= maxDelimCodePoint) && isDelimiter(c)) |
| position += Character.charCount(c); |
| } |
| } |
| return position; |
| } |
| |
| private boolean isDelimiter(int codePoint) { |
| for (int delimiterCodePoint : delimiterCodePoints) { |
| if (delimiterCodePoint == codePoint) { |
| return true; |
| } |
| } |
| return false; |
| } |
| |
| /** |
| * Tests if there are more tokens available from this tokenizer's string. |
| * If this method returns {@code true}, then a subsequent call to |
| * {@code nextToken} with no argument will successfully return a token. |
| * |
| * @return {@code true} if and only if there is at least one token |
| * in the string after the current position; {@code false} |
| * otherwise. |
| */ |
| public boolean hasMoreTokens() { |
| /* |
| * Temporarily store this position and use it in the following |
| * nextToken() method only if the delimiters haven't been changed in |
| * that nextToken() invocation. |
| */ |
| newPosition = skipDelimiters(currentPosition); |
| return (newPosition < maxPosition); |
| } |
| |
| /** |
| * Returns the next token from this string tokenizer. |
| * |
| * @return the next token from this string tokenizer. |
| * @throws NoSuchElementException if there are no more tokens in this |
| * tokenizer's string. |
| */ |
| public String nextToken() { |
| /* |
| * If next position already computed in hasMoreElements() and |
| * delimiters have changed between the computation and this invocation, |
| * then use the computed value. |
| */ |
| |
| currentPosition = (newPosition >= 0 && !delimsChanged) ? |
| newPosition : skipDelimiters(currentPosition); |
| |
| /* Reset these anyway */ |
| delimsChanged = false; |
| newPosition = -1; |
| |
| if (currentPosition >= maxPosition) |
| throw new NoSuchElementException(); |
| int start = currentPosition; |
| currentPosition = scanToken(currentPosition); |
| return str.substring(start, currentPosition); |
| } |
| |
| /** |
| * Returns the next token in this string tokenizer's string. First, |
| * the set of characters considered to be delimiters by this |
| * {@code StringTokenizer} object is changed to be the characters in |
| * the string {@code delim}. Then the next token in the string |
| * after the current position is returned. The current position is |
| * advanced beyond the recognized token. The new delimiter set |
| * remains the default after this call. |
| * |
| * @param delim the new delimiters. |
| * @return the next token, after switching to the new delimiter set. |
| * @throws NoSuchElementException if there are no more tokens in this |
| * tokenizer's string. |
| * @throws NullPointerException if delim is {@code null} |
| */ |
| public String nextToken(String delim) { |
| delimiters = delim; |
| |
| /* delimiter string specified, so set the appropriate flag. */ |
| delimsChanged = true; |
| |
| setMaxDelimCodePoint(); |
| return nextToken(); |
| } |
| |
| /** |
| * Returns the same value as the {@code hasMoreTokens} |
| * method. It exists so that this class can implement the |
| * {@code Enumeration} interface. |
| * |
| * @return {@code true} if there are more tokens; |
| * {@code false} otherwise. |
| * @see java.util.Enumeration |
| * @see java.util.StringTokenizer#hasMoreTokens() |
| */ |
| public boolean hasMoreElements() { |
| return hasMoreTokens(); |
| } |
| |
| /** |
| * Returns the same value as the {@code nextToken} method, |
| * except that its declared return value is {@code Object} rather than |
| * {@code String}. It exists so that this class can implement the |
| * {@code Enumeration} interface. |
| * |
| * @return the next token in the string. |
| * @throws NoSuchElementException if there are no more tokens in this |
| * tokenizer's string. |
| * @see java.util.Enumeration |
| * @see java.util.StringTokenizer#nextToken() |
| */ |
| public Object nextElement() { |
| return nextToken(); |
| } |
| |
| /** |
| * Calculates the number of times that this tokenizer's |
| * {@code nextToken} method can be called before it generates an |
| * exception. The current position is not advanced. |
| * |
| * @return the number of tokens remaining in the string using the current |
| * delimiter set. |
| * @see java.util.StringTokenizer#nextToken() |
| */ |
| public int countTokens() { |
| int count = 0; |
| int currpos = currentPosition; |
| while (currpos < maxPosition) { |
| currpos = skipDelimiters(currpos); |
| if (currpos >= maxPosition) |
| break; |
| currpos = scanToken(currpos); |
| count++; |
| } |
| return count; |
| } |
| } |
