Mercurial > jdk8-mips64-public > langtools / changeset

     1.1 --- a/src/share/classes/com/sun/tools/apt/main/AptJavaCompiler.java	Fri Oct 21 14:14:29 2011 -0700
     1.2 +++ b/src/share/classes/com/sun/tools/apt/main/AptJavaCompiler.java	Mon Oct 24 13:00:20 2011 +0100
     1.3 @@ -42,7 +42,6 @@
     1.4  import com.sun.tools.apt.comp.*;
     1.5  import com.sun.tools.apt.util.Bark;
     1.6  import com.sun.mirror.apt.AnnotationProcessorFactory;
     1.7 -import com.sun.tools.javac.parser.DocCommentScanner;
     1.8  
     1.9  /**
    1.10   *  <p><b>This is NOT part of any supported API.

     2.1 --- a/src/share/classes/com/sun/tools/javac/parser/DocCommentScanner.java	Fri Oct 21 14:14:29 2011 -0700
     2.2 +++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
     2.3 @@ -1,420 +0,0 @@
     2.4 -/*
     2.5 - * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
     2.6 - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
     2.7 - *
     2.8 - * This code is free software; you can redistribute it and/or modify it
     2.9 - * under the terms of the GNU General Public License version 2 only, as
    2.10 - * published by the Free Software Foundation.  Oracle designates this
    2.11 - * particular file as subject to the "Classpath" exception as provided
    2.12 - * by Oracle in the LICENSE file that accompanied this code.
    2.13 - *
    2.14 - * This code is distributed in the hope that it will be useful, but WITHOUT
    2.15 - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    2.16 - * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
    2.17 - * version 2 for more details (a copy is included in the LICENSE file that
    2.18 - * accompanied this code).
    2.19 - *
    2.20 - * You should have received a copy of the GNU General Public License version
    2.21 - * 2 along with this work; if not, write to the Free Software Foundation,
    2.22 - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
    2.23 - *
    2.24 - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
    2.25 - * or visit www.oracle.com if you need additional information or have any
    2.26 - * questions.
    2.27 - */
    2.28 -
    2.29 -package com.sun.tools.javac.parser;
    2.30 -
    2.31 -import java.nio.*;
    2.32 -
    2.33 -import com.sun.tools.javac.util.*;
    2.34 -import static com.sun.tools.javac.util.LayoutCharacters.*;
    2.35 -
    2.36 -/** An extension to the base lexical analyzer that captures
    2.37 - *  and processes the contents of doc comments.  It does so by
    2.38 - *  translating Unicode escape sequences and by stripping the
    2.39 - *  leading whitespace and starts from each line of the comment.
    2.40 - *
    2.41 - *  <p><b>This is NOT part of any supported API.
    2.42 - *  If you write code that depends on this, you do so at your own risk.
    2.43 - *  This code and its internal interfaces are subject to change or
    2.44 - *  deletion without notice.</b>
    2.45 - */
    2.46 -public class DocCommentScanner extends Scanner {
    2.47 -
    2.48 -    /** Create a scanner from the input buffer.  buffer must implement
    2.49 -     *  array() and compact(), and remaining() must be less than limit().
    2.50 -     */
    2.51 -    protected DocCommentScanner(ScannerFactory fac, CharBuffer buffer) {
    2.52 -        super(fac, buffer);
    2.53 -    }
    2.54 -
    2.55 -    /** Create a scanner from the input array.  The array must have at
    2.56 -     *  least a single character of extra space.
    2.57 -     */
    2.58 -    protected DocCommentScanner(ScannerFactory fac, char[] input, int inputLength) {
    2.59 -        super(fac, input, inputLength);
    2.60 -    }
    2.61 -
    2.62 -    /** Starting position of the comment in original source
    2.63 -     */
    2.64 -    private int pos;
    2.65 -
    2.66 -    /** The comment input buffer, index of next chacter to be read,
    2.67 -     *  index of one past last character in buffer.
    2.68 -     */
    2.69 -    private char[] buf;
    2.70 -    private int bp;
    2.71 -    private int buflen;
    2.72 -
    2.73 -    /** The current character.
    2.74 -     */
    2.75 -    private char ch;
    2.76 -
    2.77 -    /** The column number position of the current character.
    2.78 -     */
    2.79 -    private int col;
    2.80 -
    2.81 -    /** The buffer index of the last converted Unicode character
    2.82 -     */
    2.83 -    private int unicodeConversionBp = 0;
    2.84 -
    2.85 -    /**
    2.86 -     * Buffer for doc comment.
    2.87 -     */
    2.88 -    private char[] docCommentBuffer = new char[1024];
    2.89 -
    2.90 -    /**
    2.91 -     * Number of characters in doc comment buffer.
    2.92 -     */
    2.93 -    private int docCommentCount;
    2.94 -
    2.95 -    /**
    2.96 -     * Translated and stripped contents of doc comment
    2.97 -     */
    2.98 -    private String docComment = null;
    2.99 -
   2.100 -
   2.101 -    /** Unconditionally expand the comment buffer.
   2.102 -     */
   2.103 -    private void expandCommentBuffer() {
   2.104 -        char[] newBuffer = new char[docCommentBuffer.length * 2];
   2.105 -        System.arraycopy(docCommentBuffer, 0, newBuffer,
   2.106 -                         0, docCommentBuffer.length);
   2.107 -        docCommentBuffer = newBuffer;
   2.108 -    }
   2.109 -
   2.110 -    /** Convert an ASCII digit from its base (8, 10, or 16)
   2.111 -     *  to its value.
   2.112 -     */
   2.113 -    private int digit(int base) {
   2.114 -        char c = ch;
   2.115 -        int result = Character.digit(c, base);
   2.116 -        if (result >= 0 && c > 0x7f) {
   2.117 -            ch = "0123456789abcdef".charAt(result);
   2.118 -        }
   2.119 -        return result;
   2.120 -    }
   2.121 -
   2.122 -    /** Convert Unicode escape; bp points to initial '\' character
   2.123 -     *  (Spec 3.3).
   2.124 -     */
   2.125 -    private void convertUnicode() {
   2.126 -        if (ch == '\\' && unicodeConversionBp != bp) {
   2.127 -            bp++; ch = buf[bp]; col++;
   2.128 -            if (ch == 'u') {
   2.129 -                do {
   2.130 -                    bp++; ch = buf[bp]; col++;
   2.131 -                } while (ch == 'u');
   2.132 -                int limit = bp + 3;
   2.133 -                if (limit < buflen) {
   2.134 -                    int d = digit(16);
   2.135 -                    int code = d;
   2.136 -                    while (bp < limit && d >= 0) {
   2.137 -                        bp++; ch = buf[bp]; col++;
   2.138 -                        d = digit(16);
   2.139 -                        code = (code << 4) + d;
   2.140 -                    }
   2.141 -                    if (d >= 0) {
   2.142 -                        ch = (char)code;
   2.143 -                        unicodeConversionBp = bp;
   2.144 -                        return;
   2.145 -                    }
   2.146 -                }
   2.147 -                // "illegal.Unicode.esc", reported by base scanner
   2.148 -            } else {
   2.149 -                bp--;
   2.150 -                ch = '\\';
   2.151 -                col--;
   2.152 -            }
   2.153 -        }
   2.154 -    }
   2.155 -
   2.156 -
   2.157 -    /** Read next character.
   2.158 -     */
   2.159 -    private void scanChar() {
   2.160 -        bp++;
   2.161 -        ch = buf[bp];
   2.162 -        switch (ch) {
   2.163 -        case '\r': // return
   2.164 -            col = 0;
   2.165 -            break;
   2.166 -        case '\n': // newline
   2.167 -            if (bp == 0 || buf[bp-1] != '\r') {
   2.168 -                col = 0;
   2.169 -            }
   2.170 -            break;
   2.171 -        case '\t': // tab
   2.172 -            col = (col / TabInc * TabInc) + TabInc;
   2.173 -            break;
   2.174 -        case '\\': // possible Unicode
   2.175 -            col++;
   2.176 -            convertUnicode();
   2.177 -            break;
   2.178 -        default:
   2.179 -            col++;
   2.180 -            break;
   2.181 -        }
   2.182 -    }
   2.183 -
   2.184 -    /**
   2.185 -     * Read next character in doc comment, skipping over double '\' characters.
   2.186 -     * If a double '\' is skipped, put in the buffer and update buffer count.
   2.187 -     */
   2.188 -    private void scanDocCommentChar() {
   2.189 -        scanChar();
   2.190 -        if (ch == '\\') {
   2.191 -            if (buf[bp+1] == '\\' && unicodeConversionBp != bp) {
   2.192 -                if (docCommentCount == docCommentBuffer.length)
   2.193 -                    expandCommentBuffer();
   2.194 -                docCommentBuffer[docCommentCount++] = ch;
   2.195 -                bp++; col++;
   2.196 -            } else {
   2.197 -                convertUnicode();
   2.198 -            }
   2.199 -        }
   2.200 -    }
   2.201 -
   2.202 -    /* Reset doc comment before reading each new token
   2.203 -     */
   2.204 -    public void nextToken() {
   2.205 -        docComment = null;
   2.206 -        super.nextToken();
   2.207 -    }
   2.208 -
   2.209 -    /**
   2.210 -     * Returns the documentation string of the current token.
   2.211 -     */
   2.212 -    public String docComment() {
   2.213 -        return docComment;
   2.214 -    }
   2.215 -
   2.216 -    /**
   2.217 -     * Process a doc comment and make the string content available.
   2.218 -     * Strips leading whitespace and stars.
   2.219 -     */
   2.220 -    @SuppressWarnings("fallthrough")
   2.221 -    protected void processComment(CommentStyle style) {
   2.222 -        if (style != CommentStyle.JAVADOC) {
   2.223 -            return;
   2.224 -        }
   2.225 -
   2.226 -        pos = pos();
   2.227 -        buf = getRawCharacters(pos, endPos());
   2.228 -        buflen = buf.length;
   2.229 -        bp = 0;
   2.230 -        col = 0;
   2.231 -
   2.232 -        docCommentCount = 0;
   2.233 -
   2.234 -        boolean firstLine = true;
   2.235 -
   2.236 -        // Skip over first slash
   2.237 -        scanDocCommentChar();
   2.238 -        // Skip over first star
   2.239 -        scanDocCommentChar();
   2.240 -
   2.241 -        // consume any number of stars
   2.242 -        while (bp < buflen && ch == '*') {
   2.243 -            scanDocCommentChar();
   2.244 -        }
   2.245 -        // is the comment in the form /**/, /***/, /****/, etc. ?
   2.246 -        if (bp < buflen && ch == '/') {
   2.247 -            docComment = "";
   2.248 -            return;
   2.249 -        }
   2.250 -
   2.251 -        // skip a newline on the first line of the comment.
   2.252 -        if (bp < buflen) {
   2.253 -            if (ch == LF) {
   2.254 -                scanDocCommentChar();
   2.255 -                firstLine = false;
   2.256 -            } else if (ch == CR) {
   2.257 -                scanDocCommentChar();
   2.258 -                if (ch == LF) {
   2.259 -                    scanDocCommentChar();
   2.260 -                    firstLine = false;
   2.261 -                }
   2.262 -            }
   2.263 -        }
   2.264 -
   2.265 -    outerLoop:
   2.266 -
   2.267 -        // The outerLoop processes the doc comment, looping once
   2.268 -        // for each line.  For each line, it first strips off
   2.269 -        // whitespace, then it consumes any stars, then it
   2.270 -        // puts the rest of the line into our buffer.
   2.271 -        while (bp < buflen) {
   2.272 -
   2.273 -            // The wsLoop consumes whitespace from the beginning
   2.274 -            // of each line.
   2.275 -        wsLoop:
   2.276 -
   2.277 -            while (bp < buflen) {
   2.278 -                switch(ch) {
   2.279 -                case ' ':
   2.280 -                    scanDocCommentChar();
   2.281 -                    break;
   2.282 -                case '\t':
   2.283 -                    col = ((col - 1) / TabInc * TabInc) + TabInc;
   2.284 -                    scanDocCommentChar();
   2.285 -                    break;
   2.286 -                case FF:
   2.287 -                    col = 0;
   2.288 -                    scanDocCommentChar();
   2.289 -                    break;
   2.290 -// Treat newline at beginning of line (blank line, no star)
   2.291 -// as comment text.  Old Javadoc compatibility requires this.
   2.292 -/*---------------------------------*
   2.293 -                case CR: // (Spec 3.4)
   2.294 -                    scanDocCommentChar();
   2.295 -                    if (ch == LF) {
   2.296 -                        col = 0;
   2.297 -                        scanDocCommentChar();
   2.298 -                    }
   2.299 -                    break;
   2.300 -                case LF: // (Spec 3.4)
   2.301 -                    scanDocCommentChar();
   2.302 -                    break;
   2.303 -*---------------------------------*/
   2.304 -                default:
   2.305 -                    // we've seen something that isn't whitespace;
   2.306 -                    // jump out.
   2.307 -                    break wsLoop;
   2.308 -                }
   2.309 -            }
   2.310 -
   2.311 -            // Are there stars here?  If so, consume them all
   2.312 -            // and check for the end of comment.
   2.313 -            if (ch == '*') {
   2.314 -                // skip all of the stars
   2.315 -                do {
   2.316 -                    scanDocCommentChar();
   2.317 -                } while (ch == '*');
   2.318 -
   2.319 -                // check for the closing slash.
   2.320 -                if (ch == '/') {
   2.321 -                    // We're done with the doc comment
   2.322 -                    // scanChar() and breakout.
   2.323 -                    break outerLoop;
   2.324 -                }
   2.325 -            } else if (! firstLine) {
   2.326 -                //The current line does not begin with a '*' so we will indent it.
   2.327 -                for (int i = 1; i < col; i++) {
   2.328 -                    if (docCommentCount == docCommentBuffer.length)
   2.329 -                        expandCommentBuffer();
   2.330 -                    docCommentBuffer[docCommentCount++] = ' ';
   2.331 -                }
   2.332 -            }
   2.333 -
   2.334 -            // The textLoop processes the rest of the characters
   2.335 -            // on the line, adding them to our buffer.
   2.336 -        textLoop:
   2.337 -            while (bp < buflen) {
   2.338 -                switch (ch) {
   2.339 -                case '*':
   2.340 -                    // Is this just a star?  Or is this the
   2.341 -                    // end of a comment?
   2.342 -                    scanDocCommentChar();
   2.343 -                    if (ch == '/') {
   2.344 -                        // This is the end of the comment,
   2.345 -                        // set ch and return our buffer.
   2.346 -                        break outerLoop;
   2.347 -                    }
   2.348 -                    // This is just an ordinary star.  Add it to
   2.349 -                    // the buffer.
   2.350 -                    if (docCommentCount == docCommentBuffer.length)
   2.351 -                        expandCommentBuffer();
   2.352 -                    docCommentBuffer[docCommentCount++] = '*';
   2.353 -                    break;
   2.354 -                case ' ':
   2.355 -                case '\t':
   2.356 -                    if (docCommentCount == docCommentBuffer.length)
   2.357 -                        expandCommentBuffer();
   2.358 -                    docCommentBuffer[docCommentCount++] = ch;
   2.359 -                    scanDocCommentChar();
   2.360 -                    break;
   2.361 -                case FF:
   2.362 -                    scanDocCommentChar();
   2.363 -                    break textLoop; // treat as end of line
   2.364 -                case CR: // (Spec 3.4)
   2.365 -                    scanDocCommentChar();
   2.366 -                    if (ch != LF) {
   2.367 -                        // Canonicalize CR-only line terminator to LF
   2.368 -                        if (docCommentCount == docCommentBuffer.length)
   2.369 -                            expandCommentBuffer();
   2.370 -                        docCommentBuffer[docCommentCount++] = (char)LF;
   2.371 -                        break textLoop;
   2.372 -                    }
   2.373 -                    /* fall through to LF case */
   2.374 -                case LF: // (Spec 3.4)
   2.375 -                    // We've seen a newline.  Add it to our
   2.376 -                    // buffer and break out of this loop,
   2.377 -                    // starting fresh on a new line.
   2.378 -                    if (docCommentCount == docCommentBuffer.length)
   2.379 -                        expandCommentBuffer();
   2.380 -                    docCommentBuffer[docCommentCount++] = ch;
   2.381 -                    scanDocCommentChar();
   2.382 -                    break textLoop;
   2.383 -                default:
   2.384 -                    // Add the character to our buffer.
   2.385 -                    if (docCommentCount == docCommentBuffer.length)
   2.386 -                        expandCommentBuffer();
   2.387 -                    docCommentBuffer[docCommentCount++] = ch;
   2.388 -                    scanDocCommentChar();
   2.389 -                }
   2.390 -            } // end textLoop
   2.391 -            firstLine = false;
   2.392 -        } // end outerLoop
   2.393 -
   2.394 -        if (docCommentCount > 0) {
   2.395 -            int i = docCommentCount - 1;
   2.396 -        trailLoop:
   2.397 -            while (i > -1) {
   2.398 -                switch (docCommentBuffer[i]) {
   2.399 -                case '*':
   2.400 -                    i--;
   2.401 -                    break;
   2.402 -                default:
   2.403 -                    break trailLoop;
   2.404 -                }
   2.405 -            }
   2.406 -            docCommentCount = i + 1;
   2.407 -
   2.408 -            // Store the text of the doc comment
   2.409 -            docComment = new String(docCommentBuffer, 0 , docCommentCount);
   2.410 -        } else {
   2.411 -            docComment = "";
   2.412 -        }
   2.413 -    }
   2.414 -
   2.415 -    /** Build a map for translating between line numbers and
   2.416 -     * positions in the input.
   2.417 -     *
   2.418 -     * @return a LineMap */
   2.419 -    public Position.LineMap getLineMap() {
   2.420 -        char[] buf = getRawCharacters();
   2.421 -        return Position.makeLineMap(buf, buf.length, true);
   2.422 -    }
   2.423 -}

     3.1 --- a/src/share/classes/com/sun/tools/javac/parser/EndPosParser.java	Fri Oct 21 14:14:29 2011 -0700
     3.2 +++ b/src/share/classes/com/sun/tools/javac/parser/EndPosParser.java	Mon Oct 24 13:00:20 2011 +0100
     3.3 @@ -67,14 +67,14 @@
     3.4      /** {@inheritDoc} */
     3.5      @Override
     3.6      protected <T extends JCTree> T to(T t) {
     3.7 -        storeEnd(t, S.endPos());
     3.8 +        storeEnd(t, token.endPos);
     3.9          return t;
    3.10      }
    3.11  
    3.12      /** {@inheritDoc} */
    3.13      @Override
    3.14      protected <T extends JCTree> T toP(T t) {
    3.15 -        storeEnd(t, S.prevEndPos());
    3.16 +        storeEnd(t, S.prevToken().endPos);
    3.17          return t;
    3.18      }
    3.19  
    3.20 @@ -88,7 +88,7 @@
    3.21      /** {@inheritDoc} */
    3.22      @Override
    3.23      JCExpression parExpression() {
    3.24 -        int pos = S.pos();
    3.25 +        int pos = token.pos;
    3.26          JCExpression t = super.parExpression();
    3.27          return toP(F.at(pos).Parens(t));
    3.28      }

     4.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     4.2 +++ b/src/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java	Mon Oct 24 13:00:20 2011 +0100
     4.3 @@ -0,0 +1,896 @@
     4.4 +/*
     4.5 + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved.
     4.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
     4.7 + *
     4.8 + * This code is free software; you can redistribute it and/or modify it
     4.9 + * under the terms of the GNU General Public License version 2 only, as
    4.10 + * published by the Free Software Foundation.  Oracle designates this
    4.11 + * particular file as subject to the "Classpath" exception as provided
    4.12 + * by Oracle in the LICENSE file that accompanied this code.
    4.13 + *
    4.14 + * This code is distributed in the hope that it will be useful, but WITHOUT
    4.15 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    4.16 + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
    4.17 + * version 2 for more details (a copy is included in the LICENSE file that
    4.18 + * accompanied this code).
    4.19 + *
    4.20 + * You should have received a copy of the GNU General Public License version
    4.21 + * 2 along with this work; if not, write to the Free Software Foundation,
    4.22 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
    4.23 + *
    4.24 + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
    4.25 + * or visit www.oracle.com if you need additional information or have any
    4.26 + * questions.
    4.27 + */
    4.28 +
    4.29 +package com.sun.tools.javac.parser;
    4.30 +
    4.31 +import java.nio.CharBuffer;
    4.32 +import com.sun.tools.javac.code.Source;
    4.33 +import com.sun.tools.javac.util.*;
    4.34 +
    4.35 +
    4.36 +import static com.sun.tools.javac.parser.Tokens.*;
    4.37 +import static com.sun.tools.javac.util.LayoutCharacters.*;
    4.38 +
    4.39 +/** The lexical analyzer maps an input stream consisting of
    4.40 + *  ASCII characters and Unicode escapes into a token sequence.
    4.41 + *
    4.42 + *  <p><b>This is NOT part of any supported API.
    4.43 + *  If you write code that depends on this, you do so at your own risk.
    4.44 + *  This code and its internal interfaces are subject to change or
    4.45 + *  deletion without notice.</b>
    4.46 + */
    4.47 +public class JavaTokenizer {
    4.48 +
    4.49 +    private static boolean scannerDebug = false;
    4.50 +
    4.51 +    /** Allow hex floating-point literals.
    4.52 +     */
    4.53 +    private boolean allowHexFloats;
    4.54 +
    4.55 +    /** Allow binary literals.
    4.56 +     */
    4.57 +    private boolean allowBinaryLiterals;
    4.58 +
    4.59 +    /** Allow underscores in literals.
    4.60 +     */
    4.61 +    private boolean allowUnderscoresInLiterals;
    4.62 +
    4.63 +    /** The source language setting.
    4.64 +     */
    4.65 +    private Source source;
    4.66 +
    4.67 +    /** The log to be used for error reporting.
    4.68 +     */
    4.69 +    private final Log log;
    4.70 +
    4.71 +    /** The name table. */
    4.72 +    private final Names names;
    4.73 +
    4.74 +    /** The token factory. */
    4.75 +    private final Tokens tokens;
    4.76 +
    4.77 +    /** The token kind, set by nextToken().
    4.78 +     */
    4.79 +    protected TokenKind tk;
    4.80 +
    4.81 +    /** The token's radix, set by nextToken().
    4.82 +     */
    4.83 +    protected int radix;
    4.84 +
    4.85 +    /** The token's name, set by nextToken().
    4.86 +     */
    4.87 +    protected Name name;
    4.88 +
    4.89 +    /** The position where a lexical error occurred;
    4.90 +     */
    4.91 +    protected int errPos = Position.NOPOS;
    4.92 +
    4.93 +    /** Has a @deprecated been encountered in last doc comment?
    4.94 +     *  this needs to be reset by client.
    4.95 +     */
    4.96 +    protected boolean deprecatedFlag = false;
    4.97 +
    4.98 +    /** A character buffer for saved chars.
    4.99 +     */
   4.100 +    protected char[] sbuf = new char[128];
   4.101 +    protected int sp;
   4.102 +
   4.103 +    protected UnicodeReader reader;
   4.104 +
   4.105 +    private static final boolean hexFloatsWork = hexFloatsWork();
   4.106 +    private static boolean hexFloatsWork() {
   4.107 +        try {
   4.108 +            Float.valueOf("0x1.0p1");
   4.109 +            return true;
   4.110 +        } catch (NumberFormatException ex) {
   4.111 +            return false;
   4.112 +        }
   4.113 +    }
   4.114 +
   4.115 +    /**
   4.116 +     * Create a scanner from the input array.  This method might
   4.117 +     * modify the array.  To avoid copying the input array, ensure
   4.118 +     * that {@code inputLength < input.length} or
   4.119 +     * {@code input[input.length -1]} is a white space character.
   4.120 +     *
   4.121 +     * @param fac the factory which created this Scanner
   4.122 +     * @param input the input, might be modified
   4.123 +     * @param inputLength the size of the input.
   4.124 +     * Must be positive and less than or equal to input.length.
   4.125 +     */
   4.126 +    protected JavaTokenizer(ScannerFactory fac, CharBuffer buf) {
   4.127 +        this(fac, new UnicodeReader(fac, buf));
   4.128 +    }
   4.129 +
   4.130 +    protected JavaTokenizer(ScannerFactory fac, char[] buf, int inputLength) {
   4.131 +        this(fac, new UnicodeReader(fac, buf, inputLength));
   4.132 +    }
   4.133 +
   4.134 +    protected JavaTokenizer(ScannerFactory fac, UnicodeReader reader) {
   4.135 +        log = fac.log;
   4.136 +        names = fac.names;
   4.137 +        tokens = fac.tokens;
   4.138 +        source = fac.source;
   4.139 +        this.reader = reader;
   4.140 +        allowBinaryLiterals = source.allowBinaryLiterals();
   4.141 +        allowHexFloats = source.allowHexFloats();
   4.142 +        allowUnderscoresInLiterals = source.allowUnderscoresInLiterals();
   4.143 +    }
   4.144 +
   4.145 +    /** Report an error at the given position using the provided arguments.
   4.146 +     */
   4.147 +    protected void lexError(int pos, String key, Object... args) {
   4.148 +        log.error(pos, key, args);
   4.149 +        tk = TokenKind.ERROR;
   4.150 +        errPos = pos;
   4.151 +    }
   4.152 +
   4.153 +    /** Read next character in comment, skipping over double '\' characters.
   4.154 +     */
   4.155 +    protected void scanCommentChar() {
   4.156 +        reader.scanChar();
   4.157 +        if (reader.ch == '\\') {
   4.158 +            if (reader.peekChar() == '\\' && !reader.isUnicode()) {
   4.159 +                reader.skipChar();
   4.160 +            } else {
   4.161 +                reader.convertUnicode();
   4.162 +            }
   4.163 +        }
   4.164 +    }
   4.165 +
   4.166 +    /** Append a character to sbuf.
   4.167 +     */
   4.168 +    private void putChar(char ch) {
   4.169 +        if (sp == sbuf.length) {
   4.170 +            char[] newsbuf = new char[sbuf.length * 2];
   4.171 +            System.arraycopy(sbuf, 0, newsbuf, 0, sbuf.length);
   4.172 +            sbuf = newsbuf;
   4.173 +        }
   4.174 +        sbuf[sp++] = ch;
   4.175 +    }
   4.176 +
   4.177 +    /** Read next character in character or string literal and copy into sbuf.
   4.178 +     */
   4.179 +    private void scanLitChar(int pos) {
   4.180 +        if (reader.ch == '\\') {
   4.181 +            if (reader.peekChar() == '\\' && !reader.isUnicode()) {
   4.182 +                reader.skipChar();
   4.183 +                putChar('\\');
   4.184 +                reader.scanChar();
   4.185 +            } else {
   4.186 +                reader.scanChar();
   4.187 +                switch (reader.ch) {
   4.188 +                case '0': case '1': case '2': case '3':
   4.189 +                case '4': case '5': case '6': case '7':
   4.190 +                    char leadch = reader.ch;
   4.191 +                    int oct = reader.digit(pos, 8);
   4.192 +                    reader.scanChar();
   4.193 +                    if ('0' <= reader.ch && reader.ch <= '7') {
   4.194 +                        oct = oct * 8 + reader.digit(pos, 8);
   4.195 +                        reader.scanChar();
   4.196 +                        if (leadch <= '3' && '0' <= reader.ch && reader.ch <= '7') {
   4.197 +                            oct = oct * 8 + reader.digit(pos, 8);
   4.198 +                            reader.scanChar();
   4.199 +                        }
   4.200 +                    }
   4.201 +                    putChar((char)oct);
   4.202 +                    break;
   4.203 +                case 'b':
   4.204 +                    putChar('\b'); reader.scanChar(); break;
   4.205 +                case 't':
   4.206 +                    putChar('\t'); reader.scanChar(); break;
   4.207 +                case 'n':
   4.208 +                    putChar('\n'); reader.scanChar(); break;
   4.209 +                case 'f':
   4.210 +                    putChar('\f'); reader.scanChar(); break;
   4.211 +                case 'r':
   4.212 +                    putChar('\r'); reader.scanChar(); break;
   4.213 +                case '\'':
   4.214 +                    putChar('\''); reader.scanChar(); break;
   4.215 +                case '\"':
   4.216 +                    putChar('\"'); reader.scanChar(); break;
   4.217 +                case '\\':
   4.218 +                    putChar('\\'); reader.scanChar(); break;
   4.219 +                default:
   4.220 +                    lexError(reader.bp, "illegal.esc.char");
   4.221 +                }
   4.222 +            }
   4.223 +        } else if (reader.bp != reader.buflen) {
   4.224 +            putChar(reader.ch); reader.scanChar();
   4.225 +        }
   4.226 +    }
   4.227 +
   4.228 +    private void scanDigits(int pos, int digitRadix) {
   4.229 +        char saveCh;
   4.230 +        int savePos;
   4.231 +        do {
   4.232 +            if (reader.ch != '_') {
   4.233 +                putChar(reader.ch);
   4.234 +            } else {
   4.235 +                if (!allowUnderscoresInLiterals) {
   4.236 +                    lexError(pos, "unsupported.underscore.lit", source.name);
   4.237 +                    allowUnderscoresInLiterals = true;
   4.238 +                }
   4.239 +            }
   4.240 +            saveCh = reader.ch;
   4.241 +            savePos = reader.bp;
   4.242 +            reader.scanChar();
   4.243 +        } while (reader.digit(pos, digitRadix) >= 0 || reader.ch == '_');
   4.244 +        if (saveCh == '_')
   4.245 +            lexError(savePos, "illegal.underscore");
   4.246 +    }
   4.247 +
   4.248 +    /** Read fractional part of hexadecimal floating point number.
   4.249 +     */
   4.250 +    private void scanHexExponentAndSuffix(int pos) {
   4.251 +        if (reader.ch == 'p' || reader.ch == 'P') {
   4.252 +            putChar(reader.ch);
   4.253 +            reader.scanChar();
   4.254 +            skipIllegalUnderscores();
   4.255 +            if (reader.ch == '+' || reader.ch == '-') {
   4.256 +                putChar(reader.ch);
   4.257 +                reader.scanChar();
   4.258 +            }
   4.259 +            skipIllegalUnderscores();
   4.260 +            if ('0' <= reader.ch && reader.ch <= '9') {
   4.261 +                scanDigits(pos, 10);
   4.262 +                if (!allowHexFloats) {
   4.263 +                    lexError(pos, "unsupported.fp.lit", source.name);
   4.264 +                    allowHexFloats = true;
   4.265 +                }
   4.266 +                else if (!hexFloatsWork)
   4.267 +                    lexError(pos, "unsupported.cross.fp.lit");
   4.268 +            } else
   4.269 +                lexError(pos, "malformed.fp.lit");
   4.270 +        } else {
   4.271 +            lexError(pos, "malformed.fp.lit");
   4.272 +        }
   4.273 +        if (reader.ch == 'f' || reader.ch == 'F') {
   4.274 +            putChar(reader.ch);
   4.275 +            reader.scanChar();
   4.276 +            tk = TokenKind.FLOATLITERAL;
   4.277 +            radix = 16;
   4.278 +        } else {
   4.279 +            if (reader.ch == 'd' || reader.ch == 'D') {
   4.280 +                putChar(reader.ch);
   4.281 +                reader.scanChar();
   4.282 +            }
   4.283 +            tk = TokenKind.DOUBLELITERAL;
   4.284 +            radix = 16;
   4.285 +        }
   4.286 +    }
   4.287 +
   4.288 +    /** Read fractional part of floating point number.
   4.289 +     */
   4.290 +    private void scanFraction(int pos) {
   4.291 +        skipIllegalUnderscores();
   4.292 +        if ('0' <= reader.ch && reader.ch <= '9') {
   4.293 +            scanDigits(pos, 10);
   4.294 +        }
   4.295 +        int sp1 = sp;
   4.296 +        if (reader.ch == 'e' || reader.ch == 'E') {
   4.297 +            putChar(reader.ch);
   4.298 +            reader.scanChar();
   4.299 +            skipIllegalUnderscores();
   4.300 +            if (reader.ch == '+' || reader.ch == '-') {
   4.301 +                putChar(reader.ch);
   4.302 +                reader.scanChar();
   4.303 +            }
   4.304 +            skipIllegalUnderscores();
   4.305 +            if ('0' <= reader.ch && reader.ch <= '9') {
   4.306 +                scanDigits(pos, 10);
   4.307 +                return;
   4.308 +            }
   4.309 +            lexError(pos, "malformed.fp.lit");
   4.310 +            sp = sp1;
   4.311 +        }
   4.312 +    }
   4.313 +
   4.314 +    /** Read fractional part and 'd' or 'f' suffix of floating point number.
   4.315 +     */
   4.316 +    private void scanFractionAndSuffix(int pos) {
   4.317 +        radix = 10;
   4.318 +        scanFraction(pos);
   4.319 +        if (reader.ch == 'f' || reader.ch == 'F') {
   4.320 +            putChar(reader.ch);
   4.321 +            reader.scanChar();
   4.322 +            tk = TokenKind.FLOATLITERAL;
   4.323 +        } else {
   4.324 +            if (reader.ch == 'd' || reader.ch == 'D') {
   4.325 +                putChar(reader.ch);
   4.326 +                reader.scanChar();
   4.327 +            }
   4.328 +            tk = TokenKind.DOUBLELITERAL;
   4.329 +        }
   4.330 +    }
   4.331 +
   4.332 +    /** Read fractional part and 'd' or 'f' suffix of floating point number.
   4.333 +     */
   4.334 +    private void scanHexFractionAndSuffix(int pos, boolean seendigit) {
   4.335 +        radix = 16;
   4.336 +        Assert.check(reader.ch == '.');
   4.337 +        putChar(reader.ch);
   4.338 +        reader.scanChar();
   4.339 +        skipIllegalUnderscores();
   4.340 +        if (reader.digit(pos, 16) >= 0) {
   4.341 +            seendigit = true;
   4.342 +            scanDigits(pos, 16);
   4.343 +        }
   4.344 +        if (!seendigit)
   4.345 +            lexError(pos, "invalid.hex.number");
   4.346 +        else
   4.347 +            scanHexExponentAndSuffix(pos);
   4.348 +    }
   4.349 +
   4.350 +    private void skipIllegalUnderscores() {
   4.351 +        if (reader.ch == '_') {
   4.352 +            lexError(reader.bp, "illegal.underscore");
   4.353 +            while (reader.ch == '_')
   4.354 +                reader.scanChar();
   4.355 +        }
   4.356 +    }
   4.357 +
   4.358 +    /** Read a number.
   4.359 +     *  @param radix  The radix of the number; one of 2, j8, 10, 16.
   4.360 +     */
   4.361 +    private void scanNumber(int pos, int radix) {
   4.362 +        // for octal, allow base-10 digit in case it's a float literal
   4.363 +        this.radix = radix;
   4.364 +        int digitRadix = (radix == 8 ? 10 : radix);
   4.365 +        boolean seendigit = false;
   4.366 +        if (reader.digit(pos, digitRadix) >= 0) {
   4.367 +            seendigit = true;
   4.368 +            scanDigits(pos, digitRadix);
   4.369 +        }
   4.370 +        if (radix == 16 && reader.ch == '.') {
   4.371 +            scanHexFractionAndSuffix(pos, seendigit);
   4.372 +        } else if (seendigit && radix == 16 && (reader.ch == 'p' || reader.ch == 'P')) {
   4.373 +            scanHexExponentAndSuffix(pos);
   4.374 +        } else if (digitRadix == 10 && reader.ch == '.') {
   4.375 +            putChar(reader.ch);
   4.376 +            reader.scanChar();
   4.377 +            scanFractionAndSuffix(pos);
   4.378 +        } else if (digitRadix == 10 &&
   4.379 +                   (reader.ch == 'e' || reader.ch == 'E' ||
   4.380 +                    reader.ch == 'f' || reader.ch == 'F' ||
   4.381 +                    reader.ch == 'd' || reader.ch == 'D')) {
   4.382 +            scanFractionAndSuffix(pos);
   4.383 +        } else {
   4.384 +            if (reader.ch == 'l' || reader.ch == 'L') {
   4.385 +                reader.scanChar();
   4.386 +                tk = TokenKind.LONGLITERAL;
   4.387 +            } else {
   4.388 +                tk = TokenKind.INTLITERAL;
   4.389 +            }
   4.390 +        }
   4.391 +    }
   4.392 +
   4.393 +    /** Read an identifier.
   4.394 +     */
   4.395 +    private void scanIdent() {
   4.396 +        boolean isJavaIdentifierPart;
   4.397 +        char high;
   4.398 +        do {
   4.399 +            if (sp == sbuf.length) putChar(reader.ch); else sbuf[sp++] = reader.ch;
   4.400 +            // optimization, was: putChar(reader.ch);
   4.401 +
   4.402 +            reader.scanChar();
   4.403 +            switch (reader.ch) {
   4.404 +            case 'A': case 'B': case 'C': case 'D': case 'E':
   4.405 +            case 'F': case 'G': case 'H': case 'I': case 'J':
   4.406 +            case 'K': case 'L': case 'M': case 'N': case 'O':
   4.407 +            case 'P': case 'Q': case 'R': case 'S': case 'T':
   4.408 +            case 'U': case 'V': case 'W': case 'X': case 'Y':
   4.409 +            case 'Z':
   4.410 +            case 'a': case 'b': case 'c': case 'd': case 'e':
   4.411 +            case 'f': case 'g': case 'h': case 'i': case 'j':
   4.412 +            case 'k': case 'l': case 'm': case 'n': case 'o':
   4.413 +            case 'p': case 'q': case 'r': case 's': case 't':
   4.414 +            case 'u': case 'v': case 'w': case 'x': case 'y':
   4.415 +            case 'z':
   4.416 +            case '$': case '_':
   4.417 +            case '0': case '1': case '2': case '3': case '4':
   4.418 +            case '5': case '6': case '7': case '8': case '9':
   4.419 +            case '\u0000': case '\u0001': case '\u0002': case '\u0003':
   4.420 +            case '\u0004': case '\u0005': case '\u0006': case '\u0007':
   4.421 +            case '\u0008': case '\u000E': case '\u000F': case '\u0010':
   4.422 +            case '\u0011': case '\u0012': case '\u0013': case '\u0014':
   4.423 +            case '\u0015': case '\u0016': case '\u0017':
   4.424 +            case '\u0018': case '\u0019': case '\u001B':
   4.425 +            case '\u007F':
   4.426 +                break;
   4.427 +            case '\u001A': // EOI is also a legal identifier part
   4.428 +                if (reader.bp >= reader.buflen) {
   4.429 +                    name = names.fromChars(sbuf, 0, sp);
   4.430 +                    tk = tokens.lookupKind(name);
   4.431 +                    return;
   4.432 +                }
   4.433 +                break;
   4.434 +            default:
   4.435 +                if (reader.ch < '\u0080') {
   4.436 +                    // all ASCII range chars already handled, above
   4.437 +                    isJavaIdentifierPart = false;
   4.438 +                } else {
   4.439 +                    high = reader.scanSurrogates();
   4.440 +                    if (high != 0) {
   4.441 +                        if (sp == sbuf.length) {
   4.442 +                            putChar(high);
   4.443 +                        } else {
   4.444 +                            sbuf[sp++] = high;
   4.445 +                        }
   4.446 +                        isJavaIdentifierPart = Character.isJavaIdentifierPart(
   4.447 +                            Character.toCodePoint(high, reader.ch));
   4.448 +                    } else {
   4.449 +                        isJavaIdentifierPart = Character.isJavaIdentifierPart(reader.ch);
   4.450 +                    }
   4.451 +                }
   4.452 +                if (!isJavaIdentifierPart) {
   4.453 +                    name = names.fromChars(sbuf, 0, sp);
   4.454 +                    tk = tokens.lookupKind(name);
   4.455 +                    return;
   4.456 +                }
   4.457 +            }
   4.458 +        } while (true);
   4.459 +    }
   4.460 +
   4.461 +    /** Return true if reader.ch can be part of an operator.
   4.462 +     */
   4.463 +    private boolean isSpecial(char ch) {
   4.464 +        switch (ch) {
   4.465 +        case '!': case '%': case '&': case '*': case '?':
   4.466 +        case '+': case '-': case ':': case '<': case '=':
   4.467 +        case '>': case '^': case '|': case '~':
   4.468 +        case '@':
   4.469 +            return true;
   4.470 +        default:
   4.471 +            return false;
   4.472 +        }
   4.473 +    }
   4.474 +
   4.475 +    /** Read longest possible sequence of special characters and convert
   4.476 +     *  to token.
   4.477 +     */
   4.478 +    private void scanOperator() {
   4.479 +        while (true) {
   4.480 +            putChar(reader.ch);
   4.481 +            Name newname = names.fromChars(sbuf, 0, sp);
   4.482 +            TokenKind tk1 = tokens.lookupKind(newname);
   4.483 +            if (tk1 == TokenKind.IDENTIFIER) {
   4.484 +                sp--;
   4.485 +                break;
   4.486 +            }
   4.487 +            tk = tk1;
   4.488 +            reader.scanChar();
   4.489 +            if (!isSpecial(reader.ch)) break;
   4.490 +        }
   4.491 +    }
   4.492 +
   4.493 +    /**
   4.494 +     * Scan a documentation comment; determine if a deprecated tag is present.
   4.495 +     * Called once the initial /, * have been skipped, positioned at the second *
   4.496 +     * (which is treated as the beginning of the first line).
   4.497 +     * Stops positioned at the closing '/'.
   4.498 +     */
   4.499 +    @SuppressWarnings("fallthrough")
   4.500 +    private void scanDocComment() {
   4.501 +        boolean deprecatedPrefix = false;
   4.502 +
   4.503 +        forEachLine:
   4.504 +        while (reader.bp < reader.buflen) {
   4.505 +
   4.506 +            // Skip optional WhiteSpace at beginning of line
   4.507 +            while (reader.bp < reader.buflen && (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF)) {
   4.508 +                scanCommentChar();
   4.509 +            }
   4.510 +
   4.511 +            // Skip optional consecutive Stars
   4.512 +            while (reader.bp < reader.buflen && reader.ch == '*') {
   4.513 +                scanCommentChar();
   4.514 +                if (reader.ch == '/') {
   4.515 +                    return;
   4.516 +                }
   4.517 +            }
   4.518 +
   4.519 +            // Skip optional WhiteSpace after Stars
   4.520 +            while (reader.bp < reader.buflen && (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF)) {
   4.521 +                scanCommentChar();
   4.522 +            }
   4.523 +
   4.524 +            deprecatedPrefix = false;
   4.525 +            // At beginning of line in the JavaDoc sense.
   4.526 +            if (reader.bp < reader.buflen && reader.ch == '@' && !deprecatedFlag) {
   4.527 +                scanCommentChar();
   4.528 +                if (reader.bp < reader.buflen && reader.ch == 'd') {
   4.529 +                    scanCommentChar();
   4.530 +                    if (reader.bp < reader.buflen && reader.ch == 'e') {
   4.531 +                        scanCommentChar();
   4.532 +                        if (reader.bp < reader.buflen && reader.ch == 'p') {
   4.533 +                            scanCommentChar();
   4.534 +                            if (reader.bp < reader.buflen && reader.ch == 'r') {
   4.535 +                                scanCommentChar();
   4.536 +                                if (reader.bp < reader.buflen && reader.ch == 'e') {
   4.537 +                                    scanCommentChar();
   4.538 +                                    if (reader.bp < reader.buflen && reader.ch == 'c') {
   4.539 +                                        scanCommentChar();
   4.540 +                                        if (reader.bp < reader.buflen && reader.ch == 'a') {
   4.541 +                                            scanCommentChar();
   4.542 +                                            if (reader.bp < reader.buflen && reader.ch == 't') {
   4.543 +                                                scanCommentChar();
   4.544 +                                                if (reader.bp < reader.buflen && reader.ch == 'e') {
   4.545 +                                                    scanCommentChar();
   4.546 +                                                    if (reader.bp < reader.buflen && reader.ch == 'd') {
   4.547 +                                                        deprecatedPrefix = true;
   4.548 +                                                        scanCommentChar();
   4.549 +                                                    }}}}}}}}}}}
   4.550 +            if (deprecatedPrefix && reader.bp < reader.buflen) {
   4.551 +                if (Character.isWhitespace(reader.ch)) {
   4.552 +                    deprecatedFlag = true;
   4.553 +                } else if (reader.ch == '*') {
   4.554 +                    scanCommentChar();
   4.555 +                    if (reader.ch == '/') {
   4.556 +                        deprecatedFlag = true;
   4.557 +                        return;
   4.558 +                    }
   4.559 +                }
   4.560 +            }
   4.561 +
   4.562 +            // Skip rest of line
   4.563 +            while (reader.bp < reader.buflen) {
   4.564 +                switch (reader.ch) {
   4.565 +                case '*':
   4.566 +                    scanCommentChar();
   4.567 +                    if (reader.ch == '/') {
   4.568 +                        return;
   4.569 +                    }
   4.570 +                    break;
   4.571 +                case CR: // (Spec 3.4)
   4.572 +                    scanCommentChar();
   4.573 +                    if (reader.ch != LF) {
   4.574 +                        continue forEachLine;
   4.575 +                    }
   4.576 +                    /* fall through to LF case */
   4.577 +                case LF: // (Spec 3.4)
   4.578 +                    scanCommentChar();
   4.579 +                    continue forEachLine;
   4.580 +                default:
   4.581 +                    scanCommentChar();
   4.582 +                }
   4.583 +            } // rest of line
   4.584 +        } // forEachLine
   4.585 +        return;
   4.586 +    }
   4.587 +
   4.588 +    /** Read token.
   4.589 +     */
   4.590 +    public Token readToken() {
   4.591 +
   4.592 +        sp = 0;
   4.593 +        name = null;
   4.594 +        deprecatedFlag = false;
   4.595 +        radix = 0;
   4.596 +        int pos = 0;
   4.597 +        int endPos = 0;
   4.598 +
   4.599 +        try {
   4.600 +            loop: while (true) {
   4.601 +                pos = reader.bp;
   4.602 +                switch (reader.ch) {
   4.603 +                case ' ': // (Spec 3.6)
   4.604 +                case '\t': // (Spec 3.6)
   4.605 +                case FF: // (Spec 3.6)
   4.606 +                    do {
   4.607 +                        reader.scanChar();
   4.608 +                    } while (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF);
   4.609 +                    processWhiteSpace(pos, reader.bp);
   4.610 +                    break;
   4.611 +                case LF: // (Spec 3.4)
   4.612 +                    reader.scanChar();
   4.613 +                    processLineTerminator(pos, reader.bp);
   4.614 +                    break;
   4.615 +                case CR: // (Spec 3.4)
   4.616 +                    reader.scanChar();
   4.617 +                    if (reader.ch == LF) {
   4.618 +                        reader.scanChar();
   4.619 +                    }
   4.620 +                    processLineTerminator(pos, reader.bp);
   4.621 +                    break;
   4.622 +                case 'A': case 'B': case 'C': case 'D': case 'E':
   4.623 +                case 'F': case 'G': case 'H': case 'I': case 'J':
   4.624 +                case 'K': case 'L': case 'M': case 'N': case 'O':
   4.625 +                case 'P': case 'Q': case 'R': case 'S': case 'T':
   4.626 +                case 'U': case 'V': case 'W': case 'X': case 'Y':
   4.627 +                case 'Z':
   4.628 +                case 'a': case 'b': case 'c': case 'd': case 'e':
   4.629 +                case 'f': case 'g': case 'h': case 'i': case 'j':
   4.630 +                case 'k': case 'l': case 'm': case 'n': case 'o':
   4.631 +                case 'p': case 'q': case 'r': case 's': case 't':
   4.632 +                case 'u': case 'v': case 'w': case 'x': case 'y':
   4.633 +                case 'z':
   4.634 +                case '$': case '_':
   4.635 +                    scanIdent();
   4.636 +                    break loop;
   4.637 +                case '0':
   4.638 +                    reader.scanChar();
   4.639 +                    if (reader.ch == 'x' || reader.ch == 'X') {
   4.640 +                        reader.scanChar();
   4.641 +                        skipIllegalUnderscores();
   4.642 +                        if (reader.ch == '.') {
   4.643 +                            scanHexFractionAndSuffix(pos, false);
   4.644 +                        } else if (reader.digit(pos, 16) < 0) {
   4.645 +                            lexError(pos, "invalid.hex.number");
   4.646 +                        } else {
   4.647 +                            scanNumber(pos, 16);
   4.648 +                        }
   4.649 +                    } else if (reader.ch == 'b' || reader.ch == 'B') {
   4.650 +                        if (!allowBinaryLiterals) {
   4.651 +                            lexError(pos, "unsupported.binary.lit", source.name);
   4.652 +                            allowBinaryLiterals = true;
   4.653 +                        }
   4.654 +                        reader.scanChar();
   4.655 +                        skipIllegalUnderscores();
   4.656 +                        if (reader.digit(pos, 2) < 0) {
   4.657 +                            lexError(pos, "invalid.binary.number");
   4.658 +                        } else {
   4.659 +                            scanNumber(pos, 2);
   4.660 +                        }
   4.661 +                    } else {
   4.662 +                        putChar('0');
   4.663 +                        if (reader.ch == '_') {
   4.664 +                            int savePos = reader.bp;
   4.665 +                            do {
   4.666 +                                reader.scanChar();
   4.667 +                            } while (reader.ch == '_');
   4.668 +                            if (reader.digit(pos, 10) < 0) {
   4.669 +                                lexError(savePos, "illegal.underscore");
   4.670 +                            }
   4.671 +                        }
   4.672 +                        scanNumber(pos, 8);
   4.673 +                    }
   4.674 +                    break loop;
   4.675 +                case '1': case '2': case '3': case '4':
   4.676 +                case '5': case '6': case '7': case '8': case '9':
   4.677 +                    scanNumber(pos, 10);
   4.678 +                    break loop;
   4.679 +                case '.':
   4.680 +                    reader.scanChar();
   4.681 +                    if ('0' <= reader.ch && reader.ch <= '9') {
   4.682 +                        putChar('.');
   4.683 +                        scanFractionAndSuffix(pos);
   4.684 +                    } else if (reader.ch == '.') {
   4.685 +                        putChar('.'); putChar('.');
   4.686 +                        reader.scanChar();
   4.687 +                        if (reader.ch == '.') {
   4.688 +                            reader.scanChar();
   4.689 +                            putChar('.');
   4.690 +                            tk = TokenKind.ELLIPSIS;
   4.691 +                        } else {
   4.692 +                            lexError(pos, "malformed.fp.lit");
   4.693 +                        }
   4.694 +                    } else {
   4.695 +                        tk = TokenKind.DOT;
   4.696 +                    }
   4.697 +                    break loop;
   4.698 +                case ',':
   4.699 +                    reader.scanChar(); tk = TokenKind.COMMA; break loop;
   4.700 +                case ';':
   4.701 +                    reader.scanChar(); tk = TokenKind.SEMI; break loop;
   4.702 +                case '(':
   4.703 +                    reader.scanChar(); tk = TokenKind.LPAREN; break loop;
   4.704 +                case ')':
   4.705 +                    reader.scanChar(); tk = TokenKind.RPAREN; break loop;
   4.706 +                case '[':
   4.707 +                    reader.scanChar(); tk = TokenKind.LBRACKET; break loop;
   4.708 +                case ']':
   4.709 +                    reader.scanChar(); tk = TokenKind.RBRACKET; break loop;
   4.710 +                case '{':
   4.711 +                    reader.scanChar(); tk = TokenKind.LBRACE; break loop;
   4.712 +                case '}':
   4.713 +                    reader.scanChar(); tk = TokenKind.RBRACE; break loop;
   4.714 +                case '/':
   4.715 +                    reader.scanChar();
   4.716 +                    if (reader.ch == '/') {
   4.717 +                        do {
   4.718 +                            scanCommentChar();
   4.719 +                        } while (reader.ch != CR && reader.ch != LF && reader.bp < reader.buflen);
   4.720 +                        if (reader.bp < reader.buflen) {
   4.721 +                            processComment(pos, reader.bp, CommentStyle.LINE);
   4.722 +                        }
   4.723 +                        break;
   4.724 +                    } else if (reader.ch == '*') {
   4.725 +                        reader.scanChar();
   4.726 +                        CommentStyle style;
   4.727 +                        if (reader.ch == '*') {
   4.728 +                            style = CommentStyle.JAVADOC;
   4.729 +                            scanDocComment();
   4.730 +                        } else {
   4.731 +                            style = CommentStyle.BLOCK;
   4.732 +                            while (reader.bp < reader.buflen) {
   4.733 +                                if (reader.ch == '*') {
   4.734 +                                    reader.scanChar();
   4.735 +                                    if (reader.ch == '/') break;
   4.736 +                                } else {
   4.737 +                                    scanCommentChar();
   4.738 +                                }
   4.739 +                            }
   4.740 +                        }
   4.741 +                        if (reader.ch == '/') {
   4.742 +                            reader.scanChar();
   4.743 +                            processComment(pos, reader.bp, style);
   4.744 +                            break;
   4.745 +                        } else {
   4.746 +                            lexError(pos, "unclosed.comment");
   4.747 +                            break loop;
   4.748 +                        }
   4.749 +                    } else if (reader.ch == '=') {
   4.750 +                        tk = TokenKind.SLASHEQ;
   4.751 +                        reader.scanChar();
   4.752 +                    } else {
   4.753 +                        tk = TokenKind.SLASH;
   4.754 +                    }
   4.755 +                    break loop;
   4.756 +                case '\'':
   4.757 +                    reader.scanChar();
   4.758 +                    if (reader.ch == '\'') {
   4.759 +                        lexError(pos, "empty.char.lit");
   4.760 +                    } else {
   4.761 +                        if (reader.ch == CR || reader.ch == LF)
   4.762 +                            lexError(pos, "illegal.line.end.in.char.lit");
   4.763 +                        scanLitChar(pos);
   4.764 +                        char ch2 = reader.ch;
   4.765 +                        if (reader.ch == '\'') {
   4.766 +                            reader.scanChar();
   4.767 +                            tk = TokenKind.CHARLITERAL;
   4.768 +                        } else {
   4.769 +                            lexError(pos, "unclosed.char.lit");
   4.770 +                        }
   4.771 +                    }
   4.772 +                    break loop;
   4.773 +                case '\"':
   4.774 +                    reader.scanChar();
   4.775 +                    while (reader.ch != '\"' && reader.ch != CR && reader.ch != LF && reader.bp < reader.buflen)
   4.776 +                        scanLitChar(pos);
   4.777 +                    if (reader.ch == '\"') {
   4.778 +                        tk = TokenKind.STRINGLITERAL;
   4.779 +                        reader.scanChar();
   4.780 +                    } else {
   4.781 +                        lexError(pos, "unclosed.str.lit");
   4.782 +                    }
   4.783 +                    break loop;
   4.784 +                default:
   4.785 +                    if (isSpecial(reader.ch)) {
   4.786 +                        scanOperator();
   4.787 +                    } else {
   4.788 +                        boolean isJavaIdentifierStart;
   4.789 +                        if (reader.ch < '\u0080') {
   4.790 +                            // all ASCII range chars already handled, above
   4.791 +                            isJavaIdentifierStart = false;
   4.792 +                        } else {
   4.793 +                            char high = reader.scanSurrogates();
   4.794 +                            if (high != 0) {
   4.795 +                                if (sp == sbuf.length) {
   4.796 +                                    putChar(high);
   4.797 +                                } else {
   4.798 +                                    sbuf[sp++] = high;
   4.799 +                                }
   4.800 +
   4.801 +                                isJavaIdentifierStart = Character.isJavaIdentifierStart(
   4.802 +                                    Character.toCodePoint(high, reader.ch));
   4.803 +                            } else {
   4.804 +                                isJavaIdentifierStart = Character.isJavaIdentifierStart(reader.ch);
   4.805 +                            }
   4.806 +                        }
   4.807 +                        if (isJavaIdentifierStart) {
   4.808 +                            scanIdent();
   4.809 +                        } else if (reader.bp == reader.buflen || reader.ch == EOI && reader.bp + 1 == reader.buflen) { // JLS 3.5
   4.810 +                            tk = TokenKind.EOF;
   4.811 +                            pos = reader.buflen;
   4.812 +                        } else {
   4.813 +                            lexError(pos, "illegal.char", String.valueOf((int)reader.ch));
   4.814 +                            reader.scanChar();
   4.815 +                        }
   4.816 +                    }
   4.817 +                    break loop;
   4.818 +                }
   4.819 +            }
   4.820 +            endPos = reader.bp;
   4.821 +            switch (tk.tag) {
   4.822 +                case DEFAULT: return new Token(tk, pos, endPos, deprecatedFlag);
   4.823 +                case NAMED: return new NamedToken(tk, pos, endPos, name, deprecatedFlag);
   4.824 +                case STRING: return new StringToken(tk, pos, endPos, new String(sbuf, 0, sp), deprecatedFlag);
   4.825 +                case NUMERIC: return new NumericToken(tk, pos, endPos, new String(sbuf, 0, sp), radix, deprecatedFlag);
   4.826 +                default: throw new AssertionError();
   4.827 +            }
   4.828 +        }
   4.829 +        finally {
   4.830 +            if (scannerDebug) {
   4.831 +                    System.out.println("nextToken(" + pos
   4.832 +                                       + "," + endPos + ")=|" +
   4.833 +                                       new String(reader.getRawCharacters(pos, endPos))
   4.834 +                                       + "|");
   4.835 +            }
   4.836 +        }
   4.837 +    }
   4.838 +
   4.839 +    /** Return the position where a lexical error occurred;
   4.840 +     */
   4.841 +    public int errPos() {
   4.842 +        return errPos;
   4.843 +    }
   4.844 +
   4.845 +    /** Set the position where a lexical error occurred;
   4.846 +     */
   4.847 +    public void errPos(int pos) {
   4.848 +        errPos = pos;
   4.849 +    }
   4.850 +
   4.851 +    public enum CommentStyle {
   4.852 +        LINE,
   4.853 +        BLOCK,
   4.854 +        JAVADOC,
   4.855 +    }
   4.856 +
   4.857 +    /**
   4.858 +     * Called when a complete comment has been scanned. pos and endPos
   4.859 +     * will mark the comment boundary.
   4.860 +     */
   4.861 +    protected void processComment(int pos, int endPos, CommentStyle style) {
   4.862 +        if (scannerDebug)
   4.863 +            System.out.println("processComment(" + pos
   4.864 +                               + "," + endPos + "," + style + ")=|"
   4.865 +                               + new String(reader.getRawCharacters(pos, endPos))
   4.866 +                               + "|");
   4.867 +    }
   4.868 +
   4.869 +    /**
   4.870 +     * Called when a complete whitespace run has been scanned. pos and endPos
   4.871 +     * will mark the whitespace boundary.
   4.872 +     */
   4.873 +    protected void processWhiteSpace(int pos, int endPos) {
   4.874 +        if (scannerDebug)
   4.875 +            System.out.println("processWhitespace(" + pos
   4.876 +                               + "," + endPos + ")=|" +
   4.877 +                               new String(reader.getRawCharacters(pos, endPos))
   4.878 +                               + "|");
   4.879 +    }
   4.880 +
   4.881 +    /**
   4.882 +     * Called when a line terminator has been processed.
   4.883 +     */
   4.884 +    protected void processLineTerminator(int pos, int endPos) {
   4.885 +        if (scannerDebug)
   4.886 +            System.out.println("processTerminator(" + pos
   4.887 +                               + "," + endPos + ")=|" +
   4.888 +                               new String(reader.getRawCharacters(pos, endPos))
   4.889 +                               + "|");
   4.890 +    }
   4.891 +
   4.892 +    /** Build a map for translating between line numbers and
   4.893 +     * positions in the input.
   4.894 +     *
   4.895 +     * @return a LineMap */
   4.896 +    public Position.LineMap getLineMap() {
   4.897 +        return Position.makeLineMap(reader.getRawCharacters(), reader.buflen, false);
   4.898 +    }
   4.899 +}

     5.1 --- a/src/share/classes/com/sun/tools/javac/parser/JavacParser.java	Fri Oct 21 14:14:29 2011 -0700
     5.2 +++ b/src/share/classes/com/sun/tools/javac/parser/JavacParser.java	Mon Oct 24 13:00:20 2011 +0100
     5.3 @@ -28,6 +28,7 @@
     5.4  import java.util.*;
     5.5  
     5.6  import com.sun.tools.javac.code.*;
     5.7 +import com.sun.tools.javac.parser.Tokens.*;
     5.8  import com.sun.tools.javac.tree.*;
     5.9  import com.sun.tools.javac.tree.JCTree.*;
    5.10  import com.sun.tools.javac.util.*;
    5.11 @@ -36,7 +37,7 @@
    5.12  import com.sun.tools.javac.util.List;
    5.13  
    5.14  import static com.sun.tools.javac.util.ListBuffer.lb;
    5.15 -import static com.sun.tools.javac.parser.Token.*;
    5.16 +import static com.sun.tools.javac.parser.Tokens.TokenKind.*;
    5.17  
    5.18  /** The parser maps a token sequence into an abstract syntax
    5.19   *  tree. It operates by recursive descent, with code derived
    5.20 @@ -67,9 +68,6 @@
    5.21       */
    5.22      private Log log;
    5.23  
    5.24 -    /** The keyword table. */
    5.25 -    private Keywords keywords;
    5.26 -
    5.27      /** The Source language setting. */
    5.28      private Source source;
    5.29  
    5.30 @@ -83,11 +81,10 @@
    5.31                       boolean keepDocComments,
    5.32                       boolean keepLineMap) {
    5.33          this.S = S;
    5.34 -        S.nextToken(); // prime the pump
    5.35 +        nextToken(); // prime the pump
    5.36          this.F = fac.F;
    5.37          this.log = fac.log;
    5.38          this.names = fac.names;
    5.39 -        this.keywords = fac.keywords;
    5.40          this.source = fac.source;
    5.41          this.allowGenerics = source.allowGenerics();
    5.42          this.allowVarargs = source.allowVarargs();
    5.43 @@ -178,7 +175,16 @@
    5.44       */
    5.45      private int lastmode = 0;
    5.46  
    5.47 -/* ---------- error recovery -------------- */
    5.48 +    /* ---------- token management -------------- */
    5.49 +
    5.50 +    protected Token token;
    5.51 +
    5.52 +    protected void nextToken() {
    5.53 +        S.nextToken();
    5.54 +        token = S.token();
    5.55 +    }
    5.56 +
    5.57 +    /* ---------- error recovery -------------- */
    5.58  
    5.59      private JCErroneous errorTree;
    5.60  
    5.61 @@ -186,9 +192,9 @@
    5.62       */
    5.63      private void skip(boolean stopAtImport, boolean stopAtMemberDecl, boolean stopAtIdentifier, boolean stopAtStatement) {
    5.64           while (true) {
    5.65 -             switch (S.token()) {
    5.66 +             switch (token.kind) {
    5.67                  case SEMI:
    5.68 -                    S.nextToken();
    5.69 +                    nextToken();
    5.70                      return;
    5.71                  case PUBLIC:
    5.72                  case FINAL:
    5.73 @@ -249,15 +255,15 @@
    5.74                          return;
    5.75                      break;
    5.76              }
    5.77 -            S.nextToken();
    5.78 +            nextToken();
    5.79          }
    5.80      }
    5.81  
    5.82 -    private JCErroneous syntaxError(int pos, String key, Token... args) {
    5.83 +    private JCErroneous syntaxError(int pos, String key, TokenKind... args) {
    5.84          return syntaxError(pos, List.<JCTree>nil(), key, args);
    5.85      }
    5.86  
    5.87 -    private JCErroneous syntaxError(int pos, List<JCTree> errs, String key, Token... args) {
    5.88 +    private JCErroneous syntaxError(int pos, List<JCTree> errs, String key, TokenKind... args) {
    5.89          setErrorEndPos(pos);
    5.90          JCErroneous err = F.at(pos).Erroneous(errs);
    5.91          reportSyntaxError(err, key, (Object[])args);
    5.92 @@ -287,16 +293,16 @@
    5.93      private void reportSyntaxError(JCDiagnostic.DiagnosticPosition diagPos, String key, Object... args) {
    5.94          int pos = diagPos.getPreferredPosition();
    5.95          if (pos > S.errPos() || pos == Position.NOPOS) {
    5.96 -            if (S.token() == EOF) {
    5.97 +            if (token.kind == EOF) {
    5.98                  error(diagPos, "premature.eof");
    5.99              } else {
   5.100                  error(diagPos, key, args);
   5.101              }
   5.102          }
   5.103          S.errPos(pos);
   5.104 -        if (S.pos() == errorPos)
   5.105 -            S.nextToken(); // guarantee progress
   5.106 -        errorPos = S.pos();
   5.107 +        if (token.pos == errorPos)
   5.108 +            nextToken(); // guarantee progress
   5.109 +        errorPos = token.pos;
   5.110      }
   5.111  
   5.112  
   5.113 @@ -304,25 +310,25 @@
   5.114       *  reported at the same position.
   5.115       */
   5.116      private JCErroneous syntaxError(String key) {
   5.117 -        return syntaxError(S.pos(), key);
   5.118 +        return syntaxError(token.pos, key);
   5.119      }
   5.120  
   5.121      /** Generate a syntax error at current position unless one was
   5.122       *  already reported at the same position.
   5.123       */
   5.124 -    private JCErroneous syntaxError(String key, Token arg) {
   5.125 -        return syntaxError(S.pos(), key, arg);
   5.126 +    private JCErroneous syntaxError(String key, TokenKind arg) {
   5.127 +        return syntaxError(token.pos, key, arg);
   5.128      }
   5.129  
   5.130      /** If next input token matches given token, skip it, otherwise report
   5.131       *  an error.
   5.132       */
   5.133 -    public void accept(Token token) {
   5.134 -        if (S.token() == token) {
   5.135 -            S.nextToken();
   5.136 +    public void accept(TokenKind tk) {
   5.137 +        if (token.kind == tk) {
   5.138 +            nextToken();
   5.139          } else {
   5.140 -            setErrorEndPos(S.pos());
   5.141 -            reportSyntaxError(S.prevEndPos(), "expected", token);
   5.142 +            setErrorEndPos(token.pos);
   5.143 +            reportSyntaxError(S.prevToken().endPos, "expected", tk);
   5.144          }
   5.145      }
   5.146  
   5.147 @@ -340,14 +346,14 @@
   5.148      /** Report an illegal start of expression/type error at current position.
   5.149       */
   5.150      JCExpression illegal() {
   5.151 -        return illegal(S.pos());
   5.152 +        return illegal(token.pos);
   5.153      }
   5.154  
   5.155      /** Diagnose a modifier flag from the set, if any. */
   5.156      void checkNoMods(long mods) {
   5.157          if (mods != 0) {
   5.158              long lowestMod = mods & -mods;
   5.159 -            error(S.pos(), "mod.not.allowed.here",
   5.160 +            error(token.pos, "mod.not.allowed.here",
   5.161                        Flags.asFlagSet(lowestMod));
   5.162          }
   5.163      }
   5.164 @@ -435,30 +441,30 @@
   5.165       * Ident = IDENTIFIER
   5.166       */
   5.167      Name ident() {
   5.168 -        if (S.token() == IDENTIFIER) {
   5.169 -            Name name = S.name();
   5.170 -            S.nextToken();
   5.171 +        if (token.kind == IDENTIFIER) {
   5.172 +            Name name = token.name();
   5.173 +            nextToken();
   5.174              return name;
   5.175 -        } else if (S.token() == ASSERT) {
   5.176 +        } else if (token.kind == ASSERT) {
   5.177              if (allowAsserts) {
   5.178 -                error(S.pos(), "assert.as.identifier");
   5.179 -                S.nextToken();
   5.180 +                error(token.pos, "assert.as.identifier");
   5.181 +                nextToken();
   5.182                  return names.error;
   5.183              } else {
   5.184 -                warning(S.pos(), "assert.as.identifier");
   5.185 -                Name name = S.name();
   5.186 -                S.nextToken();
   5.187 +                warning(token.pos, "assert.as.identifier");
   5.188 +                Name name = token.name();
   5.189 +                nextToken();
   5.190                  return name;
   5.191              }
   5.192 -        } else if (S.token() == ENUM) {
   5.193 +        } else if (token.kind == ENUM) {
   5.194              if (allowEnums) {
   5.195 -                error(S.pos(), "enum.as.identifier");
   5.196 -                S.nextToken();
   5.197 +                error(token.pos, "enum.as.identifier");
   5.198 +                nextToken();
   5.199                  return names.error;
   5.200              } else {
   5.201 -                warning(S.pos(), "enum.as.identifier");
   5.202 -                Name name = S.name();
   5.203 -                S.nextToken();
   5.204 +                warning(token.pos, "enum.as.identifier");
   5.205 +                Name name = token.name();
   5.206 +                nextToken();
   5.207                  return name;
   5.208              }
   5.209          } else {
   5.210 @@ -471,17 +477,17 @@
   5.211       * Qualident = Ident { DOT Ident }
   5.212       */
   5.213      public JCExpression qualident() {
   5.214 -        JCExpression t = toP(F.at(S.pos()).Ident(ident()));
   5.215 -        while (S.token() == DOT) {
   5.216 -            int pos = S.pos();
   5.217 -            S.nextToken();
   5.218 +        JCExpression t = toP(F.at(token.pos).Ident(ident()));
   5.219 +        while (token.kind == DOT) {
   5.220 +            int pos = token.pos;
   5.221 +            nextToken();
   5.222              t = toP(F.at(pos).Select(t, ident()));
   5.223          }
   5.224          return t;
   5.225      }
   5.226  
   5.227      JCExpression literal(Name prefix) {
   5.228 -        return literal(prefix, S.pos());
   5.229 +        return literal(prefix, token.pos);
   5.230      }
   5.231  
   5.232      /**
   5.233 @@ -498,27 +504,29 @@
   5.234       */
   5.235      JCExpression literal(Name prefix, int pos) {
   5.236          JCExpression t = errorTree;
   5.237 -        switch (S.token()) {
   5.238 +        switch (token.kind) {
   5.239          case INTLITERAL:
   5.240              try {
   5.241                  t = F.at(pos).Literal(
   5.242                      TypeTags.INT,
   5.243 -                    Convert.string2int(strval(prefix), S.radix()));
   5.244 +                    Convert.string2int(strval(prefix), token.radix()));
   5.245              } catch (NumberFormatException ex) {
   5.246 -                error(S.pos(), "int.number.too.large", strval(prefix));
   5.247 +                error(token.pos, "int.number.too.large", strval(prefix));
   5.248              }
   5.249              break;
   5.250          case LONGLITERAL:
   5.251              try {
   5.252                  t = F.at(pos).Literal(
   5.253                      TypeTags.LONG,
   5.254 -                    new Long(Convert.string2long(strval(prefix), S.radix())));
   5.255 +                    new Long(Convert.string2long(strval(prefix), token.radix())));
   5.256              } catch (NumberFormatException ex) {
   5.257 -                error(S.pos(), "int.number.too.large", strval(prefix));
   5.258 +                error(token.pos, "int.number.too.large", strval(prefix));
   5.259              }
   5.260              break;
   5.261          case FLOATLITERAL: {
   5.262 -            String proper = (S.radix() == 16 ? ("0x"+ S.stringVal()) : S.stringVal());
   5.263 +            String proper = token.radix() == 16 ?
   5.264 +                    ("0x"+ token.stringVal()) :
   5.265 +                    token.stringVal();
   5.266              Float n;
   5.267              try {
   5.268                  n = Float.valueOf(proper);
   5.269 @@ -527,15 +535,17 @@
   5.270                  n = Float.NaN;
   5.271              }
   5.272              if (n.floatValue() == 0.0f && !isZero(proper))
   5.273 -                error(S.pos(), "fp.number.too.small");
   5.274 +                error(token.pos, "fp.number.too.small");
   5.275              else if (n.floatValue() == Float.POSITIVE_INFINITY)
   5.276 -                error(S.pos(), "fp.number.too.large");
   5.277 +                error(token.pos, "fp.number.too.large");
   5.278              else
   5.279                  t = F.at(pos).Literal(TypeTags.FLOAT, n);
   5.280              break;
   5.281          }
   5.282          case DOUBLELITERAL: {
   5.283 -            String proper = (S.radix() == 16 ? ("0x"+ S.stringVal()) : S.stringVal());
   5.284 +            String proper = token.radix() == 16 ?
   5.285 +                    ("0x"+ token.stringVal()) :
   5.286 +                    token.stringVal();
   5.287              Double n;
   5.288              try {
   5.289                  n = Double.valueOf(proper);
   5.290 @@ -544,9 +554,9 @@
   5.291                  n = Double.NaN;
   5.292              }
   5.293              if (n.doubleValue() == 0.0d && !isZero(proper))
   5.294 -                error(S.pos(), "fp.number.too.small");
   5.295 +                error(token.pos, "fp.number.too.small");
   5.296              else if (n.doubleValue() == Double.POSITIVE_INFINITY)
   5.297 -                error(S.pos(), "fp.number.too.large");
   5.298 +                error(token.pos, "fp.number.too.large");
   5.299              else
   5.300                  t = F.at(pos).Literal(TypeTags.DOUBLE, n);
   5.301              break;
   5.302 @@ -554,17 +564,17 @@
   5.303          case CHARLITERAL:
   5.304              t = F.at(pos).Literal(
   5.305                  TypeTags.CHAR,
   5.306 -                S.stringVal().charAt(0) + 0);
   5.307 +                token.stringVal().charAt(0) + 0);
   5.308              break;
   5.309          case STRINGLITERAL:
   5.310              t = F.at(pos).Literal(
   5.311                  TypeTags.CLASS,
   5.312 -                S.stringVal());
   5.313 +                token.stringVal());
   5.314              break;
   5.315          case TRUE: case FALSE:
   5.316              t = F.at(pos).Literal(
   5.317                  TypeTags.BOOLEAN,
   5.318 -                (S.token() == TRUE ? 1 : 0));
   5.319 +                (token.kind == TRUE ? 1 : 0));
   5.320              break;
   5.321          case NULL:
   5.322              t = F.at(pos).Literal(
   5.323 @@ -576,8 +586,8 @@
   5.324          }
   5.325          if (t == errorTree)
   5.326              t = F.at(pos).Erroneous();
   5.327 -        storeEnd(t, S.endPos());
   5.328 -        S.nextToken();
   5.329 +        storeEnd(t, token.endPos);
   5.330 +        nextToken();
   5.331          return t;
   5.332      }
   5.333  //where
   5.334 @@ -590,7 +600,7 @@
   5.335          }
   5.336  
   5.337          String strval(Name prefix) {
   5.338 -            String s = S.stringVal();
   5.339 +            String s = token.stringVal();
   5.340              return prefix.isEmpty() ? s : prefix + s;
   5.341          }
   5.342  
   5.343 @@ -627,17 +637,17 @@
   5.344      JCExpression term() {
   5.345          JCExpression t = term1();
   5.346          if ((mode & EXPR) != 0 &&
   5.347 -            S.token() == EQ || PLUSEQ.compareTo(S.token()) <= 0 && S.token().compareTo(GTGTGTEQ) <= 0)
   5.348 +            token.kind == EQ || PLUSEQ.compareTo(token.kind) <= 0 && token.kind.compareTo(GTGTGTEQ) <= 0)
   5.349              return termRest(t);
   5.350          else
   5.351              return t;
   5.352      }
   5.353  
   5.354      JCExpression termRest(JCExpression t) {
   5.355 -        switch (S.token()) {
   5.356 +        switch (token.kind) {
   5.357          case EQ: {
   5.358 -            int pos = S.pos();
   5.359 -            S.nextToken();
   5.360 +            int pos = token.pos;
   5.361 +            nextToken();
   5.362              mode = EXPR;
   5.363              JCExpression t1 = term();
   5.364              return toP(F.at(pos).Assign(t, t1));
   5.365 @@ -653,12 +663,12 @@
   5.366          case LTLTEQ:
   5.367          case GTGTEQ:
   5.368          case GTGTGTEQ:
   5.369 -            int pos = S.pos();
   5.370 -            Token token = S.token();
   5.371 -            S.nextToken();
   5.372 +            int pos = token.pos;
   5.373 +            TokenKind tk = token.kind;
   5.374 +            nextToken();
   5.375              mode = EXPR;
   5.376              JCExpression t1 = term();
   5.377 -            return F.at(pos).Assignop(optag(token), t, t1);
   5.378 +            return F.at(pos).Assignop(optag(tk), t, t1);
   5.379          default:
   5.380              return t;
   5.381          }
   5.382 @@ -670,7 +680,7 @@
   5.383       */
   5.384      JCExpression term1() {
   5.385          JCExpression t = term2();
   5.386 -        if ((mode & EXPR) != 0 && S.token() == QUES) {
   5.387 +        if ((mode & EXPR) != 0 && token.kind == QUES) {
   5.388              mode = EXPR;
   5.389              return term1Rest(t);
   5.390          } else {
   5.391 @@ -681,9 +691,9 @@
   5.392      /** Expression1Rest = ["?" Expression ":" Expression1]
   5.393       */
   5.394      JCExpression term1Rest(JCExpression t) {
   5.395 -        if (S.token() == QUES) {
   5.396 -            int pos = S.pos();
   5.397 -            S.nextToken();
   5.398 +        if (token.kind == QUES) {
   5.399 +            int pos = token.pos;
   5.400 +            nextToken();
   5.401              JCExpression t1 = term();
   5.402              accept(COLON);
   5.403              JCExpression t2 = term1();
   5.404 @@ -699,7 +709,7 @@
   5.405       */
   5.406      JCExpression term2() {
   5.407          JCExpression t = term3();
   5.408 -        if ((mode & EXPR) != 0 && prec(S.token()) >= TreeInfo.orPrec) {
   5.409 +        if ((mode & EXPR) != 0 && prec(token.kind) >= TreeInfo.orPrec) {
   5.410              mode = EXPR;
   5.411              return term2Rest(t, TreeInfo.orPrec);
   5.412          } else {
   5.413 @@ -725,28 +735,23 @@
   5.414          JCExpression[] odStack = newOdStack();
   5.415          List<Token[]> savedOp = opStackSupply.elems;
   5.416          Token[] opStack = newOpStack();
   5.417 -        List<int[]> savedPos = posStackSupply.elems;
   5.418 -        int[] posStack = newPosStack();
   5.419 +
   5.420          // optimization, was odStack = new Tree[...]; opStack = new Tree[...];
   5.421          int top = 0;
   5.422          odStack[0] = t;
   5.423 -        int startPos = S.pos();
   5.424 -        Token topOp = ERROR;
   5.425 -        int topOpPos = Position.NOPOS;
   5.426 -        while (prec(S.token()) >= minprec) {
   5.427 -            posStack[top] = topOpPos;
   5.428 +        int startPos = token.pos;
   5.429 +        Token topOp = Tokens.DUMMY;
   5.430 +        while (prec(token.kind) >= minprec) {
   5.431              opStack[top] = topOp;
   5.432              top++;
   5.433 -            topOp = S.token();
   5.434 -            topOpPos = S.pos();
   5.435 -            S.nextToken();
   5.436 -            odStack[top] = (topOp == INSTANCEOF) ? parseType() : term3();
   5.437 -            while (top > 0 && prec(topOp) >= prec(S.token())) {
   5.438 -                odStack[top-1] = makeOp(topOpPos, topOp, odStack[top-1],
   5.439 +            topOp = token;
   5.440 +            nextToken();
   5.441 +            odStack[top] = (topOp.kind == INSTANCEOF) ? parseType() : term3();
   5.442 +            while (top > 0 && prec(topOp.kind) >= prec(token.kind)) {
   5.443 +                odStack[top-1] = makeOp(topOp.pos, topOp.kind, odStack[top-1],
   5.444                                          odStack[top]);
   5.445                  top--;
   5.446                  topOp = opStack[top];
   5.447 -                topOpPos = posStack[top];
   5.448              }
   5.449          }
   5.450          Assert.check(top == 0);
   5.451 @@ -761,14 +766,13 @@
   5.452  
   5.453          odStackSupply.elems = savedOd; // optimization
   5.454          opStackSupply.elems = savedOp; // optimization
   5.455 -        posStackSupply.elems = savedPos; // optimization
   5.456          return t;
   5.457      }
   5.458  //where
   5.459          /** Construct a binary or type test node.
   5.460           */
   5.461          private JCExpression makeOp(int pos,
   5.462 -                                    Token topOp,
   5.463 +                                    TokenKind topOp,
   5.464                                      JCExpression od1,
   5.465                                      JCExpression od2)
   5.466          {
   5.467 @@ -817,7 +821,6 @@
   5.468           */
   5.469          ListBuffer<JCExpression[]> odStackSupply = new ListBuffer<JCExpression[]>();
   5.470          ListBuffer<Token[]> opStackSupply = new ListBuffer<Token[]>();
   5.471 -        ListBuffer<int[]> posStackSupply = new ListBuffer<int[]>();
   5.472  
   5.473          private JCExpression[] newOdStack() {
   5.474              if (odStackSupply.elems == odStackSupply.last)
   5.475 @@ -835,14 +838,6 @@
   5.476              return opStack;
   5.477          }
   5.478  
   5.479 -        private int[] newPosStack() {
   5.480 -            if (posStackSupply.elems == posStackSupply.last)
   5.481 -                posStackSupply.append(new int[infixPrecedenceLevels + 1]);
   5.482 -            int[] posStack = posStackSupply.elems.head;
   5.483 -            posStackSupply.elems = posStackSupply.elems.tail;
   5.484 -            return posStack;
   5.485 -        }
   5.486 -
   5.487      /** Expression3    = PrefixOp Expression3
   5.488       *                 | "(" Expr | TypeNoParams ")" Expression3
   5.489       *                 | Primary {Selector} {PostfixOp}
   5.490 @@ -871,10 +866,10 @@
   5.491       *  SuperSuffix    = Arguments | "." Ident [Arguments]
   5.492       */
   5.493      protected JCExpression term3() {
   5.494 -        int pos = S.pos();
   5.495 +        int pos = token.pos;
   5.496          JCExpression t;
   5.497          List<JCExpression> typeArgs = typeArgumentsOpt(EXPR);
   5.498 -        switch (S.token()) {
   5.499 +        switch (token.kind) {
   5.500          case QUES:
   5.501              if ((mode & TYPE) != 0 && (mode & (TYPEARG|NOPARAMS)) == TYPEARG) {
   5.502                  mode = TYPE;
   5.503 @@ -883,49 +878,49 @@
   5.504                  return illegal();
   5.505          case PLUSPLUS: case SUBSUB: case BANG: case TILDE: case PLUS: case SUB:
   5.506              if (typeArgs == null && (mode & EXPR) != 0) {
   5.507 -                Token token = S.token();
   5.508 -                S.nextToken();
   5.509 +                TokenKind tk = token.kind;
   5.510 +                nextToken();
   5.511                  mode = EXPR;
   5.512 -                if (token == SUB &&
   5.513 -                    (S.token() == INTLITERAL || S.token() == LONGLITERAL) &&
   5.514 -                    S.radix() == 10) {
   5.515 +                if (tk == SUB &&
   5.516 +                    (token.kind == INTLITERAL || token.kind == LONGLITERAL) &&
   5.517 +                    token.radix() == 10) {
   5.518                      mode = EXPR;
   5.519                      t = literal(names.hyphen, pos);
   5.520                  } else {
   5.521                      t = term3();
   5.522 -                    return F.at(pos).Unary(unoptag(token), t);
   5.523 +                    return F.at(pos).Unary(unoptag(tk), t);
   5.524                  }
   5.525              } else return illegal();
   5.526              break;
   5.527          case LPAREN:
   5.528              if (typeArgs == null && (mode & EXPR) != 0) {
   5.529 -                S.nextToken();
   5.530 +                nextToken();
   5.531                  mode = EXPR | TYPE | NOPARAMS;
   5.532                  t = term3();
   5.533 -                if ((mode & TYPE) != 0 && S.token() == LT) {
   5.534 +                if ((mode & TYPE) != 0 && token.kind == LT) {
   5.535                      // Could be a cast to a parameterized type
   5.536                      int op = JCTree.LT;
   5.537 -                    int pos1 = S.pos();
   5.538 -                    S.nextToken();
   5.539 +                    int pos1 = token.pos;
   5.540 +                    nextToken();
   5.541                      mode &= (EXPR | TYPE);
   5.542                      mode |= TYPEARG;
   5.543                      JCExpression t1 = term3();
   5.544                      if ((mode & TYPE) != 0 &&
   5.545 -                        (S.token() == COMMA || S.token() == GT)) {
   5.546 +                        (token.kind == COMMA || token.kind == GT)) {
   5.547                          mode = TYPE;
   5.548                          ListBuffer<JCExpression> args = new ListBuffer<JCExpression>();
   5.549                          args.append(t1);
   5.550 -                        while (S.token() == COMMA) {
   5.551 -                            S.nextToken();
   5.552 +                        while (token.kind == COMMA) {
   5.553 +                            nextToken();
   5.554                              args.append(typeArgument());
   5.555                          }
   5.556                          accept(GT);
   5.557                          t = toP(F.at(pos1).TypeApply(t, args.toList()));
   5.558                          checkGenerics();
   5.559 -                        while (S.token() == DOT) {
   5.560 -                            S.nextToken();
   5.561 +                        while (token.kind == DOT) {
   5.562 +                            nextToken();
   5.563                              mode = TYPE;
   5.564 -                            t = toP(F.at(S.pos()).Select(t, ident()));
   5.565 +                            t = toP(F.at(token.pos).Select(t, ident()));
   5.566                              t = typeArgumentsOpt(t);
   5.567                          }
   5.568                          t = bracketsOpt(toP(t));
   5.569 @@ -948,7 +943,7 @@
   5.570                      JCExpression t1 = term3();
   5.571                      return F.at(pos).TypeCast(t, t1);
   5.572                  } else if ((lastmode & TYPE) != 0) {
   5.573 -                    switch (S.token()) {
   5.574 +                    switch (token.kind) {
   5.575                      /*case PLUSPLUS: case SUBSUB: */
   5.576                      case BANG: case TILDE:
   5.577                      case LPAREN: case THIS: case SUPER:
   5.578 @@ -969,7 +964,7 @@
   5.579              if ((mode & EXPR) != 0) {
   5.580                  mode = EXPR;
   5.581                  t = to(F.at(pos).Ident(names._this));
   5.582 -                S.nextToken();
   5.583 +                nextToken();
   5.584                  if (typeArgs == null)
   5.585                      t = argumentsOpt(null, t);
   5.586                  else
   5.587 @@ -997,22 +992,22 @@
   5.588              if (typeArgs != null) return illegal();
   5.589              if ((mode & EXPR) != 0) {
   5.590                  mode = EXPR;
   5.591 -                S.nextToken();
   5.592 -                if (S.token() == LT) typeArgs = typeArguments(false);
   5.593 +                nextToken();
   5.594 +                if (token.kind == LT) typeArgs = typeArguments(false);
   5.595                  t = creator(pos, typeArgs);
   5.596                  typeArgs = null;
   5.597              } else return illegal();
   5.598              break;
   5.599          case IDENTIFIER: case ASSERT: case ENUM:
   5.600              if (typeArgs != null) return illegal();
   5.601 -            t = toP(F.at(S.pos()).Ident(ident()));
   5.602 +            t = toP(F.at(token.pos).Ident(ident()));
   5.603              loop: while (true) {
   5.604 -                pos = S.pos();
   5.605 -                switch (S.token()) {
   5.606 +                pos = token.pos;
   5.607 +                switch (token.kind) {
   5.608                  case LBRACKET:
   5.609 -                    S.nextToken();
   5.610 -                    if (S.token() == RBRACKET) {
   5.611 -                        S.nextToken();
   5.612 +                    nextToken();
   5.613 +                    if (token.kind == RBRACKET) {
   5.614 +                        nextToken();
   5.615                          t = bracketsOpt(t);
   5.616                          t = toP(F.at(pos).TypeArray(t));
   5.617                          t = bracketsSuffix(t);
   5.618 @@ -1033,24 +1028,24 @@
   5.619                      }
   5.620                      break loop;
   5.621                  case DOT:
   5.622 -                    S.nextToken();
   5.623 +                    nextToken();
   5.624                      int oldmode = mode;
   5.625                      mode &= ~NOPARAMS;
   5.626                      typeArgs = typeArgumentsOpt(EXPR);
   5.627                      mode = oldmode;
   5.628                      if ((mode & EXPR) != 0) {
   5.629 -                        switch (S.token()) {
   5.630 +                        switch (token.kind) {
   5.631                          case CLASS:
   5.632                              if (typeArgs != null) return illegal();
   5.633                              mode = EXPR;
   5.634                              t = to(F.at(pos).Select(t, names._class));
   5.635 -                            S.nextToken();
   5.636 +                            nextToken();
   5.637                              break loop;
   5.638                          case THIS:
   5.639                              if (typeArgs != null) return illegal();
   5.640                              mode = EXPR;
   5.641                              t = to(F.at(pos).Select(t, names._this));
   5.642 -                            S.nextToken();
   5.643 +                            nextToken();
   5.644                              break loop;
   5.645                          case SUPER:
   5.646                              mode = EXPR;
   5.647 @@ -1061,9 +1056,9 @@
   5.648                          case NEW:
   5.649                              if (typeArgs != null) return illegal();
   5.650                              mode = EXPR;
   5.651 -                            int pos1 = S.pos();
   5.652 -                            S.nextToken();
   5.653 -                            if (S.token() == LT) typeArgs = typeArguments(false);
   5.654 +                            int pos1 = token.pos;
   5.655 +                            nextToken();
   5.656 +                            if (token.kind == LT) typeArgs = typeArguments(false);
   5.657                              t = innerCreator(pos1, typeArgs, t);
   5.658                              typeArgs = null;
   5.659                              break loop;
   5.660 @@ -1087,8 +1082,8 @@
   5.661          case VOID:
   5.662              if (typeArgs != null) illegal();
   5.663              if ((mode & EXPR) != 0) {
   5.664 -                S.nextToken();
   5.665 -                if (S.token() == DOT) {
   5.666 +                nextToken();
   5.667 +                if (token.kind == DOT) {
   5.668                      JCPrimitiveTypeTree ti = toP(F.at(pos).TypeIdent(TypeTags.VOID));
   5.669                      t = bracketsSuffix(ti);
   5.670                  } else {
   5.671 @@ -1099,7 +1094,7 @@
   5.672                  // a void type (like other primitive types) to the next phase.
   5.673                  // The error will be reported in Attr.attribTypes or Attr.visitApply.
   5.674                  JCPrimitiveTypeTree ti = to(F.at(pos).TypeIdent(TypeTags.VOID));
   5.675 -                S.nextToken();
   5.676 +                nextToken();
   5.677                  return ti;
   5.678                  //return illegal();
   5.679              }
   5.680 @@ -1109,14 +1104,14 @@
   5.681          }
   5.682          if (typeArgs != null) illegal();
   5.683          while (true) {
   5.684 -            int pos1 = S.pos();
   5.685 -            if (S.token() == LBRACKET) {
   5.686 -                S.nextToken();
   5.687 +            int pos1 = token.pos;
   5.688 +            if (token.kind == LBRACKET) {
   5.689 +                nextToken();
   5.690                  if ((mode & TYPE) != 0) {
   5.691                      int oldmode = mode;
   5.692                      mode = TYPE;
   5.693 -                    if (S.token() == RBRACKET) {
   5.694 -                        S.nextToken();
   5.695 +                    if (token.kind == RBRACKET) {
   5.696 +                        nextToken();
   5.697                          t = bracketsOpt(t);
   5.698                          t = toP(F.at(pos1).TypeArray(t));
   5.699                          return t;
   5.700 @@ -1129,21 +1124,21 @@
   5.701                      t = to(F.at(pos1).Indexed(t, t1));
   5.702                  }
   5.703                  accept(RBRACKET);
   5.704 -            } else if (S.token() == DOT) {
   5.705 -                S.nextToken();
   5.706 +            } else if (token.kind == DOT) {
   5.707 +                nextToken();
   5.708                  typeArgs = typeArgumentsOpt(EXPR);
   5.709 -                if (S.token() == SUPER && (mode & EXPR) != 0) {
   5.710 +                if (token.kind == SUPER && (mode & EXPR) != 0) {
   5.711                      mode = EXPR;
   5.712                      t = to(F.at(pos1).Select(t, names._super));
   5.713 -                    S.nextToken();
   5.714 +                    nextToken();
   5.715                      t = arguments(typeArgs, t);
   5.716                      typeArgs = null;
   5.717 -                } else if (S.token() == NEW && (mode & EXPR) != 0) {
   5.718 +                } else if (token.kind == NEW && (mode & EXPR) != 0) {
   5.719                      if (typeArgs != null) return illegal();
   5.720                      mode = EXPR;
   5.721 -                    int pos2 = S.pos();
   5.722 -                    S.nextToken();
   5.723 -                    if (S.token() == LT) typeArgs = typeArguments(false);
   5.724 +                    int pos2 = token.pos;
   5.725 +                    nextToken();
   5.726 +                    if (token.kind == LT) typeArgs = typeArguments(false);
   5.727                      t = innerCreator(pos2, typeArgs, t);
   5.728                      typeArgs = null;
   5.729                  } else {
   5.730 @@ -1155,11 +1150,11 @@
   5.731                  break;
   5.732              }
   5.733          }
   5.734 -        while ((S.token() == PLUSPLUS || S.token() == SUBSUB) && (mode & EXPR) != 0) {
   5.735 +        while ((token.kind == PLUSPLUS || token.kind == SUBSUB) && (mode & EXPR) != 0) {
   5.736              mode = EXPR;
   5.737 -            t = to(F.at(S.pos()).Unary(
   5.738 -                  S.token() == PLUSPLUS ? JCTree.POSTINC : JCTree.POSTDEC, t));
   5.739 -            S.nextToken();
   5.740 +            t = to(F.at(token.pos).Unary(
   5.741 +                  token.kind == PLUSPLUS ? JCTree.POSTINC : JCTree.POSTDEC, t));
   5.742 +            nextToken();
   5.743          }
   5.744          return toP(t);
   5.745      }
   5.746 @@ -1167,13 +1162,13 @@
   5.747      /** SuperSuffix = Arguments | "." [TypeArguments] Ident [Arguments]
   5.748       */
   5.749      JCExpression superSuffix(List<JCExpression> typeArgs, JCExpression t) {
   5.750 -        S.nextToken();
   5.751 -        if (S.token() == LPAREN || typeArgs != null) {
   5.752 +        nextToken();
   5.753 +        if (token.kind == LPAREN || typeArgs != null) {
   5.754              t = arguments(typeArgs, t);
   5.755          } else {
   5.756 -            int pos = S.pos();
   5.757 +            int pos = token.pos;
   5.758              accept(DOT);
   5.759 -            typeArgs = (S.token() == LT) ? typeArguments(false) : null;
   5.760 +            typeArgs = (token.kind == LT) ? typeArguments(false) : null;
   5.761              t = toP(F.at(pos).Select(t, ident()));
   5.762              t = argumentsOpt(typeArgs, t);
   5.763          }
   5.764 @@ -1183,15 +1178,15 @@
   5.765      /** BasicType = BYTE | SHORT | CHAR | INT | LONG | FLOAT | DOUBLE | BOOLEAN
   5.766       */
   5.767      JCPrimitiveTypeTree basicType() {
   5.768 -        JCPrimitiveTypeTree t = to(F.at(S.pos()).TypeIdent(typetag(S.token())));
   5.769 -        S.nextToken();
   5.770 +        JCPrimitiveTypeTree t = to(F.at(token.pos).TypeIdent(typetag(token.kind)));
   5.771 +        nextToken();
   5.772          return t;
   5.773      }
   5.774  
   5.775      /** ArgumentsOpt = [ Arguments ]
   5.776       */
   5.777      JCExpression argumentsOpt(List<JCExpression> typeArgs, JCExpression t) {
   5.778 -        if ((mode & EXPR) != 0 && S.token() == LPAREN || typeArgs != null) {
   5.779 +        if ((mode & EXPR) != 0 && token.kind == LPAREN || typeArgs != null) {
   5.780              mode = EXPR;
   5.781              return arguments(typeArgs, t);
   5.782          } else {
   5.783 @@ -1203,24 +1198,24 @@
   5.784       */
   5.785      List<JCExpression> arguments() {
   5.786          ListBuffer<JCExpression> args = lb();
   5.787 -        if (S.token() == LPAREN) {
   5.788 -            S.nextToken();
   5.789 -            if (S.token() != RPAREN) {
   5.790 +        if (token.kind == LPAREN) {
   5.791 +            nextToken();
   5.792 +            if (token.kind != RPAREN) {
   5.793                  args.append(parseExpression());
   5.794 -                while (S.token() == COMMA) {
   5.795 -                    S.nextToken();
   5.796 +                while (token.kind == COMMA) {
   5.797 +                    nextToken();
   5.798                      args.append(parseExpression());
   5.799                  }
   5.800              }
   5.801              accept(RPAREN);
   5.802          } else {
   5.803 -            syntaxError(S.pos(), "expected", LPAREN);
   5.804 +            syntaxError(token.pos, "expected", LPAREN);
   5.805          }
   5.806          return args.toList();
   5.807      }
   5.808  
   5.809      JCMethodInvocation arguments(List<JCExpression> typeArgs, JCExpression t) {
   5.810 -        int pos = S.pos();
   5.811 +        int pos = token.pos;
   5.812          List<JCExpression> args = arguments();
   5.813          return toP(F.at(pos).Apply(typeArgs, t, args));
   5.814      }
   5.815 @@ -1228,7 +1223,7 @@
   5.816      /**  TypeArgumentsOpt = [ TypeArguments ]
   5.817       */
   5.818      JCExpression typeArgumentsOpt(JCExpression t) {
   5.819 -        if (S.token() == LT &&
   5.820 +        if (token.kind == LT &&
   5.821              (mode & TYPE) != 0 &&
   5.822              (mode & NOPARAMS) == 0) {
   5.823              mode = TYPE;
   5.824 @@ -1243,7 +1238,7 @@
   5.825      }
   5.826  
   5.827      List<JCExpression> typeArgumentsOpt(int useMode) {
   5.828 -        if (S.token() == LT) {
   5.829 +        if (token.kind == LT) {
   5.830              checkGenerics();
   5.831              if ((mode & useMode) == 0 ||
   5.832                  (mode & NOPARAMS) != 0) {
   5.833 @@ -1258,47 +1253,37 @@
   5.834      /**  TypeArguments  = "<" TypeArgument {"," TypeArgument} ">"
   5.835       */
   5.836      List<JCExpression> typeArguments(boolean diamondAllowed) {
   5.837 -        if (S.token() == LT) {
   5.838 -            S.nextToken();
   5.839 -            if (S.token() == GT && diamondAllowed) {
   5.840 +        if (token.kind == LT) {
   5.841 +            nextToken();
   5.842 +            if (token.kind == GT && diamondAllowed) {
   5.843                  checkDiamond();
   5.844                  mode |= DIAMOND;
   5.845 -                S.nextToken();
   5.846 +                nextToken();
   5.847                  return List.nil();
   5.848              } else {
   5.849                  ListBuffer<JCExpression> args = ListBuffer.lb();
   5.850                  args.append(((mode & EXPR) == 0) ? typeArgument() : parseType());
   5.851 -                while (S.token() == COMMA) {
   5.852 -                    S.nextToken();
   5.853 +                while (token.kind == COMMA) {
   5.854 +                    nextToken();
   5.855                      args.append(((mode & EXPR) == 0) ? typeArgument() : parseType());
   5.856                  }
   5.857 -                switch (S.token()) {
   5.858 -                case GTGTGTEQ:
   5.859 -                    S.token(GTGTEQ);
   5.860 -                    break;
   5.861 -                case GTGTEQ:
   5.862 -                    S.token(GTEQ);
   5.863 -                    break;
   5.864 -                case GTEQ:
   5.865 -                    S.token(EQ);
   5.866 -                    break;
   5.867 -                case GTGTGT:
   5.868 -                    S.token(GTGT);
   5.869 -                    break;
   5.870 -                case GTGT:
   5.871 -                    S.token(GT);
   5.872 +                switch (token.kind) {
   5.873 +
   5.874 +                case GTGTGTEQ: case GTGTEQ: case GTEQ:
   5.875 +                case GTGTGT: case GTGT:
   5.876 +                    token = S.split();
   5.877                      break;
   5.878                  case GT:
   5.879 -                    S.nextToken();
   5.880 +                    nextToken();
   5.881                      break;
   5.882                  default:
   5.883 -                    args.append(syntaxError(S.pos(), "expected", GT));
   5.884 +                    args.append(syntaxError(token.pos, "expected", GT));
   5.885                      break;
   5.886                  }
   5.887                  return args.toList();
   5.888              }
   5.889          } else {
   5.890 -            return List.<JCExpression>of(syntaxError(S.pos(), "expected", LT));
   5.891 +            return List.<JCExpression>of(syntaxError(token.pos, "expected", LT));
   5.892          }
   5.893      }
   5.894  
   5.895 @@ -1308,24 +1293,24 @@
   5.896       *               | "?" SUPER Type
   5.897       */
   5.898      JCExpression typeArgument() {
   5.899 -        if (S.token() != QUES) return parseType();
   5.900 -        int pos = S.pos();
   5.901 -        S.nextToken();
   5.902 -        if (S.token() == EXTENDS) {
   5.903 +        if (token.kind != QUES) return parseType();
   5.904 +        int pos = token.pos;
   5.905 +        nextToken();
   5.906 +        if (token.kind == EXTENDS) {
   5.907              TypeBoundKind t = to(F.at(pos).TypeBoundKind(BoundKind.EXTENDS));
   5.908 -            S.nextToken();
   5.909 +            nextToken();
   5.910              JCExpression bound = parseType();
   5.911              return F.at(pos).Wildcard(t, bound);
   5.912 -        } else if (S.token() == SUPER) {
   5.913 +        } else if (token.kind == SUPER) {
   5.914              TypeBoundKind t = to(F.at(pos).TypeBoundKind(BoundKind.SUPER));
   5.915 -            S.nextToken();
   5.916 +            nextToken();
   5.917              JCExpression bound = parseType();
   5.918              return F.at(pos).Wildcard(t, bound);
   5.919 -        } else if (S.token() == IDENTIFIER) {
   5.920 +        } else if (token.kind == IDENTIFIER) {
   5.921              //error recovery
   5.922              TypeBoundKind t = F.at(Position.NOPOS).TypeBoundKind(BoundKind.UNBOUND);
   5.923              JCExpression wc = toP(F.at(pos).Wildcard(t, null));
   5.924 -            JCIdent id = toP(F.at(S.pos()).Ident(ident()));
   5.925 +            JCIdent id = toP(F.at(token.pos).Ident(ident()));
   5.926              JCErroneous err = F.at(pos).Erroneous(List.<JCTree>of(wc, id));
   5.927              reportSyntaxError(err, "expected3", GT, EXTENDS, SUPER);
   5.928              return err;
   5.929 @@ -1336,7 +1321,7 @@
   5.930      }
   5.931  
   5.932      JCTypeApply typeArguments(JCExpression t, boolean diamondAllowed) {
   5.933 -        int pos = S.pos();
   5.934 +        int pos = token.pos;
   5.935          List<JCExpression> args = typeArguments(diamondAllowed);
   5.936          return toP(F.at(pos).TypeApply(t, args));
   5.937      }
   5.938 @@ -1344,9 +1329,9 @@
   5.939      /** BracketsOpt = {"[" "]"}
   5.940       */
   5.941      private JCExpression bracketsOpt(JCExpression t) {
   5.942 -        if (S.token() == LBRACKET) {
   5.943 -            int pos = S.pos();
   5.944 -            S.nextToken();
   5.945 +        if (token.kind == LBRACKET) {
   5.946 +            int pos = token.pos;
   5.947 +            nextToken();
   5.948              t = bracketsOptCont(t, pos);
   5.949              F.at(pos);
   5.950          }
   5.951 @@ -1363,17 +1348,17 @@
   5.952       *  BracketsSuffixType =
   5.953       */
   5.954      JCExpression bracketsSuffix(JCExpression t) {
   5.955 -        if ((mode & EXPR) != 0 && S.token() == DOT) {
   5.956 +        if ((mode & EXPR) != 0 && token.kind == DOT) {
   5.957              mode = EXPR;
   5.958 -            int pos = S.pos();
   5.959 -            S.nextToken();
   5.960 +            int pos = token.pos;
   5.961 +            nextToken();
   5.962              accept(CLASS);
   5.963 -            if (S.pos() == errorEndPos) {
   5.964 +            if (token.pos == errorEndPos) {
   5.965                  // error recovery
   5.966                  Name name = null;
   5.967 -                if (S.token() == IDENTIFIER) {
   5.968 -                    name = S.name();
   5.969 -                    S.nextToken();
   5.970 +                if (token.kind == IDENTIFIER) {
   5.971 +                    name = token.name();
   5.972 +                    nextToken();
   5.973                  } else {
   5.974                      name = names.error;
   5.975                  }
   5.976 @@ -1384,7 +1369,7 @@
   5.977          } else if ((mode & TYPE) != 0) {
   5.978              mode = TYPE;
   5.979          } else {
   5.980 -            syntaxError(S.pos(), "dot.class.expected");
   5.981 +            syntaxError(token.pos, "dot.class.expected");
   5.982          }
   5.983          return t;
   5.984      }
   5.985 @@ -1392,7 +1377,7 @@
   5.986      /** Creator = Qualident [TypeArguments] ( ArrayCreatorRest | ClassCreatorRest )
   5.987       */
   5.988      JCExpression creator(int newpos, List<JCExpression> typeArgs) {
   5.989 -        switch (S.token()) {
   5.990 +        switch (token.kind) {
   5.991          case BYTE: case SHORT: case CHAR: case INT: case LONG: case FLOAT:
   5.992          case DOUBLE: case BOOLEAN:
   5.993              if (typeArgs == null)
   5.994 @@ -1405,29 +1390,29 @@
   5.995          mode = TYPE;
   5.996          boolean diamondFound = false;
   5.997          int lastTypeargsPos = -1;
   5.998 -        if (S.token() == LT) {
   5.999 +        if (token.kind == LT) {
  5.1000              checkGenerics();
  5.1001 -            lastTypeargsPos = S.pos();
  5.1002 +            lastTypeargsPos = token.pos;
  5.1003              t = typeArguments(t, true);
  5.1004              diamondFound = (mode & DIAMOND) != 0;
  5.1005          }
  5.1006 -        while (S.token() == DOT) {
  5.1007 +        while (token.kind == DOT) {
  5.1008              if (diamondFound) {
  5.1009                  //cannot select after a diamond
  5.1010                  illegal();
  5.1011              }
  5.1012 -            int pos = S.pos();
  5.1013 -            S.nextToken();
  5.1014 +            int pos = token.pos;
  5.1015 +            nextToken();
  5.1016              t = toP(F.at(pos).Select(t, ident()));
  5.1017 -            if (S.token() == LT) {
  5.1018 -                lastTypeargsPos = S.pos();
  5.1019 +            if (token.kind == LT) {
  5.1020 +                lastTypeargsPos = token.pos;
  5.1021                  checkGenerics();
  5.1022                  t = typeArguments(t, true);
  5.1023                  diamondFound = (mode & DIAMOND) != 0;
  5.1024              }
  5.1025          }
  5.1026          mode = oldmode;
  5.1027 -        if (S.token() == LBRACKET) {
  5.1028 +        if (token.kind == LBRACKET) {
  5.1029              JCExpression e = arrayCreatorRest(newpos, t);
  5.1030              if (diamondFound) {
  5.1031                  reportSyntaxError(lastTypeargsPos, "cannot.create.array.with.diamond");
  5.1032 @@ -1441,17 +1426,17 @@
  5.1033                      // modified to improve error recovery.
  5.1034                      pos = typeArgs.head.pos;
  5.1035                  }
  5.1036 -                setErrorEndPos(S.prevEndPos());
  5.1037 +                setErrorEndPos(S.prevToken().endPos);
  5.1038                  JCErroneous err = F.at(pos).Erroneous(typeArgs.prepend(e));
  5.1039                  reportSyntaxError(err, "cannot.create.array.with.type.arguments");
  5.1040                  return toP(err);
  5.1041              }
  5.1042              return e;
  5.1043 -        } else if (S.token() == LPAREN) {
  5.1044 +        } else if (token.kind == LPAREN) {
  5.1045              return classCreatorRest(newpos, null, typeArgs, t);
  5.1046          } else {
  5.1047 -            setErrorEndPos(S.pos());
  5.1048 -            reportSyntaxError(S.pos(), "expected2", LPAREN, LBRACKET);
  5.1049 +            setErrorEndPos(token.pos);
  5.1050 +            reportSyntaxError(token.pos, "expected2", LPAREN, LBRACKET);
  5.1051              t = toP(F.at(newpos).NewClass(null, typeArgs, t, List.<JCExpression>nil(), null));
  5.1052              return toP(F.at(newpos).Erroneous(List.<JCTree>of(t)));
  5.1053          }
  5.1054 @@ -1460,8 +1445,8 @@
  5.1055      /** InnerCreator = Ident [TypeArguments] ClassCreatorRest
  5.1056       */
  5.1057      JCExpression innerCreator(int newpos, List<JCExpression> typeArgs, JCExpression encl) {
  5.1058 -        JCExpression t = toP(F.at(S.pos()).Ident(ident()));
  5.1059 -        if (S.token() == LT) {
  5.1060 +        JCExpression t = toP(F.at(token.pos).Ident(ident()));
  5.1061 +        if (token.kind == LT) {
  5.1062              int oldmode = mode;
  5.1063              checkGenerics();
  5.1064              t = typeArguments(t, true);
  5.1065 @@ -1475,23 +1460,23 @@
  5.1066       */
  5.1067      JCExpression arrayCreatorRest(int newpos, JCExpression elemtype) {
  5.1068          accept(LBRACKET);
  5.1069 -        if (S.token() == RBRACKET) {
  5.1070 +        if (token.kind == RBRACKET) {
  5.1071              accept(RBRACKET);
  5.1072              elemtype = bracketsOpt(elemtype);
  5.1073 -            if (S.token() == LBRACE) {
  5.1074 +            if (token.kind == LBRACE) {
  5.1075                  return arrayInitializer(newpos, elemtype);
  5.1076              } else {
  5.1077                  JCExpression t = toP(F.at(newpos).NewArray(elemtype, List.<JCExpression>nil(), null));
  5.1078 -                return syntaxError(S.pos(), List.<JCTree>of(t), "array.dimension.missing");
  5.1079 +                return syntaxError(token.pos, List.<JCTree>of(t), "array.dimension.missing");
  5.1080              }
  5.1081          } else {
  5.1082              ListBuffer<JCExpression> dims = new ListBuffer<JCExpression>();
  5.1083              dims.append(parseExpression());
  5.1084              accept(RBRACKET);
  5.1085 -            while (S.token() == LBRACKET) {
  5.1086 -                int pos = S.pos();
  5.1087 -                S.nextToken();
  5.1088 -                if (S.token() == RBRACKET) {
  5.1089 +            while (token.kind == LBRACKET) {
  5.1090 +                int pos = token.pos;
  5.1091 +                nextToken();
  5.1092 +                if (token.kind == RBRACKET) {
  5.1093                      elemtype = bracketsOptCont(elemtype, pos);
  5.1094                  } else {
  5.1095                      dims.append(parseExpression());
  5.1096 @@ -1511,8 +1496,8 @@
  5.1097      {
  5.1098          List<JCExpression> args = arguments();
  5.1099          JCClassDecl body = null;
  5.1100 -        if (S.token() == LBRACE) {
  5.1101 -            int pos = S.pos();
  5.1102 +        if (token.kind == LBRACE) {
  5.1103 +            int pos = token.pos;
  5.1104              List<JCTree> defs = classOrInterfaceBody(names.empty, false);
  5.1105              JCModifiers mods = F.at(Position.NOPOS).Modifiers(0);
  5.1106              body = toP(F.at(pos).AnonymousClassDef(mods, defs));
  5.1107 @@ -1525,13 +1510,13 @@
  5.1108      JCExpression arrayInitializer(int newpos, JCExpression t) {
  5.1109          accept(LBRACE);
  5.1110          ListBuffer<JCExpression> elems = new ListBuffer<JCExpression>();
  5.1111 -        if (S.token() == COMMA) {
  5.1112 -            S.nextToken();
  5.1113 -        } else if (S.token() != RBRACE) {
  5.1114 +        if (token.kind == COMMA) {
  5.1115 +            nextToken();
  5.1116 +        } else if (token.kind != RBRACE) {
  5.1117              elems.append(variableInitializer());
  5.1118 -            while (S.token() == COMMA) {
  5.1119 -                S.nextToken();
  5.1120 -                if (S.token() == RBRACE) break;
  5.1121 +            while (token.kind == COMMA) {
  5.1122 +                nextToken();
  5.1123 +                if (token.kind == RBRACE) break;
  5.1124                  elems.append(variableInitializer());
  5.1125              }
  5.1126          }
  5.1127 @@ -1542,7 +1527,7 @@
  5.1128      /** VariableInitializer = ArrayInitializer | Expression
  5.1129       */
  5.1130      public JCExpression variableInitializer() {
  5.1131 -        return S.token() == LBRACE ? arrayInitializer(S.pos(), null) : parseExpression();
  5.1132 +        return token.kind == LBRACE ? arrayInitializer(token.pos, null) : parseExpression();
  5.1133      }
  5.1134  
  5.1135      /** ParExpression = "(" Expression ")"
  5.1136 @@ -1560,19 +1545,19 @@
  5.1137          accept(LBRACE);
  5.1138          List<JCStatement> stats = blockStatements();
  5.1139          JCBlock t = F.at(pos).Block(flags, stats);
  5.1140 -        while (S.token() == CASE || S.token() == DEFAULT) {
  5.1141 -            syntaxError("orphaned", S.token());
  5.1142 +        while (token.kind == CASE || token.kind == DEFAULT) {
  5.1143 +            syntaxError("orphaned", token.kind);
  5.1144              switchBlockStatementGroups();
  5.1145          }
  5.1146          // the Block node has a field "endpos" for first char of last token, which is
  5.1147          // usually but not necessarily the last char of the last token.
  5.1148 -        t.endpos = S.pos();
  5.1149 +        t.endpos = token.pos;
  5.1150          accept(RBRACE);
  5.1151          return toP(t);
  5.1152      }
  5.1153  
  5.1154      public JCBlock block() {
  5.1155 -        return block(S.pos(), 0);
  5.1156 +        return block(token.pos, 0);
  5.1157      }
  5.1158  
  5.1159      /** BlockStatements = { BlockStatement }
  5.1160 @@ -1588,8 +1573,8 @@
  5.1161          int lastErrPos = -1;
  5.1162          ListBuffer<JCStatement> stats = new ListBuffer<JCStatement>();
  5.1163          while (true) {
  5.1164 -            int pos = S.pos();
  5.1165 -            switch (S.token()) {
  5.1166 +            int pos = token.pos;
  5.1167 +            switch (token.kind) {
  5.1168              case RBRACE: case CASE: case DEFAULT: case EOF:
  5.1169                  return stats.toList();
  5.1170              case LBRACE: case IF: case FOR: case WHILE: case DO: case TRY:
  5.1171 @@ -1599,64 +1584,63 @@
  5.1172                  break;
  5.1173              case MONKEYS_AT:
  5.1174              case FINAL: {
  5.1175 -                String dc = S.docComment();
  5.1176 +                String dc = token.docComment;
  5.1177                  JCModifiers mods = modifiersOpt();
  5.1178 -                if (S.token() == INTERFACE ||
  5.1179 -                    S.token() == CLASS ||
  5.1180 -                    allowEnums && S.token() == ENUM) {
  5.1181 +                if (token.kind == INTERFACE ||
  5.1182 +                    token.kind == CLASS ||
  5.1183 +                    allowEnums && token.kind == ENUM) {
  5.1184                      stats.append(classOrInterfaceOrEnumDeclaration(mods, dc));
  5.1185                  } else {
  5.1186                      JCExpression t = parseType();
  5.1187                      stats.appendList(variableDeclarators(mods, t,
  5.1188                                                           new ListBuffer<JCStatement>()));
  5.1189                      // A "LocalVariableDeclarationStatement" subsumes the terminating semicolon
  5.1190 -                    storeEnd(stats.elems.last(), S.endPos());
  5.1191 +                    storeEnd(stats.elems.last(), token.endPos);
  5.1192                      accept(SEMI);
  5.1193                  }
  5.1194                  break;
  5.1195              }
  5.1196              case ABSTRACT: case STRICTFP: {
  5.1197 -                String dc = S.docComment();
  5.1198 +                String dc = token.docComment;
  5.1199                  JCModifiers mods = modifiersOpt();
  5.1200                  stats.append(classOrInterfaceOrEnumDeclaration(mods, dc));
  5.1201                  break;
  5.1202              }
  5.1203              case INTERFACE:
  5.1204              case CLASS:
  5.1205 -                stats.append(classOrInterfaceOrEnumDeclaration(modifiersOpt(),
  5.1206 -                                                               S.docComment()));
  5.1207 +                String dc = token.docComment;
  5.1208 +                stats.append(classOrInterfaceOrEnumDeclaration(modifiersOpt(), dc));
  5.1209                  break;
  5.1210              case ENUM:
  5.1211              case ASSERT:
  5.1212 -                if (allowEnums && S.token() == ENUM) {
  5.1213 -                    error(S.pos(), "local.enum");
  5.1214 -                    stats.
  5.1215 -                        append(classOrInterfaceOrEnumDeclaration(modifiersOpt(),
  5.1216 -                                                                 S.docComment()));
  5.1217 +                if (allowEnums && token.kind == ENUM) {
  5.1218 +                    error(token.pos, "local.enum");
  5.1219 +                    dc = token.docComment;
  5.1220 +                    stats.append(classOrInterfaceOrEnumDeclaration(modifiersOpt(), dc));
  5.1221                      break;
  5.1222 -                } else if (allowAsserts && S.token() == ASSERT) {
  5.1223 +                } else if (allowAsserts && token.kind == ASSERT) {
  5.1224                      stats.append(parseStatement());
  5.1225                      break;
  5.1226                  }
  5.1227                  /* fall through to default */
  5.1228              default:
  5.1229 -                Name name = S.name();
  5.1230 +                Token prevToken = token;
  5.1231                  JCExpression t = term(EXPR | TYPE);
  5.1232 -                if (S.token() == COLON && t.getTag() == JCTree.IDENT) {
  5.1233 -                    S.nextToken();
  5.1234 +                if (token.kind == COLON && t.getTag() == JCTree.IDENT) {
  5.1235 +                    nextToken();
  5.1236                      JCStatement stat = parseStatement();
  5.1237 -                    stats.append(F.at(pos).Labelled(name, stat));
  5.1238 +                    stats.append(F.at(pos).Labelled(prevToken.name(), stat));
  5.1239                  } else if ((lastmode & TYPE) != 0 &&
  5.1240 -                           (S.token() == IDENTIFIER ||
  5.1241 -                            S.token() == ASSERT ||
  5.1242 -                            S.token() == ENUM)) {
  5.1243 -                    pos = S.pos();
  5.1244 +                           (token.kind == IDENTIFIER ||
  5.1245 +                            token.kind == ASSERT ||
  5.1246 +                            token.kind == ENUM)) {
  5.1247 +                    pos = token.pos;
  5.1248                      JCModifiers mods = F.at(Position.NOPOS).Modifiers(0);
  5.1249                      F.at(pos);
  5.1250                      stats.appendList(variableDeclarators(mods, t,
  5.1251                                                           new ListBuffer<JCStatement>()));
  5.1252                      // A "LocalVariableDeclarationStatement" subsumes the terminating semicolon
  5.1253 -                    storeEnd(stats.elems.last(), S.endPos());
  5.1254 +                    storeEnd(stats.elems.last(), token.endPos);
  5.1255                      accept(SEMI);
  5.1256                  } else {
  5.1257                      // This Exec is an "ExpressionStatement"; it subsumes the terminating semicolon
  5.1258 @@ -1666,15 +1650,12 @@
  5.1259              }
  5.1260  
  5.1261              // error recovery
  5.1262 -            if (S.pos() == lastErrPos)
  5.1263 +            if (token.pos == lastErrPos)
  5.1264                  return stats.toList();
  5.1265 -            if (S.pos() <= errorEndPos) {
  5.1266 +            if (token.pos <= errorEndPos) {
  5.1267                  skip(false, true, true, true);
  5.1268 -                lastErrPos = S.pos();
  5.1269 +                lastErrPos = token.pos;
  5.1270              }
  5.1271 -
  5.1272 -            // ensure no dangling /** @deprecated */ active
  5.1273 -            S.resetDeprecatedFlag();
  5.1274          }
  5.1275      }
  5.1276  
  5.1277 @@ -1700,29 +1681,29 @@
  5.1278       */
  5.1279      @SuppressWarnings("fallthrough")
  5.1280      public JCStatement parseStatement() {
  5.1281 -        int pos = S.pos();
  5.1282 -        switch (S.token()) {
  5.1283 +        int pos = token.pos;
  5.1284 +        switch (token.kind) {
  5.1285          case LBRACE:
  5.1286              return block();
  5.1287          case IF: {
  5.1288 -            S.nextToken();
  5.1289 +            nextToken();
  5.1290              JCExpression cond = parExpression();
  5.1291              JCStatement thenpart = parseStatement();
  5.1292              JCStatement elsepart = null;
  5.1293 -            if (S.token() == ELSE) {
  5.1294 -                S.nextToken();
  5.1295 +            if (token.kind == ELSE) {
  5.1296 +                nextToken();
  5.1297                  elsepart = parseStatement();
  5.1298              }
  5.1299              return F.at(pos).If(cond, thenpart, elsepart);
  5.1300          }
  5.1301          case FOR: {
  5.1302 -            S.nextToken();
  5.1303 +            nextToken();
  5.1304              accept(LPAREN);
  5.1305 -            List<JCStatement> inits = S.token() == SEMI ? List.<JCStatement>nil() : forInit();
  5.1306 +            List<JCStatement> inits = token.kind == SEMI ? List.<JCStatement>nil() : forInit();
  5.1307              if (inits.length() == 1 &&
  5.1308                  inits.head.getTag() == JCTree.VARDEF &&
  5.1309                  ((JCVariableDecl) inits.head).init == null &&
  5.1310 -                S.token() == COLON) {
  5.1311 +                token.kind == COLON) {
  5.1312                  checkForeach();
  5.1313                  JCVariableDecl var = (JCVariableDecl)inits.head;
  5.1314                  accept(COLON);
  5.1315 @@ -1732,22 +1713,22 @@
  5.1316                  return F.at(pos).ForeachLoop(var, expr, body);
  5.1317              } else {
  5.1318                  accept(SEMI);
  5.1319 -                JCExpression cond = S.token() == SEMI ? null : parseExpression();
  5.1320 +                JCExpression cond = token.kind == SEMI ? null : parseExpression();
  5.1321                  accept(SEMI);
  5.1322 -                List<JCExpressionStatement> steps = S.token() == RPAREN ? List.<JCExpressionStatement>nil() : forUpdate();
  5.1323 +                List<JCExpressionStatement> steps = token.kind == RPAREN ? List.<JCExpressionStatement>nil() : forUpdate();
  5.1324                  accept(RPAREN);
  5.1325                  JCStatement body = parseStatement();
  5.1326                  return F.at(pos).ForLoop(inits, cond, steps, body);
  5.1327              }
  5.1328          }
  5.1329          case WHILE: {
  5.1330 -            S.nextToken();
  5.1331 +            nextToken();
  5.1332              JCExpression cond = parExpression();
  5.1333              JCStatement body = parseStatement();
  5.1334              return F.at(pos).WhileLoop(cond, body);
  5.1335          }
  5.1336          case DO: {
  5.1337 -            S.nextToken();
  5.1338 +            nextToken();
  5.1339              JCStatement body = parseStatement();
  5.1340              accept(WHILE);
  5.1341              JCExpression cond = parExpression();
  5.1342 @@ -1756,21 +1737,21 @@
  5.1343              return t;
  5.1344          }
  5.1345          case TRY: {
  5.1346 -            S.nextToken();
  5.1347 +            nextToken();
  5.1348              List<JCTree> resources = List.<JCTree>nil();
  5.1349 -            if (S.token() == LPAREN) {
  5.1350 +            if (token.kind == LPAREN) {
  5.1351                  checkTryWithResources();
  5.1352 -                S.nextToken();
  5.1353 +                nextToken();
  5.1354                  resources = resources();
  5.1355                  accept(RPAREN);
  5.1356              }
  5.1357              JCBlock body = block();
  5.1358              ListBuffer<JCCatch> catchers = new ListBuffer<JCCatch>();
  5.1359              JCBlock finalizer = null;
  5.1360 -            if (S.token() == CATCH || S.token() == FINALLY) {
  5.1361 -                while (S.token() == CATCH) catchers.append(catchClause());
  5.1362 -                if (S.token() == FINALLY) {
  5.1363 -                    S.nextToken();
  5.1364 +            if (token.kind == CATCH || token.kind == FINALLY) {
  5.1365 +                while (token.kind == CATCH) catchers.append(catchClause());
  5.1366 +                if (token.kind == FINALLY) {
  5.1367 +                    nextToken();
  5.1368                      finalizer = block();
  5.1369                  }
  5.1370              } else {
  5.1371 @@ -1783,7 +1764,7 @@
  5.1372              return F.at(pos).Try(resources, body, catchers.toList(), finalizer);
  5.1373          }
  5.1374          case SWITCH: {
  5.1375 -            S.nextToken();
  5.1376 +            nextToken();
  5.1377              JCExpression selector = parExpression();
  5.1378              accept(LBRACE);
  5.1379              List<JCCase> cases = switchBlockStatementGroups();
  5.1380 @@ -1792,41 +1773,41 @@
  5.1381              return t;
  5.1382          }
  5.1383          case SYNCHRONIZED: {
  5.1384 -            S.nextToken();
  5.1385 +            nextToken();
  5.1386              JCExpression lock = parExpression();
  5.1387              JCBlock body = block();
  5.1388              return F.at(pos).Synchronized(lock, body);
  5.1389          }
  5.1390          case RETURN: {
  5.1391 -            S.nextToken();
  5.1392 -            JCExpression result = S.token() == SEMI ? null : parseExpression();
  5.1393 +            nextToken();
  5.1394 +            JCExpression result = token.kind == SEMI ? null : parseExpression();
  5.1395              JCReturn t = to(F.at(pos).Return(result));
  5.1396              accept(SEMI);
  5.1397              return t;
  5.1398          }
  5.1399          case THROW: {
  5.1400 -            S.nextToken();
  5.1401 +            nextToken();
  5.1402              JCExpression exc = parseExpression();
  5.1403              JCThrow t = to(F.at(pos).Throw(exc));
  5.1404              accept(SEMI);
  5.1405              return t;
  5.1406          }
  5.1407          case BREAK: {
  5.1408 -            S.nextToken();
  5.1409 -            Name label = (S.token() == IDENTIFIER || S.token() == ASSERT || S.token() == ENUM) ? ident() : null;
  5.1410 +            nextToken();
  5.1411 +            Name label = (token.kind == IDENTIFIER || token.kind == ASSERT || token.kind == ENUM) ? ident() : null;
  5.1412              JCBreak t = to(F.at(pos).Break(label));
  5.1413              accept(SEMI);
  5.1414              return t;
  5.1415          }
  5.1416          case CONTINUE: {
  5.1417 -            S.nextToken();
  5.1418 -            Name label = (S.token() == IDENTIFIER || S.token() == ASSERT || S.token() == ENUM) ? ident() : null;
  5.1419 +            nextToken();
  5.1420 +            Name label = (token.kind == IDENTIFIER || token.kind == ASSERT || token.kind == ENUM) ? ident() : null;
  5.1421              JCContinue t =  to(F.at(pos).Continue(label));
  5.1422              accept(SEMI);
  5.1423              return t;
  5.1424          }
  5.1425          case SEMI:
  5.1426 -            S.nextToken();
  5.1427 +            nextToken();
  5.1428              return toP(F.at(pos).Skip());
  5.1429          case ELSE:
  5.1430              return toP(F.Exec(syntaxError("else.without.if")));
  5.1431 @@ -1835,12 +1816,12 @@
  5.1432          case CATCH:
  5.1433              return toP(F.Exec(syntaxError("catch.without.try")));
  5.1434          case ASSERT: {
  5.1435 -            if (allowAsserts && S.token() == ASSERT) {
  5.1436 -                S.nextToken();
  5.1437 +            if (allowAsserts && token.kind == ASSERT) {
  5.1438 +                nextToken();
  5.1439                  JCExpression assertion = parseExpression();
  5.1440                  JCExpression message = null;
  5.1441 -                if (S.token() == COLON) {
  5.1442 -                    S.nextToken();
  5.1443 +                if (token.kind == COLON) {
  5.1444 +                    nextToken();
  5.1445                      message = parseExpression();
  5.1446                  }
  5.1447                  JCAssert t = to(F.at(pos).Assert(assertion, message));
  5.1448 @@ -1851,12 +1832,12 @@
  5.1449          }
  5.1450          case ENUM:
  5.1451          default:
  5.1452 -            Name name = S.name();
  5.1453 +            Token prevToken = token;
  5.1454              JCExpression expr = parseExpression();
  5.1455 -            if (S.token() == COLON && expr.getTag() == JCTree.IDENT) {
  5.1456 -                S.nextToken();
  5.1457 +            if (token.kind == COLON && expr.getTag() == JCTree.IDENT) {
  5.1458 +                nextToken();
  5.1459                  JCStatement stat = parseStatement();
  5.1460 -                return F.at(pos).Labelled(name, stat);
  5.1461 +                return F.at(pos).Labelled(prevToken.name(), stat);
  5.1462              } else {
  5.1463                  // This Exec is an "ExpressionStatement"; it subsumes the terminating semicolon
  5.1464                  JCExpressionStatement stat = to(F.at(pos).Exec(checkExprStat(expr)));
  5.1465 @@ -1869,7 +1850,7 @@
  5.1466      /** CatchClause     = CATCH "(" FormalParameter ")" Block
  5.1467       */
  5.1468      protected JCCatch catchClause() {
  5.1469 -        int pos = S.pos();
  5.1470 +        int pos = token.pos;
  5.1471          accept(CATCH);
  5.1472          accept(LPAREN);
  5.1473          JCModifiers mods = optFinal(Flags.PARAMETER);
  5.1474 @@ -1886,9 +1867,9 @@
  5.1475      List<JCExpression> catchTypes() {
  5.1476          ListBuffer<JCExpression> catchTypes = ListBuffer.lb();
  5.1477          catchTypes.add(parseType());
  5.1478 -        while (S.token() == BAR) {
  5.1479 +        while (token.kind == BAR) {
  5.1480              checkMulticatch();
  5.1481 -            S.nextToken();
  5.1482 +            nextToken();
  5.1483              catchTypes.add(qualident());
  5.1484          }
  5.1485          return catchTypes.toList();
  5.1486 @@ -1901,33 +1882,33 @@
  5.1487      List<JCCase> switchBlockStatementGroups() {
  5.1488          ListBuffer<JCCase> cases = new ListBuffer<JCCase>();
  5.1489          while (true) {
  5.1490 -            int pos = S.pos();
  5.1491 -            switch (S.token()) {
  5.1492 +            int pos = token.pos;
  5.1493 +            switch (token.kind) {
  5.1494              case CASE: {
  5.1495 -                S.nextToken();
  5.1496 +                nextToken();
  5.1497                  JCExpression pat = parseExpression();
  5.1498                  accept(COLON);
  5.1499                  List<JCStatement> stats = blockStatements();
  5.1500                  JCCase c = F.at(pos).Case(pat, stats);
  5.1501                  if (stats.isEmpty())
  5.1502 -                    storeEnd(c, S.prevEndPos());
  5.1503 +                    storeEnd(c, S.prevToken().endPos);
  5.1504                  cases.append(c);
  5.1505                  break;
  5.1506              }
  5.1507              case DEFAULT: {
  5.1508 -                S.nextToken();
  5.1509 +                nextToken();
  5.1510                  accept(COLON);
  5.1511                  List<JCStatement> stats = blockStatements();
  5.1512                  JCCase c = F.at(pos).Case(null, stats);
  5.1513                  if (stats.isEmpty())
  5.1514 -                    storeEnd(c, S.prevEndPos());
  5.1515 +                    storeEnd(c, S.prevToken().endPos);
  5.1516                  cases.append(c);
  5.1517                  break;
  5.1518              }
  5.1519              case RBRACE: case EOF:
  5.1520                  return cases.toList();
  5.1521              default:
  5.1522 -                S.nextToken(); // to ensure progress
  5.1523 +                nextToken(); // to ensure progress
  5.1524                  syntaxError(pos, "expected3",
  5.1525                      CASE, DEFAULT, RBRACE);
  5.1526              }
  5.1527 @@ -1941,9 +1922,9 @@
  5.1528                                                                      T stats) {
  5.1529          // This Exec is a "StatementExpression"; it subsumes no terminating token
  5.1530          stats.append(toP(F.at(pos).Exec(checkExprStat(first))));
  5.1531 -        while (S.token() == COMMA) {
  5.1532 -            S.nextToken();
  5.1533 -            pos = S.pos();
  5.1534 +        while (token.kind == COMMA) {
  5.1535 +            nextToken();
  5.1536 +            pos = token.pos;
  5.1537              JCExpression t = parseExpression();
  5.1538              // This Exec is a "StatementExpression"; it subsumes no terminating token
  5.1539              stats.append(toP(F.at(pos).Exec(checkExprStat(t))));
  5.1540 @@ -1956,13 +1937,13 @@
  5.1541       */
  5.1542      List<JCStatement> forInit() {
  5.1543          ListBuffer<JCStatement> stats = lb();
  5.1544 -        int pos = S.pos();
  5.1545 -        if (S.token() == FINAL || S.token() == MONKEYS_AT) {
  5.1546 +        int pos = token.pos;
  5.1547 +        if (token.kind == FINAL || token.kind == MONKEYS_AT) {
  5.1548              return variableDeclarators(optFinal(0), parseType(), stats).toList();
  5.1549          } else {
  5.1550              JCExpression t = term(EXPR | TYPE);
  5.1551              if ((lastmode & TYPE) != 0 &&
  5.1552 -                (S.token() == IDENTIFIER || S.token() == ASSERT || S.token() == ENUM))
  5.1553 +                (token.kind == IDENTIFIER || token.kind == ASSERT || token.kind == ENUM))
  5.1554                  return variableDeclarators(modifiersOpt(), t, stats).toList();
  5.1555              else
  5.1556                  return moreStatementExpressions(pos, t, stats).toList();
  5.1557 @@ -1972,7 +1953,7 @@
  5.1558      /** ForUpdate = StatementExpression MoreStatementExpressions
  5.1559       */
  5.1560      List<JCExpressionStatement> forUpdate() {
  5.1561 -        return moreStatementExpressions(S.pos(),
  5.1562 +        return moreStatementExpressions(token.pos,
  5.1563                                          parseExpression(),
  5.1564                                          new ListBuffer<JCExpressionStatement>()).toList();
  5.1565      }
  5.1566 @@ -1980,11 +1961,11 @@
  5.1567      /** AnnotationsOpt = { '@' Annotation }
  5.1568       */
  5.1569      List<JCAnnotation> annotationsOpt() {
  5.1570 -        if (S.token() != MONKEYS_AT) return List.nil(); // optimization
  5.1571 +        if (token.kind != MONKEYS_AT) return List.nil(); // optimization
  5.1572          ListBuffer<JCAnnotation> buf = new ListBuffer<JCAnnotation>();
  5.1573 -        while (S.token() == MONKEYS_AT) {
  5.1574 -            int pos = S.pos();
  5.1575 -            S.nextToken();
  5.1576 +        while (token.kind == MONKEYS_AT) {
  5.1577 +            int pos = token.pos;
  5.1578 +            nextToken();
  5.1579              buf.append(annotation(pos));
  5.1580          }
  5.1581          return buf.toList();
  5.1582 @@ -2004,21 +1985,20 @@
  5.1583          int pos;
  5.1584          if (partial == null) {
  5.1585              flags = 0;
  5.1586 -            pos = S.pos();
  5.1587 +            pos = token.pos;
  5.1588          } else {
  5.1589              flags = partial.flags;
  5.1590              annotations.appendList(partial.annotations);
  5.1591              pos = partial.pos;
  5.1592          }
  5.1593 -        if (S.deprecatedFlag()) {
  5.1594 +        if (token.deprecatedFlag) {
  5.1595              flags |= Flags.DEPRECATED;
  5.1596 -            S.resetDeprecatedFlag();
  5.1597          }
  5.1598          int lastPos = Position.NOPOS;
  5.1599      loop:
  5.1600          while (true) {
  5.1601              long flag;
  5.1602 -            switch (S.token()) {
  5.1603 +            switch (token.kind) {
  5.1604              case PRIVATE     : flag = Flags.PRIVATE; break;
  5.1605              case PROTECTED   : flag = Flags.PROTECTED; break;
  5.1606              case PUBLIC      : flag = Flags.PUBLIC; break;
  5.1607 @@ -2031,15 +2011,15 @@
  5.1608              case SYNCHRONIZED: flag = Flags.SYNCHRONIZED; break;
  5.1609              case STRICTFP    : flag = Flags.STRICTFP; break;
  5.1610              case MONKEYS_AT  : flag = Flags.ANNOTATION; break;
  5.1611 -            case ERROR       : flag = 0; S.nextToken(); break;
  5.1612 +            case ERROR       : flag = 0; nextToken(); break;
  5.1613              default: break loop;
  5.1614              }
  5.1615 -            if ((flags & flag) != 0) error(S.pos(), "repeated.modifier");
  5.1616 -            lastPos = S.pos();
  5.1617 -            S.nextToken();
  5.1618 +            if ((flags & flag) != 0) error(token.pos, "repeated.modifier");
  5.1619 +            lastPos = token.pos;
  5.1620 +            nextToken();
  5.1621              if (flag == Flags.ANNOTATION) {
  5.1622                  checkAnnotations();
  5.1623 -                if (S.token() != INTERFACE) {
  5.1624 +                if (token.kind != INTERFACE) {
  5.1625                      JCAnnotation ann = annotation(lastPos);
  5.1626                      // if first modifier is an annotation, set pos to annotation's.
  5.1627                      if (flags == 0 && annotations.isEmpty())
  5.1628 @@ -2051,7 +2031,7 @@
  5.1629              }
  5.1630              flags |= flag;
  5.1631          }
  5.1632 -        switch (S.token()) {
  5.1633 +        switch (token.kind) {
  5.1634          case ENUM: flags |= Flags.ENUM; break;
  5.1635          case INTERFACE: flags |= Flags.INTERFACE; break;
  5.1636          default: break;
  5.1637 @@ -2064,7 +2044,7 @@
  5.1638  
  5.1639          JCModifiers mods = F.at(pos).Modifiers(flags, annotations.toList());
  5.1640          if (pos != Position.NOPOS)
  5.1641 -            storeEnd(mods, S.prevEndPos());
  5.1642 +            storeEnd(mods, S.prevToken().endPos);
  5.1643          return mods;
  5.1644      }
  5.1645  
  5.1646 @@ -2077,22 +2057,22 @@
  5.1647          JCTree ident = qualident();
  5.1648          List<JCExpression> fieldValues = annotationFieldValuesOpt();
  5.1649          JCAnnotation ann = F.at(pos).Annotation(ident, fieldValues);
  5.1650 -        storeEnd(ann, S.prevEndPos());
  5.1651 +        storeEnd(ann, S.prevToken().endPos);
  5.1652          return ann;
  5.1653      }
  5.1654  
  5.1655      List<JCExpression> annotationFieldValuesOpt() {
  5.1656 -        return (S.token() == LPAREN) ? annotationFieldValues() : List.<JCExpression>nil();
  5.1657 +        return (token.kind == LPAREN) ? annotationFieldValues() : List.<JCExpression>nil();
  5.1658      }
  5.1659  
  5.1660      /** AnnotationFieldValues   = "(" [ AnnotationFieldValue { "," AnnotationFieldValue } ] ")" */
  5.1661      List<JCExpression> annotationFieldValues() {
  5.1662          accept(LPAREN);
  5.1663          ListBuffer<JCExpression> buf = new ListBuffer<JCExpression>();
  5.1664 -        if (S.token() != RPAREN) {
  5.1665 +        if (token.kind != RPAREN) {
  5.1666              buf.append(annotationFieldValue());
  5.1667 -            while (S.token() == COMMA) {
  5.1668 -                S.nextToken();
  5.1669 +            while (token.kind == COMMA) {
  5.1670 +                nextToken();
  5.1671                  buf.append(annotationFieldValue());
  5.1672              }
  5.1673          }
  5.1674 @@ -2104,11 +2084,11 @@
  5.1675       *                          | Identifier "=" AnnotationValue
  5.1676       */
  5.1677      JCExpression annotationFieldValue() {
  5.1678 -        if (S.token() == IDENTIFIER) {
  5.1679 +        if (token.kind == IDENTIFIER) {
  5.1680              mode = EXPR;
  5.1681              JCExpression t1 = term1();
  5.1682 -            if (t1.getTag() == JCTree.IDENT && S.token() == EQ) {
  5.1683 -                int pos = S.pos();
  5.1684 +            if (t1.getTag() == JCTree.IDENT && token.kind == EQ) {
  5.1685 +                int pos = token.pos;
  5.1686                  accept(EQ);
  5.1687                  JCExpression v = annotationValue();
  5.1688                  return toP(F.at(pos).Assign(t1, v));
  5.1689 @@ -2125,20 +2105,20 @@
  5.1690       */
  5.1691      JCExpression annotationValue() {
  5.1692          int pos;
  5.1693 -        switch (S.token()) {
  5.1694 +        switch (token.kind) {
  5.1695          case MONKEYS_AT:
  5.1696 -            pos = S.pos();
  5.1697 -            S.nextToken();
  5.1698 +            pos = token.pos;
  5.1699 +            nextToken();
  5.1700              return annotation(pos);
  5.1701          case LBRACE:
  5.1702 -            pos = S.pos();
  5.1703 +            pos = token.pos;
  5.1704              accept(LBRACE);
  5.1705              ListBuffer<JCExpression> buf = new ListBuffer<JCExpression>();
  5.1706 -            if (S.token() != RBRACE) {
  5.1707 +            if (token.kind != RBRACE) {
  5.1708                  buf.append(annotationValue());
  5.1709 -                while (S.token() == COMMA) {
  5.1710 -                    S.nextToken();
  5.1711 -                    if (S.token() == RBRACE) break;
  5.1712 +                while (token.kind == COMMA) {
  5.1713 +                    nextToken();
  5.1714 +                    if (token.kind == RBRACE) break;
  5.1715                      buf.append(annotationValue());
  5.1716                  }
  5.1717              }
  5.1718 @@ -2156,7 +2136,7 @@
  5.1719                                                                           JCExpression type,
  5.1720                                                                           T vdefs)
  5.1721      {
  5.1722 -        return variableDeclaratorsRest(S.pos(), mods, type, ident(), false, null, vdefs);
  5.1723 +        return variableDeclaratorsRest(token.pos, mods, type, ident(), false, null, vdefs);
  5.1724      }
  5.1725  
  5.1726      /** VariableDeclaratorsRest = VariableDeclaratorRest { "," VariableDeclarator }
  5.1727 @@ -2174,10 +2154,10 @@
  5.1728                                                                       T vdefs)
  5.1729      {
  5.1730          vdefs.append(variableDeclaratorRest(pos, mods, type, name, reqInit, dc));
  5.1731 -        while (S.token() == COMMA) {
  5.1732 +        while (token.kind == COMMA) {
  5.1733              // All but last of multiple declarators subsume a comma
  5.1734 -            storeEnd((JCTree)vdefs.elems.last(), S.endPos());
  5.1735 -            S.nextToken();
  5.1736 +            storeEnd((JCTree)vdefs.elems.last(), token.endPos);
  5.1737 +            nextToken();
  5.1738              vdefs.append(variableDeclarator(mods, type, reqInit, dc));
  5.1739          }
  5.1740          return vdefs;
  5.1741 @@ -2187,7 +2167,7 @@
  5.1742       *  ConstantDeclarator = Ident ConstantDeclaratorRest
  5.1743       */
  5.1744      JCVariableDecl variableDeclarator(JCModifiers mods, JCExpression type, boolean reqInit, String dc) {
  5.1745 -        return variableDeclaratorRest(S.pos(), mods, type, ident(), reqInit, dc);
  5.1746 +        return variableDeclaratorRest(token.pos, mods, type, ident(), reqInit, dc);
  5.1747      }
  5.1748  
  5.1749      /** VariableDeclaratorRest = BracketsOpt ["=" VariableInitializer]
  5.1750 @@ -2200,11 +2180,11 @@
  5.1751                                    boolean reqInit, String dc) {
  5.1752          type = bracketsOpt(type);
  5.1753          JCExpression init = null;
  5.1754 -        if (S.token() == EQ) {
  5.1755 -            S.nextToken();
  5.1756 +        if (token.kind == EQ) {
  5.1757 +            nextToken();
  5.1758              init = variableInitializer();
  5.1759          }
  5.1760 -        else if (reqInit) syntaxError(S.pos(), "expected", EQ);
  5.1761 +        else if (reqInit) syntaxError(token.pos, "expected", EQ);
  5.1762          JCVariableDecl result =
  5.1763              toP(F.at(pos).VarDef(mods, name, type, init));
  5.1764          attach(result, dc);
  5.1765 @@ -2214,11 +2194,11 @@
  5.1766      /** VariableDeclaratorId = Ident BracketsOpt
  5.1767       */
  5.1768      JCVariableDecl variableDeclaratorId(JCModifiers mods, JCExpression type) {
  5.1769 -        int pos = S.pos();
  5.1770 +        int pos = token.pos;
  5.1771          Name name = ident();
  5.1772          if ((mods.flags & Flags.VARARGS) != 0 &&
  5.1773 -                S.token() == LBRACKET) {
  5.1774 -            log.error(S.pos(), "varargs.and.old.array.syntax");
  5.1775 +                token.kind == LBRACKET) {
  5.1776 +            log.error(token.pos, "varargs.and.old.array.syntax");
  5.1777          }
  5.1778          type = bracketsOpt(type);
  5.1779          return toP(F.at(pos).VarDef(mods, name, type, null));
  5.1780 @@ -2229,12 +2209,12 @@
  5.1781      List<JCTree> resources() {
  5.1782          ListBuffer<JCTree> defs = new ListBuffer<JCTree>();
  5.1783          defs.append(resource());
  5.1784 -        while (S.token() == SEMI) {
  5.1785 +        while (token.kind == SEMI) {
  5.1786              // All but last of multiple declarators must subsume a semicolon
  5.1787 -            storeEnd(defs.elems.last(), S.endPos());
  5.1788 -            int semiColonPos = S.pos();
  5.1789 -            S.nextToken();
  5.1790 -            if (S.token() == RPAREN) { // Optional trailing semicolon
  5.1791 +            storeEnd(defs.elems.last(), token.endPos);
  5.1792 +            int semiColonPos = token.pos;
  5.1793 +            nextToken();
  5.1794 +            if (token.kind == RPAREN) { // Optional trailing semicolon
  5.1795                                         // after last resource
  5.1796                  break;
  5.1797              }
  5.1798 @@ -2248,7 +2228,7 @@
  5.1799      protected JCTree resource() {
  5.1800          JCModifiers optFinal = optFinal(Flags.FINAL);
  5.1801          JCExpression type = parseType();
  5.1802 -        int pos = S.pos();
  5.1803 +        int pos = token.pos;
  5.1804          Name ident = ident();
  5.1805          return variableDeclaratorRest(pos, optFinal, type, ident, true, null);
  5.1806      }
  5.1807 @@ -2256,54 +2236,61 @@
  5.1808      /** CompilationUnit = [ { "@" Annotation } PACKAGE Qualident ";"] {ImportDeclaration} {TypeDeclaration}
  5.1809       */
  5.1810      public JCTree.JCCompilationUnit parseCompilationUnit() {
  5.1811 -        int pos = S.pos();
  5.1812 +        Token firstToken = token;
  5.1813          JCExpression pid = null;
  5.1814 -        String dc = S.docComment();
  5.1815          JCModifiers mods = null;
  5.1816 +        boolean consumedToplevelDoc = false;
  5.1817 +        boolean seenImport = false;
  5.1818 +        boolean seenPackage = false;
  5.1819          List<JCAnnotation> packageAnnotations = List.nil();
  5.1820 -        if (S.token() == MONKEYS_AT)
  5.1821 +        if (token.kind == MONKEYS_AT)
  5.1822              mods = modifiersOpt();
  5.1823  
  5.1824 -        if (S.token() == PACKAGE) {
  5.1825 +        if (token.kind == PACKAGE) {
  5.1826 +            seenPackage = true;
  5.1827              if (mods != null) {
  5.1828                  checkNoMods(mods.flags);
  5.1829                  packageAnnotations = mods.annotations;
  5.1830                  mods = null;
  5.1831              }
  5.1832 -            S.nextToken();
  5.1833 +            nextToken();
  5.1834              pid = qualident();
  5.1835              accept(SEMI);
  5.1836          }
  5.1837          ListBuffer<JCTree> defs = new ListBuffer<JCTree>();
  5.1838          boolean checkForImports = true;
  5.1839 -        while (S.token() != EOF) {
  5.1840 -            if (S.pos() <= errorEndPos) {
  5.1841 +        boolean firstTypeDecl = true;
  5.1842 +        while (token.kind != EOF) {
  5.1843 +            if (token.pos <= errorEndPos) {
  5.1844                  // error recovery
  5.1845                  skip(checkForImports, false, false, false);
  5.1846 -                if (S.token() == EOF)
  5.1847 +                if (token.kind == EOF)
  5.1848                      break;
  5.1849              }
  5.1850 -            if (checkForImports && mods == null && S.token() == IMPORT) {
  5.1851 +            if (checkForImports && mods == null && token.kind == IMPORT) {
  5.1852 +                seenImport = true;
  5.1853                  defs.append(importDeclaration());
  5.1854              } else {
  5.1855 -                JCTree def = typeDeclaration(mods);
  5.1856 -                if (keepDocComments && dc != null && docComments.get(def) == dc) {
  5.1857 -                    // If the first type declaration has consumed the first doc
  5.1858 -                    // comment, then don't use it for the top level comment as well.
  5.1859 -                    dc = null;
  5.1860 +                String docComment = token.docComment;
  5.1861 +                if (firstTypeDecl && !seenImport && !seenPackage) {
  5.1862 +                    docComment = firstToken.docComment;
  5.1863 +                    consumedToplevelDoc = true;
  5.1864                  }
  5.1865 +                JCTree def = typeDeclaration(mods, docComment);
  5.1866                  if (def instanceof JCExpressionStatement)
  5.1867                      def = ((JCExpressionStatement)def).expr;
  5.1868                  defs.append(def);
  5.1869                  if (def instanceof JCClassDecl)
  5.1870                      checkForImports = false;
  5.1871                  mods = null;
  5.1872 +                firstTypeDecl = false;
  5.1873              }
  5.1874          }
  5.1875 -        JCTree.JCCompilationUnit toplevel = F.at(pos).TopLevel(packageAnnotations, pid, defs.toList());
  5.1876 -        attach(toplevel, dc);
  5.1877 +        JCTree.JCCompilationUnit toplevel = F.at(firstToken.pos).TopLevel(packageAnnotations, pid, defs.toList());
  5.1878 +        if (!consumedToplevelDoc)
  5.1879 +            attach(toplevel, firstToken.docComment);
  5.1880          if (defs.elems.isEmpty())
  5.1881 -            storeEnd(toplevel, S.prevEndPos());
  5.1882 +            storeEnd(toplevel, S.prevToken().endPos);
  5.1883          if (keepDocComments)
  5.1884              toplevel.docComments = docComments;
  5.1885          if (keepLineMap)
  5.1886 @@ -2314,26 +2301,26 @@
  5.1887      /** ImportDeclaration = IMPORT [ STATIC ] Ident { "." Ident } [ "." "*" ] ";"
  5.1888       */
  5.1889      JCTree importDeclaration() {
  5.1890 -        int pos = S.pos();
  5.1891 -        S.nextToken();
  5.1892 +        int pos = token.pos;
  5.1893 +        nextToken();
  5.1894          boolean importStatic = false;
  5.1895 -        if (S.token() == STATIC) {
  5.1896 +        if (token.kind == STATIC) {
  5.1897              checkStaticImports();
  5.1898              importStatic = true;
  5.1899 -            S.nextToken();
  5.1900 +            nextToken();
  5.1901          }
  5.1902 -        JCExpression pid = toP(F.at(S.pos()).Ident(ident()));
  5.1903 +        JCExpression pid = toP(F.at(token.pos).Ident(ident()));
  5.1904          do {
  5.1905 -            int pos1 = S.pos();
  5.1906 +            int pos1 = token.pos;
  5.1907              accept(DOT);
  5.1908 -            if (S.token() == STAR) {
  5.1909 +            if (token.kind == STAR) {
  5.1910                  pid = to(F.at(pos1).Select(pid, names.asterisk));
  5.1911 -                S.nextToken();
  5.1912 +                nextToken();
  5.1913                  break;
  5.1914              } else {
  5.1915                  pid = toP(F.at(pos1).Select(pid, ident()));
  5.1916              }
  5.1917 -        } while (S.token() == DOT);
  5.1918 +        } while (token.kind == DOT);
  5.1919          accept(SEMI);
  5.1920          return toP(F.at(pos).Import(pid, importStatic));
  5.1921      }
  5.1922 @@ -2341,14 +2328,13 @@
  5.1923      /** TypeDeclaration = ClassOrInterfaceOrEnumDeclaration
  5.1924       *                  | ";"
  5.1925       */
  5.1926 -    JCTree typeDeclaration(JCModifiers mods) {
  5.1927 -        int pos = S.pos();
  5.1928 -        if (mods == null && S.token() == SEMI) {
  5.1929 -            S.nextToken();
  5.1930 +    JCTree typeDeclaration(JCModifiers mods, String docComment) {
  5.1931 +        int pos = token.pos;
  5.1932 +        if (mods == null && token.kind == SEMI) {
  5.1933 +            nextToken();
  5.1934              return toP(F.at(pos).Skip());
  5.1935          } else {
  5.1936 -            String dc = S.docComment();
  5.1937 -            return classOrInterfaceOrEnumDeclaration(modifiersOpt(mods), dc);
  5.1938 +            return classOrInterfaceOrEnumDeclaration(modifiersOpt(mods), docComment);
  5.1939          }
  5.1940      }
  5.1941  
  5.1942 @@ -2358,19 +2344,19 @@
  5.1943       *  @param dc       The documentation comment for the class, or null.
  5.1944       */
  5.1945      JCStatement classOrInterfaceOrEnumDeclaration(JCModifiers mods, String dc) {
  5.1946 -        if (S.token() == CLASS) {
  5.1947 +        if (token.kind == CLASS) {
  5.1948              return classDeclaration(mods, dc);
  5.1949 -        } else if (S.token() == INTERFACE) {
  5.1950 +        } else if (token.kind == INTERFACE) {
  5.1951              return interfaceDeclaration(mods, dc);
  5.1952          } else if (allowEnums) {
  5.1953 -            if (S.token() == ENUM) {
  5.1954 +            if (token.kind == ENUM) {
  5.1955                  return enumDeclaration(mods, dc);
  5.1956              } else {
  5.1957 -                int pos = S.pos();
  5.1958 +                int pos = token.pos;
  5.1959                  List<JCTree> errs;
  5.1960 -                if (S.token() == IDENTIFIER) {
  5.1961 +                if (token.kind == IDENTIFIER) {
  5.1962                      errs = List.<JCTree>of(mods, toP(F.at(pos).Ident(ident())));
  5.1963 -                    setErrorEndPos(S.pos());
  5.1964 +                    setErrorEndPos(token.pos);
  5.1965                  } else {
  5.1966                      errs = List.<JCTree>of(mods);
  5.1967                  }
  5.1968 @@ -2378,16 +2364,16 @@
  5.1969                                                CLASS, INTERFACE, ENUM)));
  5.1970              }
  5.1971          } else {
  5.1972 -            if (S.token() == ENUM) {
  5.1973 -                error(S.pos(), "enums.not.supported.in.source", source.name);
  5.1974 +            if (token.kind == ENUM) {
  5.1975 +                error(token.pos, "enums.not.supported.in.source", source.name);
  5.1976                  allowEnums = true;
  5.1977                  return enumDeclaration(mods, dc);
  5.1978              }
  5.1979 -            int pos = S.pos();
  5.1980 +            int pos = token.pos;
  5.1981              List<JCTree> errs;
  5.1982 -            if (S.token() == IDENTIFIER) {
  5.1983 +            if (token.kind == IDENTIFIER) {
  5.1984                  errs = List.<JCTree>of(mods, toP(F.at(pos).Ident(ident())));
  5.1985 -                setErrorEndPos(S.pos());
  5.1986 +                setErrorEndPos(token.pos);
  5.1987              } else {
  5.1988                  errs = List.<JCTree>of(mods);
  5.1989              }
  5.1990 @@ -2402,20 +2388,20 @@
  5.1991       *  @param dc       The documentation comment for the class, or null.
  5.1992       */
  5.1993      JCClassDecl classDeclaration(JCModifiers mods, String dc) {
  5.1994 -        int pos = S.pos();
  5.1995 +        int pos = token.pos;
  5.1996          accept(CLASS);
  5.1997          Name name = ident();
  5.1998  
  5.1999          List<JCTypeParameter> typarams = typeParametersOpt();
  5.2000  
  5.2001          JCExpression extending = null;
  5.2002 -        if (S.token() == EXTENDS) {
  5.2003 -            S.nextToken();
  5.2004 +        if (token.kind == EXTENDS) {
  5.2005 +            nextToken();
  5.2006              extending = parseType();
  5.2007          }
  5.2008          List<JCExpression> implementing = List.nil();
  5.2009 -        if (S.token() == IMPLEMENTS) {
  5.2010 -            S.nextToken();
  5.2011 +        if (token.kind == IMPLEMENTS) {
  5.2012 +            nextToken();
  5.2013              implementing = typeList();
  5.2014          }
  5.2015          List<JCTree> defs = classOrInterfaceBody(name, false);
  5.2016 @@ -2431,15 +2417,15 @@
  5.2017       *  @param dc       The documentation comment for the interface, or null.
  5.2018       */
  5.2019      JCClassDecl interfaceDeclaration(JCModifiers mods, String dc) {
  5.2020 -        int pos = S.pos();
  5.2021 +        int pos = token.pos;
  5.2022          accept(INTERFACE);
  5.2023          Name name = ident();
  5.2024  
  5.2025          List<JCTypeParameter> typarams = typeParametersOpt();
  5.2026  
  5.2027          List<JCExpression> extending = List.nil();
  5.2028 -        if (S.token() == EXTENDS) {
  5.2029 -            S.nextToken();
  5.2030 +        if (token.kind == EXTENDS) {
  5.2031 +            nextToken();
  5.2032              extending = typeList();
  5.2033          }
  5.2034          List<JCTree> defs = classOrInterfaceBody(name, true);
  5.2035 @@ -2454,13 +2440,13 @@
  5.2036       *  @param dc       The documentation comment for the enum, or null.
  5.2037       */
  5.2038      JCClassDecl enumDeclaration(JCModifiers mods, String dc) {
  5.2039 -        int pos = S.pos();
  5.2040 +        int pos = token.pos;
  5.2041          accept(ENUM);
  5.2042          Name name = ident();
  5.2043  
  5.2044          List<JCExpression> implementing = List.nil();
  5.2045 -        if (S.token() == IMPLEMENTS) {
  5.2046 -            S.nextToken();
  5.2047 +        if (token.kind == IMPLEMENTS) {
  5.2048 +            nextToken();
  5.2049              implementing = typeList();
  5.2050          }
  5.2051  
  5.2052 @@ -2479,27 +2465,27 @@
  5.2053      List<JCTree> enumBody(Name enumName) {
  5.2054          accept(LBRACE);
  5.2055          ListBuffer<JCTree> defs = new ListBuffer<JCTree>();
  5.2056 -        if (S.token() == COMMA) {
  5.2057 -            S.nextToken();
  5.2058 -        } else if (S.token() != RBRACE && S.token() != SEMI) {
  5.2059 +        if (token.kind == COMMA) {
  5.2060 +            nextToken();
  5.2061 +        } else if (token.kind != RBRACE && token.kind != SEMI) {
  5.2062              defs.append(enumeratorDeclaration(enumName));
  5.2063 -            while (S.token() == COMMA) {
  5.2064 -                S.nextToken();
  5.2065 -                if (S.token() == RBRACE || S.token() == SEMI) break;
  5.2066 +            while (token.kind == COMMA) {
  5.2067 +                nextToken();
  5.2068 +                if (token.kind == RBRACE || token.kind == SEMI) break;
  5.2069                  defs.append(enumeratorDeclaration(enumName));
  5.2070              }
  5.2071 -            if (S.token() != SEMI && S.token() != RBRACE) {
  5.2072 -                defs.append(syntaxError(S.pos(), "expected3",
  5.2073 +            if (token.kind != SEMI && token.kind != RBRACE) {
  5.2074 +                defs.append(syntaxError(token.pos, "expected3",
  5.2075                                  COMMA, RBRACE, SEMI));
  5.2076 -                S.nextToken();
  5.2077 +                nextToken();
  5.2078              }
  5.2079          }
  5.2080 -        if (S.token() == SEMI) {
  5.2081 -            S.nextToken();
  5.2082 -            while (S.token() != RBRACE && S.token() != EOF) {
  5.2083 +        if (token.kind == SEMI) {
  5.2084 +            nextToken();
  5.2085 +            while (token.kind != RBRACE && token.kind != EOF) {
  5.2086                  defs.appendList(classOrInterfaceBodyDeclaration(enumName,
  5.2087                                                                  false));
  5.2088 -                if (S.pos() <= errorEndPos) {
  5.2089 +                if (token.pos <= errorEndPos) {
  5.2090                      // error recovery
  5.2091                     skip(false, true, true, false);
  5.2092                  }
  5.2093 @@ -2512,23 +2498,22 @@
  5.2094      /** EnumeratorDeclaration = AnnotationsOpt [TypeArguments] IDENTIFIER [ Arguments ] [ "{" ClassBody "}" ]
  5.2095       */
  5.2096      JCTree enumeratorDeclaration(Name enumName) {
  5.2097 -        String dc = S.docComment();
  5.2098 +        String dc = token.docComment;
  5.2099          int flags = Flags.PUBLIC|Flags.STATIC|Flags.FINAL|Flags.ENUM;
  5.2100 -        if (S.deprecatedFlag()) {
  5.2101 +        if (token.deprecatedFlag) {
  5.2102              flags |= Flags.DEPRECATED;
  5.2103 -            S.resetDeprecatedFlag();
  5.2104          }
  5.2105 -        int pos = S.pos();
  5.2106 +        int pos = token.pos;
  5.2107          List<JCAnnotation> annotations = annotationsOpt();
  5.2108          JCModifiers mods = F.at(annotations.isEmpty() ? Position.NOPOS : pos).Modifiers(flags, annotations);
  5.2109          List<JCExpression> typeArgs = typeArgumentsOpt();
  5.2110 -        int identPos = S.pos();
  5.2111 +        int identPos = token.pos;
  5.2112          Name name = ident();
  5.2113 -        int createPos = S.pos();
  5.2114 -        List<JCExpression> args = (S.token() == LPAREN)
  5.2115 +        int createPos = token.pos;
  5.2116 +        List<JCExpression> args = (token.kind == LPAREN)
  5.2117              ? arguments() : List.<JCExpression>nil();
  5.2118          JCClassDecl body = null;
  5.2119 -        if (S.token() == LBRACE) {
  5.2120 +        if (token.kind == LBRACE) {
  5.2121              JCModifiers mods1 = F.at(Position.NOPOS).Modifiers(Flags.ENUM | Flags.STATIC);
  5.2122              List<JCTree> defs = classOrInterfaceBody(names.empty, false);
  5.2123              body = toP(F.at(identPos).AnonymousClassDef(mods1, defs));
  5.2124 @@ -2538,7 +2523,7 @@
  5.2125          JCIdent ident = F.at(identPos).Ident(enumName);
  5.2126          JCNewClass create = F.at(createPos).NewClass(null, typeArgs, ident, args, body);
  5.2127          if (createPos != identPos)
  5.2128 -            storeEnd(create, S.prevEndPos());
  5.2129 +            storeEnd(create, S.prevToken().endPos);
  5.2130          ident = F.at(identPos).Ident(enumName);
  5.2131          JCTree result = toP(F.at(pos).VarDef(mods, name, ident, create));
  5.2132          attach(result, dc);
  5.2133 @@ -2550,8 +2535,8 @@
  5.2134      List<JCExpression> typeList() {
  5.2135          ListBuffer<JCExpression> ts = new ListBuffer<JCExpression>();
  5.2136          ts.append(parseType());
  5.2137 -        while (S.token() == COMMA) {
  5.2138 -            S.nextToken();
  5.2139 +        while (token.kind == COMMA) {
  5.2140 +            nextToken();
  5.2141              ts.append(parseType());
  5.2142          }
  5.2143          return ts.toList();
  5.2144 @@ -2562,16 +2547,16 @@
  5.2145       */
  5.2146      List<JCTree> classOrInterfaceBody(Name className, boolean isInterface) {
  5.2147          accept(LBRACE);
  5.2148 -        if (S.pos() <= errorEndPos) {
  5.2149 +        if (token.pos <= errorEndPos) {
  5.2150              // error recovery
  5.2151              skip(false, true, false, false);
  5.2152 -            if (S.token() == LBRACE)
  5.2153 -                S.nextToken();
  5.2154 +            if (token.kind == LBRACE)
  5.2155 +                nextToken();
  5.2156          }
  5.2157          ListBuffer<JCTree> defs = new ListBuffer<JCTree>();
  5.2158 -        while (S.token() != RBRACE && S.token() != EOF) {
  5.2159 +        while (token.kind != RBRACE && token.kind != EOF) {
  5.2160              defs.appendList(classOrInterfaceBodyDeclaration(className, isInterface));
  5.2161 -            if (S.pos() <= errorEndPos) {
  5.2162 +            if (token.pos <= errorEndPos) {
  5.2163                 // error recovery
  5.2164                 skip(false, true, true, false);
  5.2165             }
  5.2166 @@ -2598,23 +2583,23 @@
  5.2167       *      ( ConstantDeclaratorsRest | InterfaceMethodDeclaratorRest ";" )
  5.2168       */
  5.2169      protected List<JCTree> classOrInterfaceBodyDeclaration(Name className, boolean isInterface) {
  5.2170 -        if (S.token() == SEMI) {
  5.2171 -            S.nextToken();
  5.2172 +        if (token.kind == SEMI) {
  5.2173 +            nextToken();
  5.2174              return List.<JCTree>nil();
  5.2175          } else {
  5.2176 -            String dc = S.docComment();
  5.2177 -            int pos = S.pos();
  5.2178 +            String dc = token.docComment;
  5.2179 +            int pos = token.pos;
  5.2180              JCModifiers mods = modifiersOpt();
  5.2181 -            if (S.token() == CLASS ||
  5.2182 -                S.token() == INTERFACE ||
  5.2183 -                allowEnums && S.token() == ENUM) {
  5.2184 +            if (token.kind == CLASS ||
  5.2185 +                token.kind == INTERFACE ||
  5.2186 +                allowEnums && token.kind == ENUM) {
  5.2187                  return List.<JCTree>of(classOrInterfaceOrEnumDeclaration(mods, dc));
  5.2188 -            } else if (S.token() == LBRACE && !isInterface &&
  5.2189 +            } else if (token.kind == LBRACE && !isInterface &&
  5.2190                         (mods.flags & Flags.StandardFlags & ~Flags.STATIC) == 0 &&
  5.2191                         mods.annotations.isEmpty()) {
  5.2192                  return List.<JCTree>of(block(pos, mods.flags));
  5.2193              } else {
  5.2194 -                pos = S.pos();
  5.2195 +                pos = token.pos;
  5.2196                  List<JCTypeParameter> typarams = typeParametersOpt();
  5.2197                  // if there are type parameters but no modifiers, save the start
  5.2198                  // position of the method in the modifiers.
  5.2199 @@ -2622,26 +2607,26 @@
  5.2200                      mods.pos = pos;
  5.2201                      storeEnd(mods, pos);
  5.2202                  }
  5.2203 -                Name name = S.name();
  5.2204 -                pos = S.pos();
  5.2205 +                Token tk = token;
  5.2206 +                pos = token.pos;
  5.2207                  JCExpression type;
  5.2208 -                boolean isVoid = S.token() == VOID;
  5.2209 +                boolean isVoid = token.kind == VOID;
  5.2210                  if (isVoid) {
  5.2211                      type = to(F.at(pos).TypeIdent(TypeTags.VOID));
  5.2212 -                    S.nextToken();
  5.2213 +                    nextToken();
  5.2214                  } else {
  5.2215                      type = parseType();
  5.2216                  }
  5.2217 -                if (S.token() == LPAREN && !isInterface && type.getTag() == JCTree.IDENT) {
  5.2218 -                    if (isInterface || name != className)
  5.2219 +                if (token.kind == LPAREN && !isInterface && type.getTag() == JCTree.IDENT) {
  5.2220 +                    if (isInterface || tk.name() != className)
  5.2221                          error(pos, "invalid.meth.decl.ret.type.req");
  5.2222                      return List.of(methodDeclaratorRest(
  5.2223                          pos, mods, null, names.init, typarams,
  5.2224                          isInterface, true, dc));
  5.2225                  } else {
  5.2226 -                    pos = S.pos();
  5.2227 -                    name = ident();
  5.2228 -                    if (S.token() == LPAREN) {
  5.2229 +                    pos = token.pos;
  5.2230 +                    Name name = ident();
  5.2231 +                    if (token.kind == LPAREN) {
  5.2232                          return List.of(methodDeclaratorRest(
  5.2233                              pos, mods, type, name, typarams,
  5.2234                              isInterface, isVoid, dc));
  5.2235 @@ -2649,16 +2634,16 @@
  5.2236                          List<JCTree> defs =
  5.2237                              variableDeclaratorsRest(pos, mods, type, name, isInterface, dc,
  5.2238                                                      new ListBuffer<JCTree>()).toList();
  5.2239 -                        storeEnd(defs.last(), S.endPos());
  5.2240 +                        storeEnd(defs.last(), token.endPos);
  5.2241                          accept(SEMI);
  5.2242                          return defs;
  5.2243                      } else {
  5.2244 -                        pos = S.pos();
  5.2245 +                        pos = token.pos;
  5.2246                          List<JCTree> err = isVoid
  5.2247                              ? List.<JCTree>of(toP(F.at(pos).MethodDef(mods, name, type, typarams,
  5.2248                                  List.<JCVariableDecl>nil(), List.<JCExpression>nil(), null, null)))
  5.2249                              : null;
  5.2250 -                        return List.<JCTree>of(syntaxError(S.pos(), err, "expected", LPAREN));
  5.2251 +                        return List.<JCTree>of(syntaxError(token.pos, err, "expected", LPAREN));
  5.2252                      }
  5.2253                  }
  5.2254              }
  5.2255 @@ -2686,27 +2671,27 @@
  5.2256          List<JCVariableDecl> params = formalParameters();
  5.2257          if (!isVoid) type = bracketsOpt(type);
  5.2258          List<JCExpression> thrown = List.nil();
  5.2259 -        if (S.token() == THROWS) {
  5.2260 -            S.nextToken();
  5.2261 +        if (token.kind == THROWS) {
  5.2262 +            nextToken();
  5.2263              thrown = qualidentList();
  5.2264          }
  5.2265          JCBlock body = null;
  5.2266          JCExpression defaultValue;
  5.2267 -        if (S.token() == LBRACE) {
  5.2268 +        if (token.kind == LBRACE) {
  5.2269              body = block();
  5.2270              defaultValue = null;
  5.2271          } else {
  5.2272 -            if (S.token() == DEFAULT) {
  5.2273 +            if (token.kind == DEFAULT) {
  5.2274                  accept(DEFAULT);
  5.2275                  defaultValue = annotationValue();
  5.2276              } else {
  5.2277                  defaultValue = null;
  5.2278              }
  5.2279              accept(SEMI);
  5.2280 -            if (S.pos() <= errorEndPos) {
  5.2281 +            if (token.pos <= errorEndPos) {
  5.2282                  // error recovery
  5.2283                  skip(false, true, false, false);
  5.2284 -                if (S.token() == LBRACE) {
  5.2285 +                if (token.kind == LBRACE) {
  5.2286                      body = block();
  5.2287                  }
  5.2288              }
  5.2289 @@ -2725,8 +2710,8 @@
  5.2290      List<JCExpression> qualidentList() {
  5.2291          ListBuffer<JCExpression> ts = new ListBuffer<JCExpression>();
  5.2292          ts.append(qualident());
  5.2293 -        while (S.token() == COMMA) {
  5.2294 -            S.nextToken();
  5.2295 +        while (token.kind == COMMA) {
  5.2296 +            nextToken();
  5.2297              ts.append(qualident());
  5.2298          }
  5.2299          return ts.toList();
  5.2300 @@ -2735,13 +2720,13 @@
  5.2301      /** TypeParametersOpt = ["<" TypeParameter {"," TypeParameter} ">"]
  5.2302       */
  5.2303      List<JCTypeParameter> typeParametersOpt() {
  5.2304 -        if (S.token() == LT) {
  5.2305 +        if (token.kind == LT) {
  5.2306              checkGenerics();
  5.2307              ListBuffer<JCTypeParameter> typarams = new ListBuffer<JCTypeParameter>();
  5.2308 -            S.nextToken();
  5.2309 +            nextToken();
  5.2310              typarams.append(typeParameter());
  5.2311 -            while (S.token() == COMMA) {
  5.2312 -                S.nextToken();
  5.2313 +            while (token.kind == COMMA) {
  5.2314 +                nextToken();
  5.2315                  typarams.append(typeParameter());
  5.2316              }
  5.2317              accept(GT);
  5.2318 @@ -2756,14 +2741,14 @@
  5.2319       *  TypeVariable = Ident
  5.2320       */
  5.2321      JCTypeParameter typeParameter() {
  5.2322 -        int pos = S.pos();
  5.2323 +        int pos = token.pos;
  5.2324          Name name = ident();
  5.2325          ListBuffer<JCExpression> bounds = new ListBuffer<JCExpression>();
  5.2326 -        if (S.token() == EXTENDS) {
  5.2327 -            S.nextToken();
  5.2328 +        if (token.kind == EXTENDS) {
  5.2329 +            nextToken();
  5.2330              bounds.append(parseType());
  5.2331 -            while (S.token() == AMP) {
  5.2332 -                S.nextToken();
  5.2333 +            while (token.kind == AMP) {
  5.2334 +                nextToken();
  5.2335                  bounds.append(parseType());
  5.2336              }
  5.2337          }
  5.2338 @@ -2778,10 +2763,10 @@
  5.2339          ListBuffer<JCVariableDecl> params = new ListBuffer<JCVariableDecl>();
  5.2340          JCVariableDecl lastParam = null;
  5.2341          accept(LPAREN);
  5.2342 -        if (S.token() != RPAREN) {
  5.2343 +        if (token.kind != RPAREN) {
  5.2344              params.append(lastParam = formalParameter());
  5.2345 -            while ((lastParam.mods.flags & Flags.VARARGS) == 0 && S.token() == COMMA) {
  5.2346 -                S.nextToken();
  5.2347 +            while ((lastParam.mods.flags & Flags.VARARGS) == 0 && token.kind == COMMA) {
  5.2348 +                nextToken();
  5.2349                  params.append(lastParam = formalParameter());
  5.2350              }
  5.2351          }
  5.2352 @@ -2802,11 +2787,11 @@
  5.2353      protected JCVariableDecl formalParameter() {
  5.2354          JCModifiers mods = optFinal(Flags.PARAMETER);
  5.2355          JCExpression type = parseType();
  5.2356 -        if (S.token() == ELLIPSIS) {
  5.2357 +        if (token.kind == ELLIPSIS) {
  5.2358              checkVarargs();
  5.2359              mods.flags |= Flags.VARARGS;
  5.2360 -            type = to(F.at(S.pos()).TypeArray(type));
  5.2361 -            S.nextToken();
  5.2362 +            type = to(F.at(token.pos).TypeArray(type));
  5.2363 +            nextToken();
  5.2364          }
  5.2365          return variableDeclaratorId(mods, type);
  5.2366      }
  5.2367 @@ -2849,7 +2834,7 @@
  5.2368      /** Return precedence of operator represented by token,
  5.2369       *  -1 if token is not a binary operator. @see TreeInfo.opPrec
  5.2370       */
  5.2371 -    static int prec(Token token) {
  5.2372 +    static int prec(TokenKind token) {
  5.2373          int oc = optag(token);
  5.2374          return (oc >= 0) ? TreeInfo.opPrec(oc) : -1;
  5.2375      }
  5.2376 @@ -2869,7 +2854,7 @@
  5.2377      /** Return operation tag of binary operator represented by token,
  5.2378       *  -1 if token is not a binary operator.
  5.2379       */
  5.2380 -    static int optag(Token token) {
  5.2381 +    static int optag(TokenKind token) {
  5.2382          switch (token) {
  5.2383          case BARBAR:
  5.2384              return JCTree.OR;
  5.2385 @@ -2941,7 +2926,7 @@
  5.2386      /** Return operation tag of unary operator represented by token,
  5.2387       *  -1 if token is not a binary operator.
  5.2388       */
  5.2389 -    static int unoptag(Token token) {
  5.2390 +    static int unoptag(TokenKind token) {
  5.2391          switch (token) {
  5.2392          case PLUS:
  5.2393              return JCTree.POS;
  5.2394 @@ -2963,7 +2948,7 @@
  5.2395      /** Return type tag of basic type represented by token,
  5.2396       *  -1 if token is not a basic type identifier.
  5.2397       */
  5.2398 -    static int typetag(Token token) {
  5.2399 +    static int typetag(TokenKind token) {
  5.2400          switch (token) {
  5.2401          case BYTE:
  5.2402              return TypeTags.BYTE;
  5.2403 @@ -2988,49 +2973,49 @@
  5.2404  
  5.2405      void checkGenerics() {
  5.2406          if (!allowGenerics) {
  5.2407 -            error(S.pos(), "generics.not.supported.in.source", source.name);
  5.2408 +            error(token.pos, "generics.not.supported.in.source", source.name);
  5.2409              allowGenerics = true;
  5.2410          }
  5.2411      }
  5.2412      void checkVarargs() {
  5.2413          if (!allowVarargs) {
  5.2414 -            error(S.pos(), "varargs.not.supported.in.source", source.name);
  5.2415 +            error(token.pos, "varargs.not.supported.in.source", source.name);
  5.2416              allowVarargs = true;
  5.2417          }
  5.2418      }
  5.2419      void checkForeach() {
  5.2420          if (!allowForeach) {
  5.2421 -            error(S.pos(), "foreach.not.supported.in.source", source.name);
  5.2422 +            error(token.pos, "foreach.not.supported.in.source", source.name);
  5.2423              allowForeach = true;
  5.2424          }
  5.2425      }
  5.2426      void checkStaticImports() {
  5.2427          if (!allowStaticImport) {
  5.2428 -            error(S.pos(), "static.import.not.supported.in.source", source.name);
  5.2429 +            error(token.pos, "static.import.not.supported.in.source", source.name);
  5.2430              allowStaticImport = true;
  5.2431          }
  5.2432      }
  5.2433      void checkAnnotations() {
  5.2434          if (!allowAnnotations) {
  5.2435 -            error(S.pos(), "annotations.not.supported.in.source", source.name);
  5.2436 +            error(token.pos, "annotations.not.supported.in.source", source.name);
  5.2437              allowAnnotations = true;
  5.2438          }
  5.2439      }
  5.2440      void checkDiamond() {
  5.2441          if (!allowDiamond) {
  5.2442 -            error(S.pos(), "diamond.not.supported.in.source", source.name);
  5.2443 +            error(token.pos, "diamond.not.supported.in.source", source.name);
  5.2444              allowDiamond = true;
  5.2445          }
  5.2446      }
  5.2447      void checkMulticatch() {
  5.2448          if (!allowMulticatch) {
  5.2449 -            error(S.pos(), "multicatch.not.supported.in.source", source.name);
  5.2450 +            error(token.pos, "multicatch.not.supported.in.source", source.name);
  5.2451              allowMulticatch = true;
  5.2452          }
  5.2453      }
  5.2454      void checkTryWithResources() {
  5.2455          if (!allowTWR) {
  5.2456 -            error(S.pos(), "try.with.resources.not.supported.in.source", source.name);
  5.2457 +            error(token.pos, "try.with.resources.not.supported.in.source", source.name);
  5.2458              allowTWR = true;
  5.2459          }
  5.2460      }

     6.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     6.2 +++ b/src/share/classes/com/sun/tools/javac/parser/JavadocTokenizer.java	Mon Oct 24 13:00:20 2011 +0100
     6.3 @@ -0,0 +1,412 @@
     6.4 +/*
     6.5 + * Copyright (c) 2004, 2011, Oracle and/or its affiliates. All rights reserved.
     6.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
     6.7 + *
     6.8 + * This code is free software; you can redistribute it and/or modify it
     6.9 + * under the terms of the GNU General Public License version 2 only, as
    6.10 + * published by the Free Software Foundation.  Oracle designates this
    6.11 + * particular file as subject to the "Classpath" exception as provided
    6.12 + * by Oracle in the LICENSE file that accompanied this code.
    6.13 + *
    6.14 + * This code is distributed in the hope that it will be useful, but WITHOUT
    6.15 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    6.16 + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
    6.17 + * version 2 for more details (a copy is included in the LICENSE file that
    6.18 + * accompanied this code).
    6.19 + *
    6.20 + * You should have received a copy of the GNU General Public License version
    6.21 + * 2 along with this work; if not, write to the Free Software Foundation,
    6.22 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
    6.23 + *
    6.24 + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
    6.25 + * or visit www.oracle.com if you need additional information or have any
    6.26 + * questions.
    6.27 + */
    6.28 +
    6.29 +package com.sun.tools.javac.parser;
    6.30 +
    6.31 +import com.sun.tools.javac.file.JavacFileManager;
    6.32 +import com.sun.tools.javac.parser.Tokens.Token;
    6.33 +import com.sun.tools.javac.util.*;
    6.34 +
    6.35 +import java.nio.*;
    6.36 +
    6.37 +import static com.sun.tools.javac.util.LayoutCharacters.*;
    6.38 +
    6.39 +/** An extension to the base lexical analyzer that captures
    6.40 + *  and processes the contents of doc comments.  It does so by
    6.41 + *  translating Unicode escape sequences and by stripping the
    6.42 + *  leading whitespace and starts from each line of the comment.
    6.43 + *
    6.44 + *  <p><b>This is NOT part of any supported API.
    6.45 + *  If you write code that depends on this, you do so at your own risk.
    6.46 + *  This code and its internal interfaces are subject to change or
    6.47 + *  deletion without notice.</b>
    6.48 + */
    6.49 +public class JavadocTokenizer extends JavaTokenizer {
    6.50 +
    6.51 +    /** Create a scanner from the input buffer.  buffer must implement
    6.52 +     *  array() and compact(), and remaining() must be less than limit().
    6.53 +     */
    6.54 +    protected JavadocTokenizer(ScannerFactory fac, CharBuffer buffer) {
    6.55 +        super(fac, buffer);
    6.56 +    }
    6.57 +
    6.58 +    /** Create a scanner from the input array.  The array must have at
    6.59 +     *  least a single character of extra space.
    6.60 +     */
    6.61 +    protected JavadocTokenizer(ScannerFactory fac, char[] input, int inputLength) {
    6.62 +        super(fac, input, inputLength);
    6.63 +    }
    6.64 +
    6.65 +    /** The comment input buffer, index of next chacter to be read,
    6.66 +     *  index of one past last character in buffer.
    6.67 +     */
    6.68 +    private char[] buf;
    6.69 +    private int bp;
    6.70 +    private int buflen;
    6.71 +
    6.72 +    /** The current character.
    6.73 +     */
    6.74 +    private char ch;
    6.75 +
    6.76 +    /** The column number position of the current character.
    6.77 +     */
    6.78 +    private int col;
    6.79 +
    6.80 +    /** The buffer index of the last converted Unicode character
    6.81 +     */
    6.82 +    private int unicodeConversionBp = 0;
    6.83 +
    6.84 +    /**
    6.85 +     * Buffer for doc comment.
    6.86 +     */
    6.87 +    private char[] docCommentBuffer = new char[1024];
    6.88 +
    6.89 +    /**
    6.90 +     * Number of characters in doc comment buffer.
    6.91 +     */
    6.92 +    private int docCommentCount;
    6.93 +
    6.94 +    /**
    6.95 +     * Translated and stripped contents of doc comment
    6.96 +     */
    6.97 +    private String docComment = null;
    6.98 +
    6.99 +
   6.100 +    /** Unconditionally expand the comment buffer.
   6.101 +     */
   6.102 +    private void expandCommentBuffer() {
   6.103 +        char[] newBuffer = new char[docCommentBuffer.length * 2];
   6.104 +        System.arraycopy(docCommentBuffer, 0, newBuffer,
   6.105 +                         0, docCommentBuffer.length);
   6.106 +        docCommentBuffer = newBuffer;
   6.107 +    }
   6.108 +
   6.109 +    /** Convert an ASCII digit from its base (8, 10, or 16)
   6.110 +     *  to its value.
   6.111 +     */
   6.112 +    private int digit(int base) {
   6.113 +        char c = ch;
   6.114 +        int result = Character.digit(c, base);
   6.115 +        if (result >= 0 && c > 0x7f) {
   6.116 +            ch = "0123456789abcdef".charAt(result);
   6.117 +        }
   6.118 +        return result;
   6.119 +    }
   6.120 +
   6.121 +    /** Convert Unicode escape; bp points to initial '\' character
   6.122 +     *  (Spec 3.3).
   6.123 +     */
   6.124 +    private void convertUnicode() {
   6.125 +        if (ch == '\\' && unicodeConversionBp != bp) {
   6.126 +            bp++; ch = buf[bp]; col++;
   6.127 +            if (ch == 'u') {
   6.128 +                do {
   6.129 +                    bp++; ch = buf[bp]; col++;
   6.130 +                } while (ch == 'u');
   6.131 +                int limit = bp + 3;
   6.132 +                if (limit < buflen) {
   6.133 +                    int d = digit(16);
   6.134 +                    int code = d;
   6.135 +                    while (bp < limit && d >= 0) {
   6.136 +                        bp++; ch = buf[bp]; col++;
   6.137 +                        d = digit(16);
   6.138 +                        code = (code << 4) + d;
   6.139 +                    }
   6.140 +                    if (d >= 0) {
   6.141 +                        ch = (char)code;
   6.142 +                        unicodeConversionBp = bp;
   6.143 +                        return;
   6.144 +                    }
   6.145 +                }
   6.146 +                // "illegal.Unicode.esc", reported by base scanner
   6.147 +            } else {
   6.148 +                bp--;
   6.149 +                ch = '\\';
   6.150 +                col--;
   6.151 +            }
   6.152 +        }
   6.153 +    }
   6.154 +
   6.155 +
   6.156 +    /** Read next character.
   6.157 +     */
   6.158 +    private void scanChar() {
   6.159 +        bp++;
   6.160 +        ch = buf[bp];
   6.161 +        switch (ch) {
   6.162 +        case '\r': // return
   6.163 +            col = 0;
   6.164 +            break;
   6.165 +        case '\n': // newline
   6.166 +            if (bp == 0 || buf[bp-1] != '\r') {
   6.167 +                col = 0;
   6.168 +            }
   6.169 +            break;
   6.170 +        case '\t': // tab
   6.171 +            col = (col / TabInc * TabInc) + TabInc;
   6.172 +            break;
   6.173 +        case '\\': // possible Unicode
   6.174 +            col++;
   6.175 +            convertUnicode();
   6.176 +            break;
   6.177 +        default:
   6.178 +            col++;
   6.179 +            break;
   6.180 +        }
   6.181 +    }
   6.182 +
   6.183 +    @Override
   6.184 +    public Token readToken() {
   6.185 +        docComment = null;
   6.186 +        Token tk = super.readToken();
   6.187 +        tk.docComment = docComment;
   6.188 +        return tk;
   6.189 +    }
   6.190 +
   6.191 +    /**
   6.192 +     * Read next character in doc comment, skipping over double '\' characters.
   6.193 +     * If a double '\' is skipped, put in the buffer and update buffer count.
   6.194 +     */
   6.195 +    private void scanDocCommentChar() {
   6.196 +        scanChar();
   6.197 +        if (ch == '\\') {
   6.198 +            if (buf[bp+1] == '\\' && unicodeConversionBp != bp) {
   6.199 +                if (docCommentCount == docCommentBuffer.length)
   6.200 +                    expandCommentBuffer();
   6.201 +                docCommentBuffer[docCommentCount++] = ch;
   6.202 +                bp++; col++;
   6.203 +            } else {
   6.204 +                convertUnicode();
   6.205 +            }
   6.206 +        }
   6.207 +    }
   6.208 +
   6.209 +    /**
   6.210 +     * Process a doc comment and make the string content available.
   6.211 +     * Strips leading whitespace and stars.
   6.212 +     */
   6.213 +    @SuppressWarnings("fallthrough")
   6.214 +    protected void processComment(int pos, int endPos, CommentStyle style) {
   6.215 +        if (style != CommentStyle.JAVADOC) {
   6.216 +            return;
   6.217 +        }
   6.218 +
   6.219 +        buf = reader.getRawCharacters(pos, endPos);
   6.220 +        buflen = buf.length;
   6.221 +        bp = 0;
   6.222 +        col = 0;
   6.223 +
   6.224 +        docCommentCount = 0;
   6.225 +
   6.226 +        boolean firstLine = true;
   6.227 +
   6.228 +        // Skip over first slash
   6.229 +        scanDocCommentChar();
   6.230 +        // Skip over first star
   6.231 +        scanDocCommentChar();
   6.232 +
   6.233 +        // consume any number of stars
   6.234 +        while (bp < buflen && ch == '*') {
   6.235 +            scanDocCommentChar();
   6.236 +        }
   6.237 +        // is the comment in the form /**/, /***/, /****/, etc. ?
   6.238 +        if (bp < buflen && ch == '/') {
   6.239 +            docComment = "";
   6.240 +            return;
   6.241 +        }
   6.242 +
   6.243 +        // skip a newline on the first line of the comment.
   6.244 +        if (bp < buflen) {
   6.245 +            if (ch == LF) {
   6.246 +                scanDocCommentChar();
   6.247 +                firstLine = false;
   6.248 +            } else if (ch == CR) {
   6.249 +                scanDocCommentChar();
   6.250 +                if (ch == LF) {
   6.251 +                    scanDocCommentChar();
   6.252 +                    firstLine = false;
   6.253 +                }
   6.254 +            }
   6.255 +        }
   6.256 +
   6.257 +    outerLoop:
   6.258 +
   6.259 +        // The outerLoop processes the doc comment, looping once
   6.260 +        // for each line.  For each line, it first strips off
   6.261 +        // whitespace, then it consumes any stars, then it
   6.262 +        // puts the rest of the line into our buffer.
   6.263 +        while (bp < buflen) {
   6.264 +
   6.265 +            // The wsLoop consumes whitespace from the beginning
   6.266 +            // of each line.
   6.267 +        wsLoop:
   6.268 +
   6.269 +            while (bp < buflen) {
   6.270 +                switch(ch) {
   6.271 +                case ' ':
   6.272 +                    scanDocCommentChar();
   6.273 +                    break;
   6.274 +                case '\t':
   6.275 +                    col = ((col - 1) / TabInc * TabInc) + TabInc;
   6.276 +                    scanDocCommentChar();
   6.277 +                    break;
   6.278 +                case FF:
   6.279 +                    col = 0;
   6.280 +                    scanDocCommentChar();
   6.281 +                    break;
   6.282 +// Treat newline at beginning of line (blank line, no star)
   6.283 +// as comment text.  Old Javadoc compatibility requires this.
   6.284 +/*---------------------------------*
   6.285 +                case CR: // (Spec 3.4)
   6.286 +                    scanDocCommentChar();
   6.287 +                    if (ch == LF) {
   6.288 +                        col = 0;
   6.289 +                        scanDocCommentChar();
   6.290 +                    }
   6.291 +                    break;
   6.292 +                case LF: // (Spec 3.4)
   6.293 +                    scanDocCommentChar();
   6.294 +                    break;
   6.295 +*---------------------------------*/
   6.296 +                default:
   6.297 +                    // we've seen something that isn't whitespace;
   6.298 +                    // jump out.
   6.299 +                    break wsLoop;
   6.300 +                }
   6.301 +            }
   6.302 +
   6.303 +            // Are there stars here?  If so, consume them all
   6.304 +            // and check for the end of comment.
   6.305 +            if (ch == '*') {
   6.306 +                // skip all of the stars
   6.307 +                do {
   6.308 +                    scanDocCommentChar();
   6.309 +                } while (ch == '*');
   6.310 +
   6.311 +                // check for the closing slash.
   6.312 +                if (ch == '/') {
   6.313 +                    // We're done with the doc comment
   6.314 +                    // scanChar() and breakout.
   6.315 +                    break outerLoop;
   6.316 +                }
   6.317 +            } else if (! firstLine) {
   6.318 +                //The current line does not begin with a '*' so we will indent it.
   6.319 +                for (int i = 1; i < col; i++) {
   6.320 +                    if (docCommentCount == docCommentBuffer.length)
   6.321 +                        expandCommentBuffer();
   6.322 +                    docCommentBuffer[docCommentCount++] = ' ';
   6.323 +                }
   6.324 +            }
   6.325 +
   6.326 +            // The textLoop processes the rest of the characters
   6.327 +            // on the line, adding them to our buffer.
   6.328 +        textLoop:
   6.329 +            while (bp < buflen) {
   6.330 +                switch (ch) {
   6.331 +                case '*':
   6.332 +                    // Is this just a star?  Or is this the
   6.333 +                    // end of a comment?
   6.334 +                    scanDocCommentChar();
   6.335 +                    if (ch == '/') {
   6.336 +                        // This is the end of the comment,
   6.337 +                        // set ch and return our buffer.
   6.338 +                        break outerLoop;
   6.339 +                    }
   6.340 +                    // This is just an ordinary star.  Add it to
   6.341 +                    // the buffer.
   6.342 +                    if (docCommentCount == docCommentBuffer.length)
   6.343 +                        expandCommentBuffer();
   6.344 +                    docCommentBuffer[docCommentCount++] = '*';
   6.345 +                    break;
   6.346 +                case ' ':
   6.347 +                case '\t':
   6.348 +                    if (docCommentCount == docCommentBuffer.length)
   6.349 +                        expandCommentBuffer();
   6.350 +                    docCommentBuffer[docCommentCount++] = ch;
   6.351 +                    scanDocCommentChar();
   6.352 +                    break;
   6.353 +                case FF:
   6.354 +                    scanDocCommentChar();
   6.355 +                    break textLoop; // treat as end of line
   6.356 +                case CR: // (Spec 3.4)
   6.357 +                    scanDocCommentChar();
   6.358 +                    if (ch != LF) {
   6.359 +                        // Canonicalize CR-only line terminator to LF
   6.360 +                        if (docCommentCount == docCommentBuffer.length)
   6.361 +                            expandCommentBuffer();
   6.362 +                        docCommentBuffer[docCommentCount++] = (char)LF;
   6.363 +                        break textLoop;
   6.364 +                    }
   6.365 +                    /* fall through to LF case */
   6.366 +                case LF: // (Spec 3.4)
   6.367 +                    // We've seen a newline.  Add it to our
   6.368 +                    // buffer and break out of this loop,
   6.369 +                    // starting fresh on a new line.
   6.370 +                    if (docCommentCount == docCommentBuffer.length)
   6.371 +                        expandCommentBuffer();
   6.372 +                    docCommentBuffer[docCommentCount++] = ch;
   6.373 +                    scanDocCommentChar();
   6.374 +                    break textLoop;
   6.375 +                default:
   6.376 +                    // Add the character to our buffer.
   6.377 +                    if (docCommentCount == docCommentBuffer.length)
   6.378 +                        expandCommentBuffer();
   6.379 +                    docCommentBuffer[docCommentCount++] = ch;
   6.380 +                    scanDocCommentChar();
   6.381 +                }
   6.382 +            } // end textLoop
   6.383 +            firstLine = false;
   6.384 +        } // end outerLoop
   6.385 +
   6.386 +        if (docCommentCount > 0) {
   6.387 +            int i = docCommentCount - 1;
   6.388 +        trailLoop:
   6.389 +            while (i > -1) {
   6.390 +                switch (docCommentBuffer[i]) {
   6.391 +                case '*':
   6.392 +                    i--;
   6.393 +                    break;
   6.394 +                default:
   6.395 +                    break trailLoop;
   6.396 +                }
   6.397 +            }
   6.398 +            docCommentCount = i + 1;
   6.399 +
   6.400 +            // Store the text of the doc comment
   6.401 +            docComment = new String(docCommentBuffer, 0 , docCommentCount);
   6.402 +        } else {
   6.403 +            docComment = "";
   6.404 +        }
   6.405 +    }
   6.406 +
   6.407 +    /** Build a map for translating between line numbers and
   6.408 +     * positions in the input.
   6.409 +     *
   6.410 +     * @return a LineMap */
   6.411 +    public Position.LineMap getLineMap() {
   6.412 +        char[] buf = reader.getRawCharacters();
   6.413 +        return Position.makeLineMap(buf, buf.length, true);
   6.414 +    }
   6.415 +}