src/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java

changeset 1125
56830d5cb5bb
parent 1113
d346ab55031b
child 1145
3343b22e2761
     1.1 --- a/src/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java	Tue Nov 01 15:49:45 2011 -0700
     1.2 +++ b/src/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java	Fri Nov 04 12:36:40 2011 +0000
     1.3 @@ -25,10 +25,11 @@
     1.4  
     1.5  package com.sun.tools.javac.parser;
     1.6  
     1.7 -import java.nio.CharBuffer;
     1.8  import com.sun.tools.javac.code.Source;
     1.9 +import com.sun.tools.javac.parser.Tokens.Comment.CommentStyle;
    1.10  import com.sun.tools.javac.util.*;
    1.11  
    1.12 +import java.nio.CharBuffer;
    1.13  
    1.14  import static com.sun.tools.javac.parser.Tokens.*;
    1.15  import static com.sun.tools.javac.util.LayoutCharacters.*;
    1.16 @@ -65,9 +66,6 @@
    1.17       */
    1.18      private final Log log;
    1.19  
    1.20 -    /** The name table. */
    1.21 -    private final Names names;
    1.22 -
    1.23      /** The token factory. */
    1.24      private final Tokens tokens;
    1.25  
    1.26 @@ -87,17 +85,11 @@
    1.27       */
    1.28      protected int errPos = Position.NOPOS;
    1.29  
    1.30 -    /** Has a @deprecated been encountered in last doc comment?
    1.31 -     *  this needs to be reset by client.
    1.32 +    /** The Unicode reader (low-level stream reader).
    1.33       */
    1.34 -    protected boolean deprecatedFlag = false;
    1.35 +    protected UnicodeReader reader;
    1.36  
    1.37 -    /** A character buffer for saved chars.
    1.38 -     */
    1.39 -    protected char[] sbuf = new char[128];
    1.40 -    protected int sp;
    1.41 -
    1.42 -    protected UnicodeReader reader;
    1.43 +    protected ScannerFactory fac;
    1.44  
    1.45      private static final boolean hexFloatsWork = hexFloatsWork();
    1.46      private static boolean hexFloatsWork() {
    1.47 @@ -129,14 +121,14 @@
    1.48      }
    1.49  
    1.50      protected JavaTokenizer(ScannerFactory fac, UnicodeReader reader) {
    1.51 -        log = fac.log;
    1.52 -        names = fac.names;
    1.53 -        tokens = fac.tokens;
    1.54 -        source = fac.source;
    1.55 +        this.fac = fac;
    1.56 +        this.log = fac.log;
    1.57 +        this.tokens = fac.tokens;
    1.58 +        this.source = fac.source;
    1.59          this.reader = reader;
    1.60 -        allowBinaryLiterals = source.allowBinaryLiterals();
    1.61 -        allowHexFloats = source.allowHexFloats();
    1.62 -        allowUnderscoresInLiterals = source.allowUnderscoresInLiterals();
    1.63 +        this.allowBinaryLiterals = source.allowBinaryLiterals();
    1.64 +        this.allowHexFloats = source.allowHexFloats();
    1.65 +        this.allowUnderscoresInLiterals = source.allowUnderscoresInLiterals();
    1.66      }
    1.67  
    1.68      /** Report an error at the given position using the provided arguments.
    1.69 @@ -147,38 +139,13 @@
    1.70          errPos = pos;
    1.71      }
    1.72  
    1.73 -    /** Read next character in comment, skipping over double '\' characters.
    1.74 -     */
    1.75 -    protected void scanCommentChar() {
    1.76 -        reader.scanChar();
    1.77 -        if (reader.ch == '\\') {
    1.78 -            if (reader.peekChar() == '\\' && !reader.isUnicode()) {
    1.79 -                reader.skipChar();
    1.80 -            } else {
    1.81 -                reader.convertUnicode();
    1.82 -            }
    1.83 -        }
    1.84 -    }
    1.85 -
    1.86 -    /** Append a character to sbuf.
    1.87 -     */
    1.88 -    private void putChar(char ch) {
    1.89 -        if (sp == sbuf.length) {
    1.90 -            char[] newsbuf = new char[sbuf.length * 2];
    1.91 -            System.arraycopy(sbuf, 0, newsbuf, 0, sbuf.length);
    1.92 -            sbuf = newsbuf;
    1.93 -        }
    1.94 -        sbuf[sp++] = ch;
    1.95 -    }
    1.96 -
    1.97      /** Read next character in character or string literal and copy into sbuf.
    1.98       */
    1.99      private void scanLitChar(int pos) {
   1.100          if (reader.ch == '\\') {
   1.101              if (reader.peekChar() == '\\' && !reader.isUnicode()) {
   1.102                  reader.skipChar();
   1.103 -                putChar('\\');
   1.104 -                reader.scanChar();
   1.105 +                reader.putChar('\\', true);
   1.106              } else {
   1.107                  reader.scanChar();
   1.108                  switch (reader.ch) {
   1.109 @@ -195,30 +162,30 @@
   1.110                              reader.scanChar();
   1.111                          }
   1.112                      }
   1.113 -                    putChar((char)oct);
   1.114 +                    reader.putChar((char)oct);
   1.115                      break;
   1.116                  case 'b':
   1.117 -                    putChar('\b'); reader.scanChar(); break;
   1.118 +                    reader.putChar('\b', true); break;
   1.119                  case 't':
   1.120 -                    putChar('\t'); reader.scanChar(); break;
   1.121 +                    reader.putChar('\t', true); break;
   1.122                  case 'n':
   1.123 -                    putChar('\n'); reader.scanChar(); break;
   1.124 +                    reader.putChar('\n', true); break;
   1.125                  case 'f':
   1.126 -                    putChar('\f'); reader.scanChar(); break;
   1.127 +                    reader.putChar('\f', true); break;
   1.128                  case 'r':
   1.129 -                    putChar('\r'); reader.scanChar(); break;
   1.130 +                    reader.putChar('\r', true); break;
   1.131                  case '\'':
   1.132 -                    putChar('\''); reader.scanChar(); break;
   1.133 +                    reader.putChar('\'', true); break;
   1.134                  case '\"':
   1.135 -                    putChar('\"'); reader.scanChar(); break;
   1.136 +                    reader.putChar('\"', true); break;
   1.137                  case '\\':
   1.138 -                    putChar('\\'); reader.scanChar(); break;
   1.139 +                    reader.putChar('\\', true); break;
   1.140                  default:
   1.141                      lexError(reader.bp, "illegal.esc.char");
   1.142                  }
   1.143              }
   1.144          } else if (reader.bp != reader.buflen) {
   1.145 -            putChar(reader.ch); reader.scanChar();
   1.146 +            reader.putChar(true);
   1.147          }
   1.148      }
   1.149  
   1.150 @@ -227,7 +194,7 @@
   1.151          int savePos;
   1.152          do {
   1.153              if (reader.ch != '_') {
   1.154 -                putChar(reader.ch);
   1.155 +                reader.putChar(false);
   1.156              } else {
   1.157                  if (!allowUnderscoresInLiterals) {
   1.158                      lexError(pos, "unsupported.underscore.lit", source.name);
   1.159 @@ -246,12 +213,10 @@
   1.160       */
   1.161      private void scanHexExponentAndSuffix(int pos) {
   1.162          if (reader.ch == 'p' || reader.ch == 'P') {
   1.163 -            putChar(reader.ch);
   1.164 -            reader.scanChar();
   1.165 +            reader.putChar(true);
   1.166              skipIllegalUnderscores();
   1.167              if (reader.ch == '+' || reader.ch == '-') {
   1.168 -                putChar(reader.ch);
   1.169 -                reader.scanChar();
   1.170 +                reader.putChar(true);
   1.171              }
   1.172              skipIllegalUnderscores();
   1.173              if ('0' <= reader.ch && reader.ch <= '9') {
   1.174 @@ -268,14 +233,12 @@
   1.175              lexError(pos, "malformed.fp.lit");
   1.176          }
   1.177          if (reader.ch == 'f' || reader.ch == 'F') {
   1.178 -            putChar(reader.ch);
   1.179 -            reader.scanChar();
   1.180 +            reader.putChar(true);
   1.181              tk = TokenKind.FLOATLITERAL;
   1.182              radix = 16;
   1.183          } else {
   1.184              if (reader.ch == 'd' || reader.ch == 'D') {
   1.185 -                putChar(reader.ch);
   1.186 -                reader.scanChar();
   1.187 +                reader.putChar(true);
   1.188              }
   1.189              tk = TokenKind.DOUBLELITERAL;
   1.190              radix = 16;
   1.191 @@ -289,14 +252,12 @@
   1.192          if ('0' <= reader.ch && reader.ch <= '9') {
   1.193              scanDigits(pos, 10);
   1.194          }
   1.195 -        int sp1 = sp;
   1.196 +        int sp1 = reader.sp;
   1.197          if (reader.ch == 'e' || reader.ch == 'E') {
   1.198 -            putChar(reader.ch);
   1.199 -            reader.scanChar();
   1.200 +            reader.putChar(true);
   1.201              skipIllegalUnderscores();
   1.202              if (reader.ch == '+' || reader.ch == '-') {
   1.203 -                putChar(reader.ch);
   1.204 -                reader.scanChar();
   1.205 +                reader.putChar(true);
   1.206              }
   1.207              skipIllegalUnderscores();
   1.208              if ('0' <= reader.ch && reader.ch <= '9') {
   1.209 @@ -304,7 +265,7 @@
   1.210                  return;
   1.211              }
   1.212              lexError(pos, "malformed.fp.lit");
   1.213 -            sp = sp1;
   1.214 +            reader.sp = sp1;
   1.215          }
   1.216      }
   1.217  
   1.218 @@ -314,13 +275,11 @@
   1.219          radix = 10;
   1.220          scanFraction(pos);
   1.221          if (reader.ch == 'f' || reader.ch == 'F') {
   1.222 -            putChar(reader.ch);
   1.223 -            reader.scanChar();
   1.224 +            reader.putChar(true);
   1.225              tk = TokenKind.FLOATLITERAL;
   1.226          } else {
   1.227              if (reader.ch == 'd' || reader.ch == 'D') {
   1.228 -                putChar(reader.ch);
   1.229 -                reader.scanChar();
   1.230 +                reader.putChar(true);
   1.231              }
   1.232              tk = TokenKind.DOUBLELITERAL;
   1.233          }
   1.234 @@ -331,8 +290,7 @@
   1.235      private void scanHexFractionAndSuffix(int pos, boolean seendigit) {
   1.236          radix = 16;
   1.237          Assert.check(reader.ch == '.');
   1.238 -        putChar(reader.ch);
   1.239 -        reader.scanChar();
   1.240 +        reader.putChar(true);
   1.241          skipIllegalUnderscores();
   1.242          if (reader.digit(pos, 16) >= 0) {
   1.243              seendigit = true;
   1.244 @@ -369,8 +327,7 @@
   1.245          } else if (seendigit && radix == 16 && (reader.ch == 'p' || reader.ch == 'P')) {
   1.246              scanHexExponentAndSuffix(pos);
   1.247          } else if (digitRadix == 10 && reader.ch == '.') {
   1.248 -            putChar(reader.ch);
   1.249 -            reader.scanChar();
   1.250 +            reader.putChar(true);
   1.251              scanFractionAndSuffix(pos);
   1.252          } else if (digitRadix == 10 &&
   1.253                     (reader.ch == 'e' || reader.ch == 'E' ||
   1.254 @@ -393,10 +350,7 @@
   1.255          boolean isJavaIdentifierPart;
   1.256          char high;
   1.257          do {
   1.258 -            if (sp == sbuf.length) putChar(reader.ch); else sbuf[sp++] = reader.ch;
   1.259 -            // optimization, was: putChar(reader.ch);
   1.260 -
   1.261 -            reader.scanChar();
   1.262 +            reader.putChar(true);
   1.263              switch (reader.ch) {
   1.264              case 'A': case 'B': case 'C': case 'D': case 'E':
   1.265              case 'F': case 'G': case 'H': case 'I': case 'J':
   1.266 @@ -423,7 +377,7 @@
   1.267                  break;
   1.268              case '\u001A': // EOI is also a legal identifier part
   1.269                  if (reader.bp >= reader.buflen) {
   1.270 -                    name = names.fromChars(sbuf, 0, sp);
   1.271 +                    name = reader.name();
   1.272                      tk = tokens.lookupKind(name);
   1.273                      return;
   1.274                  }
   1.275 @@ -435,11 +389,7 @@
   1.276                  } else {
   1.277                      high = reader.scanSurrogates();
   1.278                      if (high != 0) {
   1.279 -                        if (sp == sbuf.length) {
   1.280 -                            putChar(high);
   1.281 -                        } else {
   1.282 -                            sbuf[sp++] = high;
   1.283 -                        }
   1.284 +                        reader.putChar(high);
   1.285                          isJavaIdentifierPart = Character.isJavaIdentifierPart(
   1.286                              Character.toCodePoint(high, reader.ch));
   1.287                      } else {
   1.288 @@ -447,7 +397,7 @@
   1.289                      }
   1.290                  }
   1.291                  if (!isJavaIdentifierPart) {
   1.292 -                    name = names.fromChars(sbuf, 0, sp);
   1.293 +                    name = reader.name();
   1.294                      tk = tokens.lookupKind(name);
   1.295                      return;
   1.296                  }
   1.297 @@ -474,11 +424,11 @@
   1.298       */
   1.299      private void scanOperator() {
   1.300          while (true) {
   1.301 -            putChar(reader.ch);
   1.302 -            Name newname = names.fromChars(sbuf, 0, sp);
   1.303 +            reader.putChar(false);
   1.304 +            Name newname = reader.name();
   1.305              TokenKind tk1 = tokens.lookupKind(newname);
   1.306              if (tk1 == TokenKind.IDENTIFIER) {
   1.307 -                sp--;
   1.308 +                reader.sp--;
   1.309                  break;
   1.310              }
   1.311              tk = tk1;
   1.312 @@ -487,111 +437,17 @@
   1.313          }
   1.314      }
   1.315  
   1.316 -    /**
   1.317 -     * Scan a documentation comment; determine if a deprecated tag is present.
   1.318 -     * Called once the initial /, * have been skipped, positioned at the second *
   1.319 -     * (which is treated as the beginning of the first line).
   1.320 -     * Stops positioned at the closing '/'.
   1.321 -     */
   1.322 -    @SuppressWarnings("fallthrough")
   1.323 -    private void scanDocComment() {
   1.324 -        boolean deprecatedPrefix = false;
   1.325 -
   1.326 -        forEachLine:
   1.327 -        while (reader.bp < reader.buflen) {
   1.328 -
   1.329 -            // Skip optional WhiteSpace at beginning of line
   1.330 -            while (reader.bp < reader.buflen && (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF)) {
   1.331 -                scanCommentChar();
   1.332 -            }
   1.333 -
   1.334 -            // Skip optional consecutive Stars
   1.335 -            while (reader.bp < reader.buflen && reader.ch == '*') {
   1.336 -                scanCommentChar();
   1.337 -                if (reader.ch == '/') {
   1.338 -                    return;
   1.339 -                }
   1.340 -            }
   1.341 -
   1.342 -            // Skip optional WhiteSpace after Stars
   1.343 -            while (reader.bp < reader.buflen && (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF)) {
   1.344 -                scanCommentChar();
   1.345 -            }
   1.346 -
   1.347 -            deprecatedPrefix = false;
   1.348 -            // At beginning of line in the JavaDoc sense.
   1.349 -            if (reader.bp < reader.buflen && reader.ch == '@' && !deprecatedFlag) {
   1.350 -                scanCommentChar();
   1.351 -                if (reader.bp < reader.buflen && reader.ch == 'd') {
   1.352 -                    scanCommentChar();
   1.353 -                    if (reader.bp < reader.buflen && reader.ch == 'e') {
   1.354 -                        scanCommentChar();
   1.355 -                        if (reader.bp < reader.buflen && reader.ch == 'p') {
   1.356 -                            scanCommentChar();
   1.357 -                            if (reader.bp < reader.buflen && reader.ch == 'r') {
   1.358 -                                scanCommentChar();
   1.359 -                                if (reader.bp < reader.buflen && reader.ch == 'e') {
   1.360 -                                    scanCommentChar();
   1.361 -                                    if (reader.bp < reader.buflen && reader.ch == 'c') {
   1.362 -                                        scanCommentChar();
   1.363 -                                        if (reader.bp < reader.buflen && reader.ch == 'a') {
   1.364 -                                            scanCommentChar();
   1.365 -                                            if (reader.bp < reader.buflen && reader.ch == 't') {
   1.366 -                                                scanCommentChar();
   1.367 -                                                if (reader.bp < reader.buflen && reader.ch == 'e') {
   1.368 -                                                    scanCommentChar();
   1.369 -                                                    if (reader.bp < reader.buflen && reader.ch == 'd') {
   1.370 -                                                        deprecatedPrefix = true;
   1.371 -                                                        scanCommentChar();
   1.372 -                                                    }}}}}}}}}}}
   1.373 -            if (deprecatedPrefix && reader.bp < reader.buflen) {
   1.374 -                if (Character.isWhitespace(reader.ch)) {
   1.375 -                    deprecatedFlag = true;
   1.376 -                } else if (reader.ch == '*') {
   1.377 -                    scanCommentChar();
   1.378 -                    if (reader.ch == '/') {
   1.379 -                        deprecatedFlag = true;
   1.380 -                        return;
   1.381 -                    }
   1.382 -                }
   1.383 -            }
   1.384 -
   1.385 -            // Skip rest of line
   1.386 -            while (reader.bp < reader.buflen) {
   1.387 -                switch (reader.ch) {
   1.388 -                case '*':
   1.389 -                    scanCommentChar();
   1.390 -                    if (reader.ch == '/') {
   1.391 -                        return;
   1.392 -                    }
   1.393 -                    break;
   1.394 -                case CR: // (Spec 3.4)
   1.395 -                    scanCommentChar();
   1.396 -                    if (reader.ch != LF) {
   1.397 -                        continue forEachLine;
   1.398 -                    }
   1.399 -                    /* fall through to LF case */
   1.400 -                case LF: // (Spec 3.4)
   1.401 -                    scanCommentChar();
   1.402 -                    continue forEachLine;
   1.403 -                default:
   1.404 -                    scanCommentChar();
   1.405 -                }
   1.406 -            } // rest of line
   1.407 -        } // forEachLine
   1.408 -        return;
   1.409 -    }
   1.410 -
   1.411      /** Read token.
   1.412       */
   1.413      public Token readToken() {
   1.414  
   1.415 -        sp = 0;
   1.416 +        reader.sp = 0;
   1.417          name = null;
   1.418 -        deprecatedFlag = false;
   1.419          radix = 0;
   1.420 +
   1.421          int pos = 0;
   1.422          int endPos = 0;
   1.423 +        List<Comment> comments = null;
   1.424  
   1.425          try {
   1.426              loop: while (true) {
   1.427 @@ -656,7 +512,7 @@
   1.428                              scanNumber(pos, 2);
   1.429                          }
   1.430                      } else {
   1.431 -                        putChar('0');
   1.432 +                        reader.putChar('0');
   1.433                          if (reader.ch == '_') {
   1.434                              int savePos = reader.bp;
   1.435                              do {
   1.436 @@ -676,14 +532,13 @@
   1.437                  case '.':
   1.438                      reader.scanChar();
   1.439                      if ('0' <= reader.ch && reader.ch <= '9') {
   1.440 -                        putChar('.');
   1.441 +                        reader.putChar('.');
   1.442                          scanFractionAndSuffix(pos);
   1.443                      } else if (reader.ch == '.') {
   1.444 -                        putChar('.'); putChar('.');
   1.445 -                        reader.scanChar();
   1.446 +                        reader.putChar('.'); reader.putChar('.', true);
   1.447                          if (reader.ch == '.') {
   1.448                              reader.scanChar();
   1.449 -                            putChar('.');
   1.450 +                            reader.putChar('.');
   1.451                              tk = TokenKind.ELLIPSIS;
   1.452                          } else {
   1.453                              lexError(pos, "malformed.fp.lit");
   1.454 @@ -712,32 +567,36 @@
   1.455                      reader.scanChar();
   1.456                      if (reader.ch == '/') {
   1.457                          do {
   1.458 -                            scanCommentChar();
   1.459 +                            reader.scanCommentChar();
   1.460                          } while (reader.ch != CR && reader.ch != LF && reader.bp < reader.buflen);
   1.461                          if (reader.bp < reader.buflen) {
   1.462 -                            processComment(pos, reader.bp, CommentStyle.LINE);
   1.463 +                            comments = addDocReader(comments, processComment(pos, reader.bp, CommentStyle.LINE));
   1.464                          }
   1.465                          break;
   1.466                      } else if (reader.ch == '*') {
   1.467 +                        boolean isEmpty = false;
   1.468                          reader.scanChar();
   1.469                          CommentStyle style;
   1.470                          if (reader.ch == '*') {
   1.471                              style = CommentStyle.JAVADOC;
   1.472 -                            scanDocComment();
   1.473 +                            reader.scanCommentChar();
   1.474 +                            if (reader.ch == '/') {
   1.475 +                                isEmpty = true;
   1.476 +                            }
   1.477                          } else {
   1.478                              style = CommentStyle.BLOCK;
   1.479 -                            while (reader.bp < reader.buflen) {
   1.480 -                                if (reader.ch == '*') {
   1.481 -                                    reader.scanChar();
   1.482 -                                    if (reader.ch == '/') break;
   1.483 -                                } else {
   1.484 -                                    scanCommentChar();
   1.485 -                                }
   1.486 +                        }
   1.487 +                        while (!isEmpty && reader.bp < reader.buflen) {
   1.488 +                            if (reader.ch == '*') {
   1.489 +                                reader.scanChar();
   1.490 +                                if (reader.ch == '/') break;
   1.491 +                            } else {
   1.492 +                                reader.scanCommentChar();
   1.493                              }
   1.494                          }
   1.495                          if (reader.ch == '/') {
   1.496                              reader.scanChar();
   1.497 -                            processComment(pos, reader.bp, style);
   1.498 +                            comments = addDocReader(comments, processComment(pos, reader.bp, style));
   1.499                              break;
   1.500                          } else {
   1.501                              lexError(pos, "unclosed.comment");
   1.502 @@ -789,11 +648,7 @@
   1.503                          } else {
   1.504                              char high = reader.scanSurrogates();
   1.505                              if (high != 0) {
   1.506 -                                if (sp == sbuf.length) {
   1.507 -                                    putChar(high);
   1.508 -                                } else {
   1.509 -                                    sbuf[sp++] = high;
   1.510 -                                }
   1.511 +                                reader.putChar(high);
   1.512  
   1.513                                  isJavaIdentifierStart = Character.isJavaIdentifierStart(
   1.514                                      Character.toCodePoint(high, reader.ch));
   1.515 @@ -816,10 +671,10 @@
   1.516              }
   1.517              endPos = reader.bp;
   1.518              switch (tk.tag) {
   1.519 -                case DEFAULT: return new Token(tk, pos, endPos, deprecatedFlag);
   1.520 -                case NAMED: return new NamedToken(tk, pos, endPos, name, deprecatedFlag);
   1.521 -                case STRING: return new StringToken(tk, pos, endPos, new String(sbuf, 0, sp), deprecatedFlag);
   1.522 -                case NUMERIC: return new NumericToken(tk, pos, endPos, new String(sbuf, 0, sp), radix, deprecatedFlag);
   1.523 +                case DEFAULT: return new Token(tk, pos, endPos, comments);
   1.524 +                case NAMED: return new NamedToken(tk, pos, endPos, name, comments);
   1.525 +                case STRING: return new StringToken(tk, pos, endPos, reader.chars(), comments);
   1.526 +                case NUMERIC: return new NumericToken(tk, pos, endPos, reader.chars(), radix, comments);
   1.527                  default: throw new AssertionError();
   1.528              }
   1.529          }
   1.530 @@ -832,6 +687,12 @@
   1.531              }
   1.532          }
   1.533      }
   1.534 +    //where
   1.535 +        List<Comment> addDocReader(List<Comment> docReaders, Comment docReader) {
   1.536 +            return docReaders == null ?
   1.537 +                    List.of(docReader) :
   1.538 +                    docReaders.prepend(docReader);
   1.539 +        }
   1.540  
   1.541      /** Return the position where a lexical error occurred;
   1.542       */
   1.543 @@ -845,22 +706,18 @@
   1.544          errPos = pos;
   1.545      }
   1.546  
   1.547 -    public enum CommentStyle {
   1.548 -        LINE,
   1.549 -        BLOCK,
   1.550 -        JAVADOC,
   1.551 -    }
   1.552 -
   1.553      /**
   1.554       * Called when a complete comment has been scanned. pos and endPos
   1.555       * will mark the comment boundary.
   1.556       */
   1.557 -    protected void processComment(int pos, int endPos, CommentStyle style) {
   1.558 +    protected Tokens.Comment processComment(int pos, int endPos, CommentStyle style) {
   1.559          if (scannerDebug)
   1.560              System.out.println("processComment(" + pos
   1.561                                 + "," + endPos + "," + style + ")=|"
   1.562                                 + new String(reader.getRawCharacters(pos, endPos))
   1.563                                 + "|");
   1.564 +        char[] buf = reader.getRawCharacters(pos, endPos);
   1.565 +        return new BasicComment<UnicodeReader>(new UnicodeReader(fac, buf, buf.length), style);
   1.566      }
   1.567  
   1.568      /**
   1.569 @@ -893,4 +750,125 @@
   1.570      public Position.LineMap getLineMap() {
   1.571          return Position.makeLineMap(reader.getRawCharacters(), reader.buflen, false);
   1.572      }
   1.573 +
   1.574 +
   1.575 +    /**
   1.576 +    * Scan a documentation comment; determine if a deprecated tag is present.
   1.577 +    * Called once the initial /, * have been skipped, positioned at the second *
   1.578 +    * (which is treated as the beginning of the first line).
   1.579 +    * Stops positioned at the closing '/'.
   1.580 +    */
   1.581 +    protected class BasicComment<U extends UnicodeReader> implements Comment {
   1.582 +
   1.583 +        CommentStyle cs;
   1.584 +        U comment_reader;
   1.585 +
   1.586 +        protected boolean deprecatedFlag = false;
   1.587 +        protected boolean scanned = false;
   1.588 +
   1.589 +        protected BasicComment(U comment_reader, CommentStyle cs) {
   1.590 +            this.comment_reader = comment_reader;
   1.591 +            this.cs = cs;
   1.592 +        }
   1.593 +
   1.594 +        public String getText() {
   1.595 +            return null;
   1.596 +        }
   1.597 +
   1.598 +        public CommentStyle getStyle() {
   1.599 +            return cs;
   1.600 +        }
   1.601 +
   1.602 +        public boolean isDeprecated() {
   1.603 +            if (!scanned && cs == CommentStyle.JAVADOC) {
   1.604 +                scanDocComment();
   1.605 +            }
   1.606 +            return deprecatedFlag;
   1.607 +        }
   1.608 +
   1.609 +        @SuppressWarnings("fallthrough")
   1.610 +        protected void scanDocComment() {
   1.611 +            try {
   1.612 +                boolean deprecatedPrefix = false;
   1.613 +
   1.614 +                comment_reader.bp += 3; // '/**'
   1.615 +                comment_reader.ch = comment_reader.buf[comment_reader.bp];
   1.616 +
   1.617 +                forEachLine:
   1.618 +                while (comment_reader.bp < comment_reader.buflen) {
   1.619 +
   1.620 +                    // Skip optional WhiteSpace at beginning of line
   1.621 +                    while (comment_reader.bp < comment_reader.buflen && (comment_reader.ch == ' ' || comment_reader.ch == '\t' || comment_reader.ch == FF)) {
   1.622 +                        comment_reader.scanCommentChar();
   1.623 +                    }
   1.624 +
   1.625 +                    // Skip optional consecutive Stars
   1.626 +                    while (comment_reader.bp < comment_reader.buflen && comment_reader.ch == '*') {
   1.627 +                        comment_reader.scanCommentChar();
   1.628 +                        if (comment_reader.ch == '/') {
   1.629 +                            return;
   1.630 +                        }
   1.631 +                    }
   1.632 +
   1.633 +                    // Skip optional WhiteSpace after Stars
   1.634 +                    while (comment_reader.bp < comment_reader.buflen && (comment_reader.ch == ' ' || comment_reader.ch == '\t' || comment_reader.ch == FF)) {
   1.635 +                        comment_reader.scanCommentChar();
   1.636 +                    }
   1.637 +
   1.638 +                    deprecatedPrefix = false;
   1.639 +                    // At beginning of line in the JavaDoc sense.
   1.640 +                    if (!deprecatedFlag) {
   1.641 +                        String deprecated = "@deprecated";
   1.642 +                        int i = 0;
   1.643 +                        while (comment_reader.bp < comment_reader.buflen && comment_reader.ch == deprecated.charAt(i)) {
   1.644 +                            comment_reader.scanCommentChar();
   1.645 +                            i++;
   1.646 +                            if (i == deprecated.length()) {
   1.647 +                                deprecatedPrefix = true;
   1.648 +                                break;
   1.649 +                            }
   1.650 +                        }
   1.651 +                    }
   1.652 +
   1.653 +                    if (deprecatedPrefix && comment_reader.bp < comment_reader.buflen) {
   1.654 +                        if (Character.isWhitespace(comment_reader.ch)) {
   1.655 +                            deprecatedFlag = true;
   1.656 +                        } else if (comment_reader.ch == '*') {
   1.657 +                            comment_reader.scanCommentChar();
   1.658 +                            if (comment_reader.ch == '/') {
   1.659 +                                deprecatedFlag = true;
   1.660 +                                return;
   1.661 +                            }
   1.662 +                        }
   1.663 +                    }
   1.664 +
   1.665 +                    // Skip rest of line
   1.666 +                    while (comment_reader.bp < comment_reader.buflen) {
   1.667 +                        switch (comment_reader.ch) {
   1.668 +                            case '*':
   1.669 +                                comment_reader.scanCommentChar();
   1.670 +                                if (comment_reader.ch == '/') {
   1.671 +                                    return;
   1.672 +                                }
   1.673 +                                break;
   1.674 +                            case CR: // (Spec 3.4)
   1.675 +                                comment_reader.scanCommentChar();
   1.676 +                                if (comment_reader.ch != LF) {
   1.677 +                                    continue forEachLine;
   1.678 +                                }
   1.679 +                            /* fall through to LF case */
   1.680 +                            case LF: // (Spec 3.4)
   1.681 +                                comment_reader.scanCommentChar();
   1.682 +                                continue forEachLine;
   1.683 +                            default:
   1.684 +                                comment_reader.scanCommentChar();
   1.685 +                        }
   1.686 +                    } // rest of line
   1.687 +                } // forEachLine
   1.688 +                return;
   1.689 +            } finally {
   1.690 +                scanned = true;
   1.691 +            }
   1.692 +        }
   1.693 +    }
   1.694  }

mercurial