jdk8-mips64-public/langtools: comparison src/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java

-:9e2eb4bc49eb
+:56830d5cb5bb
 * questions.
 */
 package com.sun.tools.javac.parser;
+import com.sun.tools.javac.code.Source;
+import com.sun.tools.javac.parser.Tokens.Comment.CommentStyle;
+import com.sun.tools.javac.util.*;
 import java.nio.CharBuffer;
-import com.sun.tools.javac.code.Source;
-import com.sun.tools.javac.util.*;
 import static com.sun.tools.javac.parser.Tokens.*;
 import static com.sun.tools.javac.util.LayoutCharacters.*;
 /** The lexical analyzer maps an input stream consisting of
 /** The log to be used for error reporting.
 */
 private final Log log;
-/** The name table. */
-private final Names names;
 /** The token factory. */
 private final Tokens tokens;
 /** The token kind, set by nextToken().
 */
 /** The position where a lexical error occurred;
 */
 protected int errPos = Position.NOPOS;
-/** Has a @deprecated been encountered in last doc comment?
+/** The Unicode reader (low-level stream reader).
-*  this needs to be reset by client.
+*/
-*/
-protected boolean deprecatedFlag = false;
-/** A character buffer for saved chars.
-*/
-protected char[] sbuf = new char[128];
-protected int sp;
 protected UnicodeReader reader;
+protected ScannerFactory fac;
 private static final boolean hexFloatsWork = hexFloatsWork();
 private static boolean hexFloatsWork() {
 try {
 Float.valueOf("0x1.0p1");
 protected JavaTokenizer(ScannerFactory fac, char[] buf, int inputLength) {
 this(fac, new UnicodeReader(fac, buf, inputLength));
 }
 protected JavaTokenizer(ScannerFactory fac, UnicodeReader reader) {
-log = fac.log;
+this.fac = fac;
-names = fac.names;
+this.log = fac.log;
-tokens = fac.tokens;
+this.tokens = fac.tokens;
-source = fac.source;
+this.source = fac.source;
 this.reader = reader;
-allowBinaryLiterals = source.allowBinaryLiterals();
+this.allowBinaryLiterals = source.allowBinaryLiterals();
-allowHexFloats = source.allowHexFloats();
+this.allowHexFloats = source.allowHexFloats();
-allowUnderscoresInLiterals = source.allowUnderscoresInLiterals();
+this.allowUnderscoresInLiterals = source.allowUnderscoresInLiterals();
 }
 /** Report an error at the given position using the provided arguments.
 */
 protected void lexError(int pos, String key, Object... args) {
 log.error(pos, key, args);
 tk = TokenKind.ERROR;
 errPos = pos;
 }
-/** Read next character in comment, skipping over double '\' characters.
-*/
-protected void scanCommentChar() {
-reader.scanChar();
-if (reader.ch == '\\') {
-if (reader.peekChar() == '\\' && !reader.isUnicode()) {
-reader.skipChar();
-} else {
-reader.convertUnicode();
-}
-}
-}
-/** Append a character to sbuf.
-*/
-private void putChar(char ch) {
-if (sp == sbuf.length) {
-char[] newsbuf = new char[sbuf.length * 2];
-System.arraycopy(sbuf, 0, newsbuf, 0, sbuf.length);
-sbuf = newsbuf;
-}
-sbuf[sp++] = ch;
-}
 /** Read next character in character or string literal and copy into sbuf.
 */
 private void scanLitChar(int pos) {
 if (reader.ch == '\\') {
 if (reader.peekChar() == '\\' && !reader.isUnicode()) {
 reader.skipChar();
-putChar('\\');
+reader.putChar('\\', true);
-reader.scanChar();
 } else {
 reader.scanChar();
 switch (reader.ch) {
 case '0': case '1': case '2': case '3':
 case '4': case '5': case '6': case '7':
 if (leadch <= '3' && '0' <= reader.ch && reader.ch <= '7') {
 oct = oct * 8 + reader.digit(pos, 8);
 reader.scanChar();
 }
 }
-putChar((char)oct);
+reader.putChar((char)oct);
 break;
 case 'b':
-putChar('\b'); reader.scanChar(); break;
+reader.putChar('\b', true); break;
 case 't':
-putChar('\t'); reader.scanChar(); break;
+reader.putChar('\t', true); break;
 case 'n':
-putChar('\n'); reader.scanChar(); break;
+reader.putChar('\n', true); break;
 case 'f':
-putChar('\f'); reader.scanChar(); break;
+reader.putChar('\f', true); break;
 case 'r':
-putChar('\r'); reader.scanChar(); break;
+reader.putChar('\r', true); break;
 case '\'':
-putChar('\''); reader.scanChar(); break;
+reader.putChar('\'', true); break;
 case '\"':
-putChar('\"'); reader.scanChar(); break;
+reader.putChar('\"', true); break;
 case '\\':
-putChar('\\'); reader.scanChar(); break;
+reader.putChar('\\', true); break;
 default:
 lexError(reader.bp, "illegal.esc.char");
 }
 }
 } else if (reader.bp != reader.buflen) {
-putChar(reader.ch); reader.scanChar();
+reader.putChar(true);
 }
 }
 private void scanDigits(int pos, int digitRadix) {
 char saveCh;
 int savePos;
 do {
 if (reader.ch != '_') {
-putChar(reader.ch);
+reader.putChar(false);
 } else {
 if (!allowUnderscoresInLiterals) {
 lexError(pos, "unsupported.underscore.lit", source.name);
 allowUnderscoresInLiterals = true;
 }
 /** Read fractional part of hexadecimal floating point number.
 */
 private void scanHexExponentAndSuffix(int pos) {
 if (reader.ch == 'p' || reader.ch == 'P') {
-putChar(reader.ch);
+reader.putChar(true);
-reader.scanChar();
 skipIllegalUnderscores();
 if (reader.ch == '+' || reader.ch == '-') {
-putChar(reader.ch);
+reader.putChar(true);
-reader.scanChar();
 }
 skipIllegalUnderscores();
 if ('0' <= reader.ch && reader.ch <= '9') {
 scanDigits(pos, 10);
 if (!allowHexFloats) {
 lexError(pos, "malformed.fp.lit");
 } else {
 lexError(pos, "malformed.fp.lit");
 }
 if (reader.ch == 'f' || reader.ch == 'F') {
-putChar(reader.ch);
+reader.putChar(true);
-reader.scanChar();
 tk = TokenKind.FLOATLITERAL;
 radix = 16;
 } else {
 if (reader.ch == 'd' || reader.ch == 'D') {
-putChar(reader.ch);
+reader.putChar(true);
-reader.scanChar();
 }
 tk = TokenKind.DOUBLELITERAL;
 radix = 16;
 }
 }
 private void scanFraction(int pos) {
 skipIllegalUnderscores();
 if ('0' <= reader.ch && reader.ch <= '9') {
 scanDigits(pos, 10);
 }
-int sp1 = sp;
+int sp1 = reader.sp;
 if (reader.ch == 'e' || reader.ch == 'E') {
-putChar(reader.ch);
+reader.putChar(true);
-reader.scanChar();
 skipIllegalUnderscores();
 if (reader.ch == '+' || reader.ch == '-') {
-putChar(reader.ch);
+reader.putChar(true);
-reader.scanChar();
 }
 skipIllegalUnderscores();
 if ('0' <= reader.ch && reader.ch <= '9') {
 scanDigits(pos, 10);
 return;
 }
 lexError(pos, "malformed.fp.lit");
-sp = sp1;
+reader.sp = sp1;
 }
 }
 /** Read fractional part and 'd' or 'f' suffix of floating point number.
 */
 private void scanFractionAndSuffix(int pos) {
 radix = 10;
 scanFraction(pos);
 if (reader.ch == 'f' || reader.ch == 'F') {
-putChar(reader.ch);
+reader.putChar(true);
-reader.scanChar();
 tk = TokenKind.FLOATLITERAL;
 } else {
 if (reader.ch == 'd' || reader.ch == 'D') {
-putChar(reader.ch);
+reader.putChar(true);
-reader.scanChar();
 }
 tk = TokenKind.DOUBLELITERAL;
 }
 }
 /** Read fractional part and 'd' or 'f' suffix of floating point number.
 */
 private void scanHexFractionAndSuffix(int pos, boolean seendigit) {
 radix = 16;
 Assert.check(reader.ch == '.');
-putChar(reader.ch);
+reader.putChar(true);
-reader.scanChar();
 skipIllegalUnderscores();
 if (reader.digit(pos, 16) >= 0) {
 seendigit = true;
 scanDigits(pos, 16);
 }
 if (radix == 16 && reader.ch == '.') {
 scanHexFractionAndSuffix(pos, seendigit);
 } else if (seendigit && radix == 16 && (reader.ch == 'p' || reader.ch == 'P')) {
 scanHexExponentAndSuffix(pos);
 } else if (digitRadix == 10 && reader.ch == '.') {
-putChar(reader.ch);
+reader.putChar(true);
-reader.scanChar();
 scanFractionAndSuffix(pos);
 } else if (digitRadix == 10 &&
 (reader.ch == 'e' || reader.ch == 'E' ||
 reader.ch == 'f' || reader.ch == 'F' ||
 reader.ch == 'd' || reader.ch == 'D')) {
 */
 private void scanIdent() {
 boolean isJavaIdentifierPart;
 char high;
 do {
-if (sp == sbuf.length) putChar(reader.ch); else sbuf[sp++] = reader.ch;
+reader.putChar(true);
-// optimization, was: putChar(reader.ch);
-reader.scanChar();
 switch (reader.ch) {
 case 'A': case 'B': case 'C': case 'D': case 'E':
 case 'F': case 'G': case 'H': case 'I': case 'J':
 case 'K': case 'L': case 'M': case 'N': case 'O':
 case 'P': case 'Q': case 'R': case 'S': case 'T':
 case '\u0018': case '\u0019': case '\u001B':
 case '\u007F':
 break;
 case '\u001A': // EOI is also a legal identifier part
 if (reader.bp >= reader.buflen) {
-name = names.fromChars(sbuf, 0, sp);
+name = reader.name();
 tk = tokens.lookupKind(name);
 return;
 }
 break;
 default:
 // all ASCII range chars already handled, above
 isJavaIdentifierPart = false;
 } else {
 high = reader.scanSurrogates();
 if (high != 0) {
-if (sp == sbuf.length) {
+reader.putChar(high);
-putChar(high);
-} else {
-sbuf[sp++] = high;
-}
 isJavaIdentifierPart = Character.isJavaIdentifierPart(
 Character.toCodePoint(high, reader.ch));
 } else {
 isJavaIdentifierPart = Character.isJavaIdentifierPart(reader.ch);
 }
 }
 if (!isJavaIdentifierPart) {
-name = names.fromChars(sbuf, 0, sp);
+name = reader.name();
 tk = tokens.lookupKind(name);
 return;
 }
 }
 } while (true);
 /** Read longest possible sequence of special characters and convert
 *  to token.
 */
 private void scanOperator() {
 while (true) {
-putChar(reader.ch);
+reader.putChar(false);
-Name newname = names.fromChars(sbuf, 0, sp);
+Name newname = reader.name();
 TokenKind tk1 = tokens.lookupKind(newname);
 if (tk1 == TokenKind.IDENTIFIER) {
-sp--;
+reader.sp--;
 break;
 }
 tk = tk1;
 reader.scanChar();
 if (!isSpecial(reader.ch)) break;
 }
 }
-/**
-* Scan a documentation comment; determine if a deprecated tag is present.
-* Called once the initial /, * have been skipped, positioned at the second *
-* (which is treated as the beginning of the first line).
-* Stops positioned at the closing '/'.
-*/
-@SuppressWarnings("fallthrough")
-private void scanDocComment() {
-boolean deprecatedPrefix = false;
-forEachLine:
-while (reader.bp < reader.buflen) {
-// Skip optional WhiteSpace at beginning of line
-while (reader.bp < reader.buflen && (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF)) {
-scanCommentChar();
-}
-// Skip optional consecutive Stars
-while (reader.bp < reader.buflen && reader.ch == '*') {
-scanCommentChar();
-if (reader.ch == '/') {
-return;
-}
-}
-// Skip optional WhiteSpace after Stars
-while (reader.bp < reader.buflen && (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF)) {
-scanCommentChar();
-}
-deprecatedPrefix = false;
-// At beginning of line in the JavaDoc sense.
-if (reader.bp < reader.buflen && reader.ch == '@' && !deprecatedFlag) {
-scanCommentChar();
-if (reader.bp < reader.buflen && reader.ch == 'd') {
-scanCommentChar();
-if (reader.bp < reader.buflen && reader.ch == 'e') {
-scanCommentChar();
-if (reader.bp < reader.buflen && reader.ch == 'p') {
-scanCommentChar();
-if (reader.bp < reader.buflen && reader.ch == 'r') {
-scanCommentChar();
-if (reader.bp < reader.buflen && reader.ch == 'e') {
-scanCommentChar();
-if (reader.bp < reader.buflen && reader.ch == 'c') {
-scanCommentChar();
-if (reader.bp < reader.buflen && reader.ch == 'a') {
-scanCommentChar();
-if (reader.bp < reader.buflen && reader.ch == 't') {
-scanCommentChar();
-if (reader.bp < reader.buflen && reader.ch == 'e') {
-scanCommentChar();
-if (reader.bp < reader.buflen && reader.ch == 'd') {
-deprecatedPrefix = true;
-scanCommentChar();
-}}}}}}}}}}}
-if (deprecatedPrefix && reader.bp < reader.buflen) {
-if (Character.isWhitespace(reader.ch)) {
-deprecatedFlag = true;
-} else if (reader.ch == '*') {
-scanCommentChar();
-if (reader.ch == '/') {
-deprecatedFlag = true;
-return;
-}
-}
-}
-// Skip rest of line
-while (reader.bp < reader.buflen) {
-switch (reader.ch) {
-case '*':
-scanCommentChar();
-if (reader.ch == '/') {
-return;
-}
-break;
-case CR: // (Spec 3.4)
-scanCommentChar();
-if (reader.ch != LF) {
-continue forEachLine;
-}
-/* fall through to LF case */
-case LF: // (Spec 3.4)
-scanCommentChar();
-continue forEachLine;
-default:
-scanCommentChar();
-}
-} // rest of line
-} // forEachLine
-return;
-}
 /** Read token.
 */
 public Token readToken() {
-sp = 0;
+reader.sp = 0;
 name = null;
-deprecatedFlag = false;
 radix = 0;
 int pos = 0;
 int endPos = 0;
+List<Comment> comments = null;
 try {
 loop: while (true) {
 pos = reader.bp;
 switch (reader.ch) {
 lexError(pos, "invalid.binary.number");
 } else {
 scanNumber(pos, 2);
 }
 } else {
-putChar('0');
+reader.putChar('0');
 if (reader.ch == '_') {
 int savePos = reader.bp;
 do {
 reader.scanChar();
 } while (reader.ch == '_');
 scanNumber(pos, 10);
 break loop;
 case '.':
 reader.scanChar();
 if ('0' <= reader.ch && reader.ch <= '9') {
-putChar('.');
+reader.putChar('.');
 scanFractionAndSuffix(pos);
 } else if (reader.ch == '.') {
-putChar('.'); putChar('.');
+reader.putChar('.'); reader.putChar('.', true);
-reader.scanChar();
 if (reader.ch == '.') {
 reader.scanChar();
-putChar('.');
+reader.putChar('.');
 tk = TokenKind.ELLIPSIS;
 } else {
 lexError(pos, "malformed.fp.lit");
 }
 } else {
 reader.scanChar(); tk = TokenKind.RBRACE; break loop;
 case '/':
 reader.scanChar();
 if (reader.ch == '/') {
 do {
-scanCommentChar();
+reader.scanCommentChar();
 } while (reader.ch != CR && reader.ch != LF && reader.bp < reader.buflen);
 if (reader.bp < reader.buflen) {
-processComment(pos, reader.bp, CommentStyle.LINE);
+comments = addDocReader(comments, processComment(pos, reader.bp, CommentStyle.LINE));
 }
 break;
 } else if (reader.ch == '*') {
+boolean isEmpty = false;
 reader.scanChar();
 CommentStyle style;
 if (reader.ch == '*') {
 style = CommentStyle.JAVADOC;
-scanDocComment();
+reader.scanCommentChar();
+if (reader.ch == '/') {
+isEmpty = true;
+}
 } else {
 style = CommentStyle.BLOCK;
-while (reader.bp < reader.buflen) {
+}
-if (reader.ch == '*') {
+while (!isEmpty && reader.bp < reader.buflen) {
-reader.scanChar();
+if (reader.ch == '*') {
-if (reader.ch == '/') break;
+reader.scanChar();
-} else {
+if (reader.ch == '/') break;
-scanCommentChar();
+} else {
-}
+reader.scanCommentChar();
 }
 }
 if (reader.ch == '/') {
 reader.scanChar();
-processComment(pos, reader.bp, style);
+comments = addDocReader(comments, processComment(pos, reader.bp, style));
 break;
 } else {
 lexError(pos, "unclosed.comment");
 break loop;
 }
 // all ASCII range chars already handled, above
 isJavaIdentifierStart = false;
 } else {
 char high = reader.scanSurrogates();
 if (high != 0) {
-if (sp == sbuf.length) {
+reader.putChar(high);
-putChar(high);
-} else {
-sbuf[sp++] = high;
-}
 isJavaIdentifierStart = Character.isJavaIdentifierStart(
 Character.toCodePoint(high, reader.ch));
 } else {
 isJavaIdentifierStart = Character.isJavaIdentifierStart(reader.ch);
 break loop;
 }
 }
 endPos = reader.bp;
 switch (tk.tag) {
-case DEFAULT: return new Token(tk, pos, endPos, deprecatedFlag);
+case DEFAULT: return new Token(tk, pos, endPos, comments);
-case NAMED: return new NamedToken(tk, pos, endPos, name, deprecatedFlag);
+case NAMED: return new NamedToken(tk, pos, endPos, name, comments);
-case STRING: return new StringToken(tk, pos, endPos, new String(sbuf, 0, sp), deprecatedFlag);
+case STRING: return new StringToken(tk, pos, endPos, reader.chars(), comments);
-case NUMERIC: return new NumericToken(tk, pos, endPos, new String(sbuf, 0, sp), radix, deprecatedFlag);
+case NUMERIC: return new NumericToken(tk, pos, endPos, reader.chars(), radix, comments);
 default: throw new AssertionError();
 }
 }
 finally {
 if (scannerDebug) {
 new String(reader.getRawCharacters(pos, endPos))
 + "|");
 }
 }
 }
+//where
+List<Comment> addDocReader(List<Comment> docReaders, Comment docReader) {
+return docReaders == null ?
+List.of(docReader) :
+docReaders.prepend(docReader);
+}
 /** Return the position where a lexical error occurred;
 */
 public int errPos() {
 return errPos;
 /** Set the position where a lexical error occurred;
 */
 public void errPos(int pos) {
 errPos = pos;
-}
-public enum CommentStyle {
-LINE,
-BLOCK,
-JAVADOC,
 }
 /**
 * Called when a complete comment has been scanned. pos and endPos
 * will mark the comment boundary.
 */
-protected void processComment(int pos, int endPos, CommentStyle style) {
+protected Tokens.Comment processComment(int pos, int endPos, CommentStyle style) {
 if (scannerDebug)
 System.out.println("processComment(" + pos
 + "," + endPos + "," + style + ")=|"
 + new String(reader.getRawCharacters(pos, endPos))
 + "|");
+char[] buf = reader.getRawCharacters(pos, endPos);
+return new BasicComment<UnicodeReader>(new UnicodeReader(fac, buf, buf.length), style);
 }
 /**
 * Called when a complete whitespace run has been scanned. pos and endPos
 * will mark the whitespace boundary.
 *
 * @return a LineMap */
 public Position.LineMap getLineMap() {
 return Position.makeLineMap(reader.getRawCharacters(), reader.buflen, false);
 }
+/**
+* Scan a documentation comment; determine if a deprecated tag is present.
+* Called once the initial /, * have been skipped, positioned at the second *
+* (which is treated as the beginning of the first line).
+* Stops positioned at the closing '/'.
+*/
+protected class BasicComment<U extends UnicodeReader> implements Comment {
+CommentStyle cs;
+U comment_reader;
+protected boolean deprecatedFlag = false;
+protected boolean scanned = false;
+protected BasicComment(U comment_reader, CommentStyle cs) {
+this.comment_reader = comment_reader;
+this.cs = cs;
+}
+public String getText() {
+return null;
+}
+public CommentStyle getStyle() {
+return cs;
+}
+public boolean isDeprecated() {
+if (!scanned && cs == CommentStyle.JAVADOC) {
+scanDocComment();
+}
+return deprecatedFlag;
+}
+@SuppressWarnings("fallthrough")
+protected void scanDocComment() {
+try {
+boolean deprecatedPrefix = false;
+comment_reader.bp += 3; // '/**'
+comment_reader.ch = comment_reader.buf[comment_reader.bp];
+forEachLine:
+while (comment_reader.bp < comment_reader.buflen) {
+// Skip optional WhiteSpace at beginning of line
+while (comment_reader.bp < comment_reader.buflen && (comment_reader.ch == ' ' || comment_reader.ch == '\t' || comment_reader.ch == FF)) {
+comment_reader.scanCommentChar();
+}
+// Skip optional consecutive Stars
+while (comment_reader.bp < comment_reader.buflen && comment_reader.ch == '*') {
+comment_reader.scanCommentChar();
+if (comment_reader.ch == '/') {
+return;
+}
+}
+// Skip optional WhiteSpace after Stars
+while (comment_reader.bp < comment_reader.buflen && (comment_reader.ch == ' ' || comment_reader.ch == '\t' || comment_reader.ch == FF)) {
+comment_reader.scanCommentChar();
+}
+deprecatedPrefix = false;
+// At beginning of line in the JavaDoc sense.
+if (!deprecatedFlag) {
+String deprecated = "@deprecated";
+int i = 0;
+while (comment_reader.bp < comment_reader.buflen && comment_reader.ch == deprecated.charAt(i)) {
+comment_reader.scanCommentChar();
+i++;
+if (i == deprecated.length()) {
+deprecatedPrefix = true;
+break;
+}
+}
+}
+if (deprecatedPrefix && comment_reader.bp < comment_reader.buflen) {
+if (Character.isWhitespace(comment_reader.ch)) {
+deprecatedFlag = true;
+} else if (comment_reader.ch == '*') {
+comment_reader.scanCommentChar();
+if (comment_reader.ch == '/') {
+deprecatedFlag = true;
+return;
+}
+}
+}
+// Skip rest of line
+while (comment_reader.bp < comment_reader.buflen) {
+switch (comment_reader.ch) {
+case '*':
+comment_reader.scanCommentChar();
+if (comment_reader.ch == '/') {
+return;
+}
+break;
+case CR: // (Spec 3.4)
+comment_reader.scanCommentChar();
+if (comment_reader.ch != LF) {
+continue forEachLine;
+}
+/* fall through to LF case */
+case LF: // (Spec 3.4)
+comment_reader.scanCommentChar();
+continue forEachLine;
+default:
+comment_reader.scanCommentChar();
+}
+} // rest of line
+} // forEachLine
+return;
+} finally {
+scanned = true;
+}
+}
+}
 }

Mercurial > jdk8-mips64-public > langtools / file comparison

comparison: src/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java

src/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java