# HG changeset patch # User mcimadamore # Date 1320410200 0 # Node ID 56830d5cb5bbc8245f5046758363328f2ebd93ad # Parent 9e2eb4bc49eb6f457ebb93ecada10c3689f5c41d 7104201: Refactor DocCommentScanner Summary: Add new Comment helper class to parse contents of comments in source code Reviewed-by: jjg diff -r 9e2eb4bc49eb -r 56830d5cb5bb src/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java --- a/src/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java Tue Nov 01 15:49:45 2011 -0700 +++ b/src/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java Fri Nov 04 12:36:40 2011 +0000 @@ -25,10 +25,11 @@ package com.sun.tools.javac.parser; -import java.nio.CharBuffer; import com.sun.tools.javac.code.Source; +import com.sun.tools.javac.parser.Tokens.Comment.CommentStyle; import com.sun.tools.javac.util.*; +import java.nio.CharBuffer; import static com.sun.tools.javac.parser.Tokens.*; import static com.sun.tools.javac.util.LayoutCharacters.*; @@ -65,9 +66,6 @@ */ private final Log log; - /** The name table. */ - private final Names names; - /** The token factory. */ private final Tokens tokens; @@ -87,17 +85,11 @@ */ protected int errPos = Position.NOPOS; - /** Has a @deprecated been encountered in last doc comment? - * this needs to be reset by client. + /** The Unicode reader (low-level stream reader). */ - protected boolean deprecatedFlag = false; + protected UnicodeReader reader; - /** A character buffer for saved chars. - */ - protected char[] sbuf = new char[128]; - protected int sp; - - protected UnicodeReader reader; + protected ScannerFactory fac; private static final boolean hexFloatsWork = hexFloatsWork(); private static boolean hexFloatsWork() { @@ -129,14 +121,14 @@ } protected JavaTokenizer(ScannerFactory fac, UnicodeReader reader) { - log = fac.log; - names = fac.names; - tokens = fac.tokens; - source = fac.source; + this.fac = fac; + this.log = fac.log; + this.tokens = fac.tokens; + this.source = fac.source; this.reader = reader; - allowBinaryLiterals = source.allowBinaryLiterals(); - allowHexFloats = source.allowHexFloats(); - allowUnderscoresInLiterals = source.allowUnderscoresInLiterals(); + this.allowBinaryLiterals = source.allowBinaryLiterals(); + this.allowHexFloats = source.allowHexFloats(); + this.allowUnderscoresInLiterals = source.allowUnderscoresInLiterals(); } /** Report an error at the given position using the provided arguments. @@ -147,38 +139,13 @@ errPos = pos; } - /** Read next character in comment, skipping over double '\' characters. - */ - protected void scanCommentChar() { - reader.scanChar(); - if (reader.ch == '\\') { - if (reader.peekChar() == '\\' && !reader.isUnicode()) { - reader.skipChar(); - } else { - reader.convertUnicode(); - } - } - } - - /** Append a character to sbuf. - */ - private void putChar(char ch) { - if (sp == sbuf.length) { - char[] newsbuf = new char[sbuf.length * 2]; - System.arraycopy(sbuf, 0, newsbuf, 0, sbuf.length); - sbuf = newsbuf; - } - sbuf[sp++] = ch; - } - /** Read next character in character or string literal and copy into sbuf. */ private void scanLitChar(int pos) { if (reader.ch == '\\') { if (reader.peekChar() == '\\' && !reader.isUnicode()) { reader.skipChar(); - putChar('\\'); - reader.scanChar(); + reader.putChar('\\', true); } else { reader.scanChar(); switch (reader.ch) { @@ -195,30 +162,30 @@ reader.scanChar(); } } - putChar((char)oct); + reader.putChar((char)oct); break; case 'b': - putChar('\b'); reader.scanChar(); break; + reader.putChar('\b', true); break; case 't': - putChar('\t'); reader.scanChar(); break; + reader.putChar('\t', true); break; case 'n': - putChar('\n'); reader.scanChar(); break; + reader.putChar('\n', true); break; case 'f': - putChar('\f'); reader.scanChar(); break; + reader.putChar('\f', true); break; case 'r': - putChar('\r'); reader.scanChar(); break; + reader.putChar('\r', true); break; case '\'': - putChar('\''); reader.scanChar(); break; + reader.putChar('\'', true); break; case '\"': - putChar('\"'); reader.scanChar(); break; + reader.putChar('\"', true); break; case '\\': - putChar('\\'); reader.scanChar(); break; + reader.putChar('\\', true); break; default: lexError(reader.bp, "illegal.esc.char"); } } } else if (reader.bp != reader.buflen) { - putChar(reader.ch); reader.scanChar(); + reader.putChar(true); } } @@ -227,7 +194,7 @@ int savePos; do { if (reader.ch != '_') { - putChar(reader.ch); + reader.putChar(false); } else { if (!allowUnderscoresInLiterals) { lexError(pos, "unsupported.underscore.lit", source.name); @@ -246,12 +213,10 @@ */ private void scanHexExponentAndSuffix(int pos) { if (reader.ch == 'p' || reader.ch == 'P') { - putChar(reader.ch); - reader.scanChar(); + reader.putChar(true); skipIllegalUnderscores(); if (reader.ch == '+' || reader.ch == '-') { - putChar(reader.ch); - reader.scanChar(); + reader.putChar(true); } skipIllegalUnderscores(); if ('0' <= reader.ch && reader.ch <= '9') { @@ -268,14 +233,12 @@ lexError(pos, "malformed.fp.lit"); } if (reader.ch == 'f' || reader.ch == 'F') { - putChar(reader.ch); - reader.scanChar(); + reader.putChar(true); tk = TokenKind.FLOATLITERAL; radix = 16; } else { if (reader.ch == 'd' || reader.ch == 'D') { - putChar(reader.ch); - reader.scanChar(); + reader.putChar(true); } tk = TokenKind.DOUBLELITERAL; radix = 16; @@ -289,14 +252,12 @@ if ('0' <= reader.ch && reader.ch <= '9') { scanDigits(pos, 10); } - int sp1 = sp; + int sp1 = reader.sp; if (reader.ch == 'e' || reader.ch == 'E') { - putChar(reader.ch); - reader.scanChar(); + reader.putChar(true); skipIllegalUnderscores(); if (reader.ch == '+' || reader.ch == '-') { - putChar(reader.ch); - reader.scanChar(); + reader.putChar(true); } skipIllegalUnderscores(); if ('0' <= reader.ch && reader.ch <= '9') { @@ -304,7 +265,7 @@ return; } lexError(pos, "malformed.fp.lit"); - sp = sp1; + reader.sp = sp1; } } @@ -314,13 +275,11 @@ radix = 10; scanFraction(pos); if (reader.ch == 'f' || reader.ch == 'F') { - putChar(reader.ch); - reader.scanChar(); + reader.putChar(true); tk = TokenKind.FLOATLITERAL; } else { if (reader.ch == 'd' || reader.ch == 'D') { - putChar(reader.ch); - reader.scanChar(); + reader.putChar(true); } tk = TokenKind.DOUBLELITERAL; } @@ -331,8 +290,7 @@ private void scanHexFractionAndSuffix(int pos, boolean seendigit) { radix = 16; Assert.check(reader.ch == '.'); - putChar(reader.ch); - reader.scanChar(); + reader.putChar(true); skipIllegalUnderscores(); if (reader.digit(pos, 16) >= 0) { seendigit = true; @@ -369,8 +327,7 @@ } else if (seendigit && radix == 16 && (reader.ch == 'p' || reader.ch == 'P')) { scanHexExponentAndSuffix(pos); } else if (digitRadix == 10 && reader.ch == '.') { - putChar(reader.ch); - reader.scanChar(); + reader.putChar(true); scanFractionAndSuffix(pos); } else if (digitRadix == 10 && (reader.ch == 'e' || reader.ch == 'E' || @@ -393,10 +350,7 @@ boolean isJavaIdentifierPart; char high; do { - if (sp == sbuf.length) putChar(reader.ch); else sbuf[sp++] = reader.ch; - // optimization, was: putChar(reader.ch); - - reader.scanChar(); + reader.putChar(true); switch (reader.ch) { case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': @@ -423,7 +377,7 @@ break; case '\u001A': // EOI is also a legal identifier part if (reader.bp >= reader.buflen) { - name = names.fromChars(sbuf, 0, sp); + name = reader.name(); tk = tokens.lookupKind(name); return; } @@ -435,11 +389,7 @@ } else { high = reader.scanSurrogates(); if (high != 0) { - if (sp == sbuf.length) { - putChar(high); - } else { - sbuf[sp++] = high; - } + reader.putChar(high); isJavaIdentifierPart = Character.isJavaIdentifierPart( Character.toCodePoint(high, reader.ch)); } else { @@ -447,7 +397,7 @@ } } if (!isJavaIdentifierPart) { - name = names.fromChars(sbuf, 0, sp); + name = reader.name(); tk = tokens.lookupKind(name); return; } @@ -474,11 +424,11 @@ */ private void scanOperator() { while (true) { - putChar(reader.ch); - Name newname = names.fromChars(sbuf, 0, sp); + reader.putChar(false); + Name newname = reader.name(); TokenKind tk1 = tokens.lookupKind(newname); if (tk1 == TokenKind.IDENTIFIER) { - sp--; + reader.sp--; break; } tk = tk1; @@ -487,111 +437,17 @@ } } - /** - * Scan a documentation comment; determine if a deprecated tag is present. - * Called once the initial /, * have been skipped, positioned at the second * - * (which is treated as the beginning of the first line). - * Stops positioned at the closing '/'. - */ - @SuppressWarnings("fallthrough") - private void scanDocComment() { - boolean deprecatedPrefix = false; - - forEachLine: - while (reader.bp < reader.buflen) { - - // Skip optional WhiteSpace at beginning of line - while (reader.bp < reader.buflen && (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF)) { - scanCommentChar(); - } - - // Skip optional consecutive Stars - while (reader.bp < reader.buflen && reader.ch == '*') { - scanCommentChar(); - if (reader.ch == '/') { - return; - } - } - - // Skip optional WhiteSpace after Stars - while (reader.bp < reader.buflen && (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF)) { - scanCommentChar(); - } - - deprecatedPrefix = false; - // At beginning of line in the JavaDoc sense. - if (reader.bp < reader.buflen && reader.ch == '@' && !deprecatedFlag) { - scanCommentChar(); - if (reader.bp < reader.buflen && reader.ch == 'd') { - scanCommentChar(); - if (reader.bp < reader.buflen && reader.ch == 'e') { - scanCommentChar(); - if (reader.bp < reader.buflen && reader.ch == 'p') { - scanCommentChar(); - if (reader.bp < reader.buflen && reader.ch == 'r') { - scanCommentChar(); - if (reader.bp < reader.buflen && reader.ch == 'e') { - scanCommentChar(); - if (reader.bp < reader.buflen && reader.ch == 'c') { - scanCommentChar(); - if (reader.bp < reader.buflen && reader.ch == 'a') { - scanCommentChar(); - if (reader.bp < reader.buflen && reader.ch == 't') { - scanCommentChar(); - if (reader.bp < reader.buflen && reader.ch == 'e') { - scanCommentChar(); - if (reader.bp < reader.buflen && reader.ch == 'd') { - deprecatedPrefix = true; - scanCommentChar(); - }}}}}}}}}}} - if (deprecatedPrefix && reader.bp < reader.buflen) { - if (Character.isWhitespace(reader.ch)) { - deprecatedFlag = true; - } else if (reader.ch == '*') { - scanCommentChar(); - if (reader.ch == '/') { - deprecatedFlag = true; - return; - } - } - } - - // Skip rest of line - while (reader.bp < reader.buflen) { - switch (reader.ch) { - case '*': - scanCommentChar(); - if (reader.ch == '/') { - return; - } - break; - case CR: // (Spec 3.4) - scanCommentChar(); - if (reader.ch != LF) { - continue forEachLine; - } - /* fall through to LF case */ - case LF: // (Spec 3.4) - scanCommentChar(); - continue forEachLine; - default: - scanCommentChar(); - } - } // rest of line - } // forEachLine - return; - } - /** Read token. */ public Token readToken() { - sp = 0; + reader.sp = 0; name = null; - deprecatedFlag = false; radix = 0; + int pos = 0; int endPos = 0; + List comments = null; try { loop: while (true) { @@ -656,7 +512,7 @@ scanNumber(pos, 2); } } else { - putChar('0'); + reader.putChar('0'); if (reader.ch == '_') { int savePos = reader.bp; do { @@ -676,14 +532,13 @@ case '.': reader.scanChar(); if ('0' <= reader.ch && reader.ch <= '9') { - putChar('.'); + reader.putChar('.'); scanFractionAndSuffix(pos); } else if (reader.ch == '.') { - putChar('.'); putChar('.'); - reader.scanChar(); + reader.putChar('.'); reader.putChar('.', true); if (reader.ch == '.') { reader.scanChar(); - putChar('.'); + reader.putChar('.'); tk = TokenKind.ELLIPSIS; } else { lexError(pos, "malformed.fp.lit"); @@ -712,32 +567,36 @@ reader.scanChar(); if (reader.ch == '/') { do { - scanCommentChar(); + reader.scanCommentChar(); } while (reader.ch != CR && reader.ch != LF && reader.bp < reader.buflen); if (reader.bp < reader.buflen) { - processComment(pos, reader.bp, CommentStyle.LINE); + comments = addDocReader(comments, processComment(pos, reader.bp, CommentStyle.LINE)); } break; } else if (reader.ch == '*') { + boolean isEmpty = false; reader.scanChar(); CommentStyle style; if (reader.ch == '*') { style = CommentStyle.JAVADOC; - scanDocComment(); + reader.scanCommentChar(); + if (reader.ch == '/') { + isEmpty = true; + } } else { style = CommentStyle.BLOCK; - while (reader.bp < reader.buflen) { - if (reader.ch == '*') { - reader.scanChar(); - if (reader.ch == '/') break; - } else { - scanCommentChar(); - } + } + while (!isEmpty && reader.bp < reader.buflen) { + if (reader.ch == '*') { + reader.scanChar(); + if (reader.ch == '/') break; + } else { + reader.scanCommentChar(); } } if (reader.ch == '/') { reader.scanChar(); - processComment(pos, reader.bp, style); + comments = addDocReader(comments, processComment(pos, reader.bp, style)); break; } else { lexError(pos, "unclosed.comment"); @@ -789,11 +648,7 @@ } else { char high = reader.scanSurrogates(); if (high != 0) { - if (sp == sbuf.length) { - putChar(high); - } else { - sbuf[sp++] = high; - } + reader.putChar(high); isJavaIdentifierStart = Character.isJavaIdentifierStart( Character.toCodePoint(high, reader.ch)); @@ -816,10 +671,10 @@ } endPos = reader.bp; switch (tk.tag) { - case DEFAULT: return new Token(tk, pos, endPos, deprecatedFlag); - case NAMED: return new NamedToken(tk, pos, endPos, name, deprecatedFlag); - case STRING: return new StringToken(tk, pos, endPos, new String(sbuf, 0, sp), deprecatedFlag); - case NUMERIC: return new NumericToken(tk, pos, endPos, new String(sbuf, 0, sp), radix, deprecatedFlag); + case DEFAULT: return new Token(tk, pos, endPos, comments); + case NAMED: return new NamedToken(tk, pos, endPos, name, comments); + case STRING: return new StringToken(tk, pos, endPos, reader.chars(), comments); + case NUMERIC: return new NumericToken(tk, pos, endPos, reader.chars(), radix, comments); default: throw new AssertionError(); } } @@ -832,6 +687,12 @@ } } } + //where + List addDocReader(List docReaders, Comment docReader) { + return docReaders == null ? + List.of(docReader) : + docReaders.prepend(docReader); + } /** Return the position where a lexical error occurred; */ @@ -845,22 +706,18 @@ errPos = pos; } - public enum CommentStyle { - LINE, - BLOCK, - JAVADOC, - } - /** * Called when a complete comment has been scanned. pos and endPos * will mark the comment boundary. */ - protected void processComment(int pos, int endPos, CommentStyle style) { + protected Tokens.Comment processComment(int pos, int endPos, CommentStyle style) { if (scannerDebug) System.out.println("processComment(" + pos + "," + endPos + "," + style + ")=|" + new String(reader.getRawCharacters(pos, endPos)) + "|"); + char[] buf = reader.getRawCharacters(pos, endPos); + return new BasicComment(new UnicodeReader(fac, buf, buf.length), style); } /** @@ -893,4 +750,125 @@ public Position.LineMap getLineMap() { return Position.makeLineMap(reader.getRawCharacters(), reader.buflen, false); } + + + /** + * Scan a documentation comment; determine if a deprecated tag is present. + * Called once the initial /, * have been skipped, positioned at the second * + * (which is treated as the beginning of the first line). + * Stops positioned at the closing '/'. + */ + protected class BasicComment implements Comment { + + CommentStyle cs; + U comment_reader; + + protected boolean deprecatedFlag = false; + protected boolean scanned = false; + + protected BasicComment(U comment_reader, CommentStyle cs) { + this.comment_reader = comment_reader; + this.cs = cs; + } + + public String getText() { + return null; + } + + public CommentStyle getStyle() { + return cs; + } + + public boolean isDeprecated() { + if (!scanned && cs == CommentStyle.JAVADOC) { + scanDocComment(); + } + return deprecatedFlag; + } + + @SuppressWarnings("fallthrough") + protected void scanDocComment() { + try { + boolean deprecatedPrefix = false; + + comment_reader.bp += 3; // '/**' + comment_reader.ch = comment_reader.buf[comment_reader.bp]; + + forEachLine: + while (comment_reader.bp < comment_reader.buflen) { + + // Skip optional WhiteSpace at beginning of line + while (comment_reader.bp < comment_reader.buflen && (comment_reader.ch == ' ' || comment_reader.ch == '\t' || comment_reader.ch == FF)) { + comment_reader.scanCommentChar(); + } + + // Skip optional consecutive Stars + while (comment_reader.bp < comment_reader.buflen && comment_reader.ch == '*') { + comment_reader.scanCommentChar(); + if (comment_reader.ch == '/') { + return; + } + } + + // Skip optional WhiteSpace after Stars + while (comment_reader.bp < comment_reader.buflen && (comment_reader.ch == ' ' || comment_reader.ch == '\t' || comment_reader.ch == FF)) { + comment_reader.scanCommentChar(); + } + + deprecatedPrefix = false; + // At beginning of line in the JavaDoc sense. + if (!deprecatedFlag) { + String deprecated = "@deprecated"; + int i = 0; + while (comment_reader.bp < comment_reader.buflen && comment_reader.ch == deprecated.charAt(i)) { + comment_reader.scanCommentChar(); + i++; + if (i == deprecated.length()) { + deprecatedPrefix = true; + break; + } + } + } + + if (deprecatedPrefix && comment_reader.bp < comment_reader.buflen) { + if (Character.isWhitespace(comment_reader.ch)) { + deprecatedFlag = true; + } else if (comment_reader.ch == '*') { + comment_reader.scanCommentChar(); + if (comment_reader.ch == '/') { + deprecatedFlag = true; + return; + } + } + } + + // Skip rest of line + while (comment_reader.bp < comment_reader.buflen) { + switch (comment_reader.ch) { + case '*': + comment_reader.scanCommentChar(); + if (comment_reader.ch == '/') { + return; + } + break; + case CR: // (Spec 3.4) + comment_reader.scanCommentChar(); + if (comment_reader.ch != LF) { + continue forEachLine; + } + /* fall through to LF case */ + case LF: // (Spec 3.4) + comment_reader.scanCommentChar(); + continue forEachLine; + default: + comment_reader.scanCommentChar(); + } + } // rest of line + } // forEachLine + return; + } finally { + scanned = true; + } + } + } } diff -r 9e2eb4bc49eb -r 56830d5cb5bb src/share/classes/com/sun/tools/javac/parser/JavacParser.java --- a/src/share/classes/com/sun/tools/javac/parser/JavacParser.java Tue Nov 01 15:49:45 2011 -0700 +++ b/src/share/classes/com/sun/tools/javac/parser/JavacParser.java Fri Nov 04 12:36:40 2011 +0000 @@ -29,6 +29,7 @@ import com.sun.tools.javac.code.*; import com.sun.tools.javac.parser.Tokens.*; +import com.sun.tools.javac.parser.Tokens.Comment.CommentStyle; import com.sun.tools.javac.tree.*; import com.sun.tools.javac.tree.JCTree.*; import com.sun.tools.javac.util.*; @@ -1584,7 +1585,7 @@ break; case MONKEYS_AT: case FINAL: { - String dc = token.docComment; + String dc = token.comment(CommentStyle.JAVADOC); JCModifiers mods = modifiersOpt(); if (token.kind == INTERFACE || token.kind == CLASS || @@ -1601,21 +1602,21 @@ break; } case ABSTRACT: case STRICTFP: { - String dc = token.docComment; + String dc = token.comment(CommentStyle.JAVADOC); JCModifiers mods = modifiersOpt(); stats.append(classOrInterfaceOrEnumDeclaration(mods, dc)); break; } case INTERFACE: case CLASS: - String dc = token.docComment; + String dc = token.comment(CommentStyle.JAVADOC); stats.append(classOrInterfaceOrEnumDeclaration(modifiersOpt(), dc)); break; case ENUM: case ASSERT: if (allowEnums && token.kind == ENUM) { error(token.pos, "local.enum"); - dc = token.docComment; + dc = token.comment(CommentStyle.JAVADOC); stats.append(classOrInterfaceOrEnumDeclaration(modifiersOpt(), dc)); break; } else if (allowAsserts && token.kind == ASSERT) { @@ -1991,7 +1992,7 @@ annotations.appendList(partial.annotations); pos = partial.pos; } - if (token.deprecatedFlag) { + if (token.deprecatedFlag()) { flags |= Flags.DEPRECATED; } int lastPos = Position.NOPOS; @@ -2271,9 +2272,9 @@ seenImport = true; defs.append(importDeclaration()); } else { - String docComment = token.docComment; + String docComment = token.comment(CommentStyle.JAVADOC); if (firstTypeDecl && !seenImport && !seenPackage) { - docComment = firstToken.docComment; + docComment = firstToken.comment(CommentStyle.JAVADOC); consumedToplevelDoc = true; } JCTree def = typeDeclaration(mods, docComment); @@ -2288,7 +2289,7 @@ } JCTree.JCCompilationUnit toplevel = F.at(firstToken.pos).TopLevel(packageAnnotations, pid, defs.toList()); if (!consumedToplevelDoc) - attach(toplevel, firstToken.docComment); + attach(toplevel, firstToken.comment(CommentStyle.JAVADOC)); if (defs.elems.isEmpty()) storeEnd(toplevel, S.prevToken().endPos); if (keepDocComments) @@ -2498,9 +2499,9 @@ /** EnumeratorDeclaration = AnnotationsOpt [TypeArguments] IDENTIFIER [ Arguments ] [ "{" ClassBody "}" ] */ JCTree enumeratorDeclaration(Name enumName) { - String dc = token.docComment; + String dc = token.comment(CommentStyle.JAVADOC); int flags = Flags.PUBLIC|Flags.STATIC|Flags.FINAL|Flags.ENUM; - if (token.deprecatedFlag) { + if (token.deprecatedFlag()) { flags |= Flags.DEPRECATED; } int pos = token.pos; @@ -2587,7 +2588,7 @@ nextToken(); return List.nil(); } else { - String dc = token.docComment; + String dc = token.comment(CommentStyle.JAVADOC); int pos = token.pos; JCModifiers mods = modifiersOpt(); if (token.kind == CLASS || diff -r 9e2eb4bc49eb -r 56830d5cb5bb src/share/classes/com/sun/tools/javac/parser/JavadocTokenizer.java --- a/src/share/classes/com/sun/tools/javac/parser/JavadocTokenizer.java Tue Nov 01 15:49:45 2011 -0700 +++ b/src/share/classes/com/sun/tools/javac/parser/JavadocTokenizer.java Fri Nov 04 12:36:40 2011 +0000 @@ -25,8 +25,8 @@ package com.sun.tools.javac.parser; -import com.sun.tools.javac.file.JavacFileManager; -import com.sun.tools.javac.parser.Tokens.Token; +import com.sun.tools.javac.parser.Tokens.Comment; +import com.sun.tools.javac.parser.Tokens.Comment.CommentStyle; import com.sun.tools.javac.util.*; import java.nio.*; @@ -59,352 +59,295 @@ super(fac, input, inputLength); } - /** The comment input buffer, index of next chacter to be read, - * index of one past last character in buffer. - */ - private char[] buf; - private int bp; - private int buflen; - - /** The current character. - */ - private char ch; - - /** The column number position of the current character. - */ - private int col; - - /** The buffer index of the last converted Unicode character - */ - private int unicodeConversionBp = 0; + @Override + protected Comment processComment(int pos, int endPos, CommentStyle style) { + char[] buf = reader.getRawCharacters(pos, endPos); + return new JavadocComment(new ColReader(fac, buf, buf.length), style); + } /** - * Buffer for doc comment. + * This is a specialized version of UnicodeReader that keeps track of the + * column position within a given character stream (used for Javadoc processing). */ - private char[] docCommentBuffer = new char[1024]; + static class ColReader extends UnicodeReader { - /** - * Number of characters in doc comment buffer. - */ - private int docCommentCount; + int col; - /** - * Translated and stripped contents of doc comment - */ - private String docComment = null; + ColReader(ScannerFactory fac, char[] input, int inputLength) { + super(fac, input, inputLength); + } + @Override + protected void convertUnicode() { + if (ch == '\\' && unicodeConversionBp != bp) { + bp++; ch = buf[bp]; col++; + if (ch == 'u') { + do { + bp++; ch = buf[bp]; col++; + } while (ch == 'u'); + int limit = bp + 3; + if (limit < buflen) { + int d = digit(bp, 16); + int code = d; + while (bp < limit && d >= 0) { + bp++; ch = buf[bp]; col++; + d = digit(bp, 16); + code = (code << 4) + d; + } + if (d >= 0) { + ch = (char)code; + unicodeConversionBp = bp; + return; + } + } + // "illegal.Unicode.esc", reported by base scanner + } else { + bp--; + ch = '\\'; + col--; + } + } + } - /** Unconditionally expand the comment buffer. - */ - private void expandCommentBuffer() { - char[] newBuffer = new char[docCommentBuffer.length * 2]; - System.arraycopy(docCommentBuffer, 0, newBuffer, - 0, docCommentBuffer.length); - docCommentBuffer = newBuffer; - } + @Override + protected void scanCommentChar() { + scanChar(); + if (ch == '\\') { + if (peekChar() == '\\' && !isUnicode()) { + putChar(ch, false); + bp++; col++; + } else { + convertUnicode(); + } + } + } - /** Convert an ASCII digit from its base (8, 10, or 16) - * to its value. - */ - private int digit(int base) { - char c = ch; - int result = Character.digit(c, base); - if (result >= 0 && c > 0x7f) { - ch = "0123456789abcdef".charAt(result); + @Override + protected void scanChar() { + bp++; + ch = buf[bp]; + switch (ch) { + case '\r': // return + col = 0; + break; + case '\n': // newline + if (bp == 0 || buf[bp-1] != '\r') { + col = 0; + } + break; + case '\t': // tab + col = (col / TabInc * TabInc) + TabInc; + break; + case '\\': // possible Unicode + col++; + convertUnicode(); + break; + default: + col++; + break; + } + } + } + + protected class JavadocComment extends JavaTokenizer.BasicComment { + + /** + * Translated and stripped contents of doc comment + */ + private String docComment = null; + + JavadocComment(ColReader comment_reader, CommentStyle cs) { + super(comment_reader, cs); } - return result; - } - /** Convert Unicode escape; bp points to initial '\' character - * (Spec 3.3). - */ - private void convertUnicode() { - if (ch == '\\' && unicodeConversionBp != bp) { - bp++; ch = buf[bp]; col++; - if (ch == 'u') { - do { - bp++; ch = buf[bp]; col++; - } while (ch == 'u'); - int limit = bp + 3; - if (limit < buflen) { - int d = digit(16); - int code = d; - while (bp < limit && d >= 0) { - bp++; ch = buf[bp]; col++; - d = digit(16); - code = (code << 4) + d; - } - if (d >= 0) { - ch = (char)code; - unicodeConversionBp = bp; - return; - } + public String getText() { + if (!scanned && cs == CommentStyle.JAVADOC) { + scanDocComment(); + } + return docComment; + } + + @Override + @SuppressWarnings("fallthrough") + protected void scanDocComment() { + try { + boolean firstLine = true; + + // Skip over first slash + comment_reader.scanCommentChar(); + // Skip over first star + comment_reader.scanCommentChar(); + + // consume any number of stars + while (comment_reader.bp < comment_reader.buflen && comment_reader.ch == '*') { + comment_reader.scanCommentChar(); + } + // is the comment in the form /**/, /***/, /****/, etc. ? + if (comment_reader.bp < comment_reader.buflen && comment_reader.ch == '/') { + docComment = ""; + return; + } + + // skip a newline on the first line of the comment. + if (comment_reader.bp < comment_reader.buflen) { + if (comment_reader.ch == LF) { + comment_reader.scanCommentChar(); + firstLine = false; + } else if (comment_reader.ch == CR) { + comment_reader.scanCommentChar(); + if (comment_reader.ch == LF) { + comment_reader.scanCommentChar(); + firstLine = false; + } + } + } + + outerLoop: + + // The outerLoop processes the doc comment, looping once + // for each line. For each line, it first strips off + // whitespace, then it consumes any stars, then it + // puts the rest of the line into our buffer. + while (comment_reader.bp < comment_reader.buflen) { + + // The wsLoop consumes whitespace from the beginning + // of each line. + wsLoop: + + while (comment_reader.bp < comment_reader.buflen) { + switch(comment_reader.ch) { + case ' ': + comment_reader.scanCommentChar(); + break; + case '\t': + comment_reader.col = ((comment_reader.col - 1) / TabInc * TabInc) + TabInc; + comment_reader.scanCommentChar(); + break; + case FF: + comment_reader.col = 0; + comment_reader.scanCommentChar(); + break; + // Treat newline at beginning of line (blank line, no star) + // as comment text. Old Javadoc compatibility requires this. + /*---------------------------------* + case CR: // (Spec 3.4) + doc_reader.scanCommentChar(); + if (ch == LF) { + col = 0; + doc_reader.scanCommentChar(); + } + break; + case LF: // (Spec 3.4) + doc_reader.scanCommentChar(); + break; + *---------------------------------*/ + default: + // we've seen something that isn't whitespace; + // jump out. + break wsLoop; + } + } + + // Are there stars here? If so, consume them all + // and check for the end of comment. + if (comment_reader.ch == '*') { + // skip all of the stars + do { + comment_reader.scanCommentChar(); + } while (comment_reader.ch == '*'); + + // check for the closing slash. + if (comment_reader.ch == '/') { + // We're done with the doc comment + // scanChar() and breakout. + break outerLoop; + } + } else if (! firstLine) { + //The current line does not begin with a '*' so we will indent it. + for (int i = 1; i < comment_reader.col; i++) { + comment_reader.putChar(' ', false); + } + } + // The textLoop processes the rest of the characters + // on the line, adding them to our buffer. + textLoop: + while (comment_reader.bp < comment_reader.buflen) { + switch (comment_reader.ch) { + case '*': + // Is this just a star? Or is this the + // end of a comment? + comment_reader.scanCommentChar(); + if (comment_reader.ch == '/') { + // This is the end of the comment, + // set ch and return our buffer. + break outerLoop; + } + // This is just an ordinary star. Add it to + // the buffer. + comment_reader.putChar('*', false); + break; + case ' ': + case '\t': + comment_reader.putChar(comment_reader.ch, false); + comment_reader.scanCommentChar(); + break; + case FF: + comment_reader.scanCommentChar(); + break textLoop; // treat as end of line + case CR: // (Spec 3.4) + comment_reader.scanCommentChar(); + if (comment_reader.ch != LF) { + // Canonicalize CR-only line terminator to LF + comment_reader.putChar((char)LF, false); + break textLoop; + } + /* fall through to LF case */ + case LF: // (Spec 3.4) + // We've seen a newline. Add it to our + // buffer and break out of this loop, + // starting fresh on a new line. + comment_reader.putChar(comment_reader.ch, false); + comment_reader.scanCommentChar(); + break textLoop; + default: + // Add the character to our buffer. + comment_reader.putChar(comment_reader.ch, false); + comment_reader.scanCommentChar(); + } + } // end textLoop + firstLine = false; + } // end outerLoop + + if (comment_reader.sp > 0) { + int i = comment_reader.sp - 1; + trailLoop: + while (i > -1) { + switch (comment_reader.sbuf[i]) { + case '*': + i--; + break; + default: + break trailLoop; + } + } + comment_reader.sp = i + 1; + + // Store the text of the doc comment + docComment = comment_reader.chars(); + } else { + docComment = ""; } - // "illegal.Unicode.esc", reported by base scanner - } else { - bp--; - ch = '\\'; - col--; + } finally { + scanned = true; + if (docComment != null && + docComment.matches("(?sm).*^\\s*@deprecated( |$).*")) { + deprecatedFlag = true; + } } } } - - /** Read next character. - */ - private void scanChar() { - bp++; - ch = buf[bp]; - switch (ch) { - case '\r': // return - col = 0; - break; - case '\n': // newline - if (bp == 0 || buf[bp-1] != '\r') { - col = 0; - } - break; - case '\t': // tab - col = (col / TabInc * TabInc) + TabInc; - break; - case '\\': // possible Unicode - col++; - convertUnicode(); - break; - default: - col++; - break; - } - } - @Override - public Token readToken() { - docComment = null; - Token tk = super.readToken(); - tk.docComment = docComment; - return tk; - } - - /** - * Read next character in doc comment, skipping over double '\' characters. - * If a double '\' is skipped, put in the buffer and update buffer count. - */ - private void scanDocCommentChar() { - scanChar(); - if (ch == '\\') { - if (buf[bp+1] == '\\' && unicodeConversionBp != bp) { - if (docCommentCount == docCommentBuffer.length) - expandCommentBuffer(); - docCommentBuffer[docCommentCount++] = ch; - bp++; col++; - } else { - convertUnicode(); - } - } - } - - /** - * Process a doc comment and make the string content available. - * Strips leading whitespace and stars. - */ - @SuppressWarnings("fallthrough") - protected void processComment(int pos, int endPos, CommentStyle style) { - if (style != CommentStyle.JAVADOC) { - return; - } - - buf = reader.getRawCharacters(pos, endPos); - buflen = buf.length; - bp = 0; - col = 0; - - docCommentCount = 0; - - boolean firstLine = true; - - // Skip over first slash - scanDocCommentChar(); - // Skip over first star - scanDocCommentChar(); - - // consume any number of stars - while (bp < buflen && ch == '*') { - scanDocCommentChar(); - } - // is the comment in the form /**/, /***/, /****/, etc. ? - if (bp < buflen && ch == '/') { - docComment = ""; - return; - } - - // skip a newline on the first line of the comment. - if (bp < buflen) { - if (ch == LF) { - scanDocCommentChar(); - firstLine = false; - } else if (ch == CR) { - scanDocCommentChar(); - if (ch == LF) { - scanDocCommentChar(); - firstLine = false; - } - } - } - - outerLoop: - - // The outerLoop processes the doc comment, looping once - // for each line. For each line, it first strips off - // whitespace, then it consumes any stars, then it - // puts the rest of the line into our buffer. - while (bp < buflen) { - - // The wsLoop consumes whitespace from the beginning - // of each line. - wsLoop: - - while (bp < buflen) { - switch(ch) { - case ' ': - scanDocCommentChar(); - break; - case '\t': - col = ((col - 1) / TabInc * TabInc) + TabInc; - scanDocCommentChar(); - break; - case FF: - col = 0; - scanDocCommentChar(); - break; -// Treat newline at beginning of line (blank line, no star) -// as comment text. Old Javadoc compatibility requires this. -/*---------------------------------* - case CR: // (Spec 3.4) - scanDocCommentChar(); - if (ch == LF) { - col = 0; - scanDocCommentChar(); - } - break; - case LF: // (Spec 3.4) - scanDocCommentChar(); - break; -*---------------------------------*/ - default: - // we've seen something that isn't whitespace; - // jump out. - break wsLoop; - } - } - - // Are there stars here? If so, consume them all - // and check for the end of comment. - if (ch == '*') { - // skip all of the stars - do { - scanDocCommentChar(); - } while (ch == '*'); - - // check for the closing slash. - if (ch == '/') { - // We're done with the doc comment - // scanChar() and breakout. - break outerLoop; - } - } else if (! firstLine) { - //The current line does not begin with a '*' so we will indent it. - for (int i = 1; i < col; i++) { - if (docCommentCount == docCommentBuffer.length) - expandCommentBuffer(); - docCommentBuffer[docCommentCount++] = ' '; - } - } - - // The textLoop processes the rest of the characters - // on the line, adding them to our buffer. - textLoop: - while (bp < buflen) { - switch (ch) { - case '*': - // Is this just a star? Or is this the - // end of a comment? - scanDocCommentChar(); - if (ch == '/') { - // This is the end of the comment, - // set ch and return our buffer. - break outerLoop; - } - // This is just an ordinary star. Add it to - // the buffer. - if (docCommentCount == docCommentBuffer.length) - expandCommentBuffer(); - docCommentBuffer[docCommentCount++] = '*'; - break; - case ' ': - case '\t': - if (docCommentCount == docCommentBuffer.length) - expandCommentBuffer(); - docCommentBuffer[docCommentCount++] = ch; - scanDocCommentChar(); - break; - case FF: - scanDocCommentChar(); - break textLoop; // treat as end of line - case CR: // (Spec 3.4) - scanDocCommentChar(); - if (ch != LF) { - // Canonicalize CR-only line terminator to LF - if (docCommentCount == docCommentBuffer.length) - expandCommentBuffer(); - docCommentBuffer[docCommentCount++] = (char)LF; - break textLoop; - } - /* fall through to LF case */ - case LF: // (Spec 3.4) - // We've seen a newline. Add it to our - // buffer and break out of this loop, - // starting fresh on a new line. - if (docCommentCount == docCommentBuffer.length) - expandCommentBuffer(); - docCommentBuffer[docCommentCount++] = ch; - scanDocCommentChar(); - break textLoop; - default: - // Add the character to our buffer. - if (docCommentCount == docCommentBuffer.length) - expandCommentBuffer(); - docCommentBuffer[docCommentCount++] = ch; - scanDocCommentChar(); - } - } // end textLoop - firstLine = false; - } // end outerLoop - - if (docCommentCount > 0) { - int i = docCommentCount - 1; - trailLoop: - while (i > -1) { - switch (docCommentBuffer[i]) { - case '*': - i--; - break; - default: - break trailLoop; - } - } - docCommentCount = i + 1; - - // Store the text of the doc comment - docComment = new String(docCommentBuffer, 0 , docCommentCount); - } else { - docComment = ""; - } - } - - /** Build a map for translating between line numbers and - * positions in the input. - * - * @return a LineMap */ public Position.LineMap getLineMap() { char[] buf = reader.getRawCharacters(); return Position.makeLineMap(buf, buf.length, true); diff -r 9e2eb4bc49eb -r 56830d5cb5bb src/share/classes/com/sun/tools/javac/parser/Tokens.java --- a/src/share/classes/com/sun/tools/javac/parser/Tokens.java Tue Nov 01 15:49:45 2011 -0700 +++ b/src/share/classes/com/sun/tools/javac/parser/Tokens.java Fri Nov 04 12:36:40 2011 +0000 @@ -30,8 +30,10 @@ import com.sun.tools.javac.api.Formattable; import com.sun.tools.javac.api.Messages; import com.sun.tools.javac.parser.Tokens.Token.Tag; +import com.sun.tools.javac.util.List; import com.sun.tools.javac.util.Name; import com.sun.tools.javac.util.Context; +import com.sun.tools.javac.util.ListBuffer; import com.sun.tools.javac.util.Names; /** A class that defines codes/utilities for Java source tokens @@ -281,6 +283,19 @@ } } + public interface Comment { + + enum CommentStyle { + LINE, + BLOCK, + JAVADOC, + } + + String getText(); + CommentStyle getStyle(); + boolean isDeprecated(); + } + /** * This is the class representing a javac token. Each token has several fields * that are set by the javac lexer (i.e. start/end position, string value, etc). @@ -304,18 +319,14 @@ /** The end position of this token */ public final int endPos; - /** Is this token preceeded by a deprecated comment? */ - public final boolean deprecatedFlag; + /** Comment reader associated with this token */ + public final List comments; - /** Is this token preceeded by a deprecated comment? */ - public String docComment; - - Token(TokenKind kind, int pos, int endPos, - boolean deprecatedFlag) { + Token(TokenKind kind, int pos, int endPos, List comments) { this.kind = kind; this.pos = pos; this.endPos = endPos; - this.deprecatedFlag = deprecatedFlag; + this.comments = comments; checkKind(); } @@ -331,8 +342,8 @@ throw new AssertionError("Cant split - bad subtokens"); } return new Token[] { - new Token(t1, pos, pos + t1.name.length(), deprecatedFlag), - new Token(t2, pos + t1.name.length(), endPos, false) + new Token(t1, pos, pos + t1.name.length(), comments), + new Token(t2, pos + t1.name.length(), endPos, null) }; } @@ -353,14 +364,52 @@ public int radix() { throw new UnsupportedOperationException(); } + + /** + * Preserve classic semantics - if multiple javadocs are found on the token + * the last one is returned + */ + public String comment(Comment.CommentStyle style) { + List readers = getReaders(Comment.CommentStyle.JAVADOC); + return readers.isEmpty() ? + null : + readers.head.getText(); + } + + /** + * Preserve classic semantics - deprecated should be set if at least one + * javadoc comment attached to this token contains the '@deprecated' string + */ + public boolean deprecatedFlag() { + for (Comment r : getReaders(Comment.CommentStyle.JAVADOC)) { + if (r.isDeprecated()) { + return true; + } + } + return false; + } + + private List getReaders(Comment.CommentStyle style) { + if (comments == null) { + return List.nil(); + } else { + ListBuffer buf = ListBuffer.lb(); + for (Comment r : comments) { + if (r.getStyle() == style) { + buf.add(r); + } + } + return buf.toList(); + } + } } final static class NamedToken extends Token { /** The name of this token */ public final Name name; - public NamedToken(TokenKind kind, int pos, int endPos, Name name, boolean deprecatedFlag) { - super(kind, pos, endPos, deprecatedFlag); + public NamedToken(TokenKind kind, int pos, int endPos, Name name, List comments) { + super(kind, pos, endPos, comments); this.name = name; } @@ -380,8 +429,8 @@ /** The string value of this token */ public final String stringVal; - public StringToken(TokenKind kind, int pos, int endPos, String stringVal, boolean deprecatedFlag) { - super(kind, pos, endPos, deprecatedFlag); + public StringToken(TokenKind kind, int pos, int endPos, String stringVal, List comments) { + super(kind, pos, endPos, comments); this.stringVal = stringVal; } @@ -401,8 +450,8 @@ /** The 'radix' value of this token */ public final int radix; - public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, boolean deprecatedFlag) { - super(kind, pos, endPos, stringVal, deprecatedFlag); + public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, List comments) { + super(kind, pos, endPos, stringVal, comments); this.radix = radix; } @@ -419,5 +468,5 @@ } public static final Token DUMMY = - new Token(TokenKind.ERROR, 0, 0, false); + new Token(TokenKind.ERROR, 0, 0, null); } diff -r 9e2eb4bc49eb -r 56830d5cb5bb src/share/classes/com/sun/tools/javac/parser/UnicodeReader.java --- a/src/share/classes/com/sun/tools/javac/parser/UnicodeReader.java Tue Nov 01 15:49:45 2011 -0700 +++ b/src/share/classes/com/sun/tools/javac/parser/UnicodeReader.java Fri Nov 04 12:36:40 2011 +0000 @@ -26,8 +26,12 @@ package com.sun.tools.javac.parser; import com.sun.tools.javac.file.JavacFileManager; +import com.sun.tools.javac.util.Log; +import com.sun.tools.javac.util.Name; +import com.sun.tools.javac.util.Names; + import java.nio.CharBuffer; -import com.sun.tools.javac.util.Log; + import static com.sun.tools.javac.util.LayoutCharacters.*; /** The char reader used by the javac lexer/tokenizer. Returns the sequence of @@ -58,6 +62,12 @@ protected int unicodeConversionBp = -1; protected Log log; + protected Names names; + + /** A character buffer for saved chars. + */ + protected char[] sbuf = new char[128]; + protected int sp; /** * Create a scanner from the input array. This method might @@ -76,6 +86,7 @@ protected UnicodeReader(ScannerFactory sf, char[] input, int inputLength) { log = sf.log; + names = sf.names; if (inputLength == input.length) { if (input.length > 0 && Character.isWhitespace(input[input.length - 1])) { inputLength--; @@ -103,6 +114,48 @@ } } + /** Read next character in comment, skipping over double '\' characters. + */ + protected void scanCommentChar() { + scanChar(); + if (ch == '\\') { + if (peekChar() == '\\' && !isUnicode()) { + skipChar(); + } else { + convertUnicode(); + } + } + } + + /** Append a character to sbuf. + */ + protected void putChar(char ch, boolean scan) { + if (sp == sbuf.length) { + char[] newsbuf = new char[sbuf.length * 2]; + System.arraycopy(sbuf, 0, newsbuf, 0, sbuf.length); + sbuf = newsbuf; + } + sbuf[sp++] = ch; + if (scan) + scanChar(); + } + + protected void putChar(char ch) { + putChar(ch, false); + } + + protected void putChar(boolean scan) { + putChar(ch, scan); + } + + Name name() { + return names.fromChars(sbuf, 0, sp); + } + + String chars() { + return new String(sbuf, 0, sp); + } + /** Convert unicode escape; bp points to initial '\' character * (Spec 3.3). */ diff -r 9e2eb4bc49eb -r 56830d5cb5bb test/tools/javac/depDocComment/DeprecatedDocComment4.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/tools/javac/depDocComment/DeprecatedDocComment4.java Fri Nov 04 12:36:40 2011 +0000 @@ -0,0 +1,20 @@ +/** + * @test /nodynamiccopyright/ + * @bug 7104201 + * @summary Refactor DocCommentScanner + * @compile/fail/ref=DeprecatedDocComment4.out -XDrawDiagnostics -Werror -Xlint:dep-ann DeprecatedDocComment4.java + */ + +class DeprecatedDocComment4 { + /** @deprecated **/ + /* block */ + void test1() {}; + + /** @deprecated **/ + /** double javadoc */ + void test2() {}; + + /** @deprecated **/ + //line comment + void test3() {}; +} diff -r 9e2eb4bc49eb -r 56830d5cb5bb test/tools/javac/depDocComment/DeprecatedDocComment4.out --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/tools/javac/depDocComment/DeprecatedDocComment4.out Fri Nov 04 12:36:40 2011 +0000 @@ -0,0 +1,6 @@ +DeprecatedDocComment4.java:11:10: compiler.warn.missing.deprecated.annotation +DeprecatedDocComment4.java:15:10: compiler.warn.missing.deprecated.annotation +DeprecatedDocComment4.java:19:10: compiler.warn.missing.deprecated.annotation +- compiler.err.warnings.and.werror +1 error +3 warnings