jdk8-mips64-public/langtools: src/share/classes/com/sun/tools/javac/parser/Tokens.java@56830d5cb5bb

7104201: Refactor DocCommentScanner
Summary: Add new Comment helper class to parse contents of comments in source code
Reviewed-by: jjg

     1 /*

     2  * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.  Oracle designates this

     8  * particular file as subject to the "Classpath" exception as provided

     9  * by Oracle in the LICENSE file that accompanied this code.

    10  *

    11  * This code is distributed in the hope that it will be useful, but WITHOUT

    12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    14  * version 2 for more details (a copy is included in the LICENSE file that

    15  * accompanied this code).

    16  *

    17  * You should have received a copy of the GNU General Public License version

    18  * 2 along with this work; if not, write to the Free Software Foundation,

    19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    20  *

    21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    22  * or visit www.oracle.com if you need additional information or have any

    23  * questions.

    24  */

    26 package com.sun.tools.javac.parser;

    28 import java.util.Locale;

    30 import com.sun.tools.javac.api.Formattable;

    31 import com.sun.tools.javac.api.Messages;

    32 import com.sun.tools.javac.parser.Tokens.Token.Tag;

    33 import com.sun.tools.javac.util.List;

    34 import com.sun.tools.javac.util.Name;

    35 import com.sun.tools.javac.util.Context;

    36 import com.sun.tools.javac.util.ListBuffer;

    37 import com.sun.tools.javac.util.Names;

    39 /** A class that defines codes/utilities for Java source tokens

    40  *  returned from lexical analysis.

    41  *

    42  *  <p><b>This is NOT part of any supported API.

    43  *  If you write code that depends on this, you do so at your own risk.

    44  *  This code and its internal interfaces are subject to change or

    45  *  deletion without notice.</b>

    46  */

    47 public class Tokens {

    49     private final Names names;

    51     /**

    52      * Keyword array. Maps name indices to Token.

    53      */

    54     private final TokenKind[] key;

    56     /**  The number of the last entered keyword.

    57      */

    58     private int maxKey = 0;

    60     /** The names of all tokens.

    61      */

    62     private Name[] tokenName = new Name[TokenKind.values().length];

    64     public static final Context.Key<Tokens> tokensKey =

    65         new Context.Key<Tokens>();

    67     public static Tokens instance(Context context) {

    68         Tokens instance = context.get(tokensKey);

    69         if (instance == null)

    70             instance = new Tokens(context);

    71         return instance;

    72     }

    74     protected Tokens(Context context) {

    75         context.put(tokensKey, this);

    76         names = Names.instance(context);

    78         for (TokenKind t : TokenKind.values()) {

    79             if (t.name != null)

    80                 enterKeyword(t.name, t);

    81             else

    82                 tokenName[t.ordinal()] = null;

    83         }

    85         key = new TokenKind[maxKey+1];

    86         for (int i = 0; i <= maxKey; i++) key[i] = TokenKind.IDENTIFIER;

    87         for (TokenKind t : TokenKind.values()) {

    88             if (t.name != null)

    89             key[tokenName[t.ordinal()].getIndex()] = t;

    90         }

    91     }

    93     private void enterKeyword(String s, TokenKind token) {

    94         Name n = names.fromString(s);

    95         tokenName[token.ordinal()] = n;

    96         if (n.getIndex() > maxKey) maxKey = n.getIndex();

    97     }

    99     /**

   100      * Create a new token given a name; if the name corresponds to a token name,

   101      * a new token of the corresponding kind is returned; otherwise, an

   102      * identifier token is returned.

   103      */

   104     TokenKind lookupKind(Name name) {

   105         return (name.getIndex() > maxKey) ? TokenKind.IDENTIFIER : key[name.getIndex()];

   106     }

   108     TokenKind lookupKind(String name) {

   109         return lookupKind(names.fromString(name));

   110     }

   112     /**

   113      * This enum defines all tokens used by the javac scanner. A token is

   114      * optionally associated with a name.

   115      */

   116     public enum TokenKind implements Formattable {

   117         EOF(),

   118         ERROR(),

   119         IDENTIFIER(Tag.NAMED),

   120         ABSTRACT("abstract"),

   121         ASSERT("assert", Tag.NAMED),

   122         BOOLEAN("boolean", Tag.NAMED),

   123         BREAK("break"),

   124         BYTE("byte", Tag.NAMED),

   125         CASE("case"),

   126         CATCH("catch"),

   127         CHAR("char", Tag.NAMED),

   128         CLASS("class"),

   129         CONST("const"),

   130         CONTINUE("continue"),

   131         DEFAULT("default"),

   132         DO("do"),

   133         DOUBLE("double", Tag.NAMED),

   134         ELSE("else"),

   135         ENUM("enum", Tag.NAMED),

   136         EXTENDS("extends"),

   137         FINAL("final"),

   138         FINALLY("finally"),

   139         FLOAT("float", Tag.NAMED),

   140         FOR("for"),

   141         GOTO("goto"),

   142         IF("if"),

   143         IMPLEMENTS("implements"),

   144         IMPORT("import"),

   145         INSTANCEOF("instanceof"),

   146         INT("int", Tag.NAMED),

   147         INTERFACE("interface"),

   148         LONG("long", Tag.NAMED),

   149         NATIVE("native"),

   150         NEW("new"),

   151         PACKAGE("package"),

   152         PRIVATE("private"),

   153         PROTECTED("protected"),

   154         PUBLIC("public"),

   155         RETURN("return"),

   156         SHORT("short", Tag.NAMED),

   157         STATIC("static"),

   158         STRICTFP("strictfp"),

   159         SUPER("super", Tag.NAMED),

   160         SWITCH("switch"),

   161         SYNCHRONIZED("synchronized"),

   162         THIS("this", Tag.NAMED),

   163         THROW("throw"),

   164         THROWS("throws"),

   165         TRANSIENT("transient"),

   166         TRY("try"),

   167         VOID("void", Tag.NAMED),

   168         VOLATILE("volatile"),

   169         WHILE("while"),

   170         INTLITERAL(Tag.NUMERIC),

   171         LONGLITERAL(Tag.NUMERIC),

   172         FLOATLITERAL(Tag.NUMERIC),

   173         DOUBLELITERAL(Tag.NUMERIC),

   174         CHARLITERAL(Tag.NUMERIC),

   175         STRINGLITERAL(Tag.STRING),

   176         TRUE("true", Tag.NAMED),

   177         FALSE("false", Tag.NAMED),

   178         NULL("null", Tag.NAMED),

   179         LPAREN("("),

   180         RPAREN(")"),

   181         LBRACE("{"),

   182         RBRACE("}"),

   183         LBRACKET("["),

   184         RBRACKET("]"),

   185         SEMI(";"),

   186         COMMA(","),

   187         DOT("."),

   188         ELLIPSIS("..."),

   189         EQ("="),

   190         GT(">"),

   191         LT("<"),

   192         BANG("!"),

   193         TILDE("~"),

   194         QUES("?"),

   195         COLON(":"),

   196         EQEQ("=="),

   197         LTEQ("<="),

   198         GTEQ(">="),

   199         BANGEQ("!="),

   200         AMPAMP("&&"),

   201         BARBAR("||"),

   202         PLUSPLUS("++"),

   203         SUBSUB("--"),

   204         PLUS("+"),

   205         SUB("-"),

   206         STAR("*"),

   207         SLASH("/"),

   208         AMP("&"),

   209         BAR("|"),

   210         CARET("^"),

   211         PERCENT("%"),

   212         LTLT("<<"),

   213         GTGT(">>"),

   214         GTGTGT(">>>"),

   215         PLUSEQ("+="),

   216         SUBEQ("-="),

   217         STAREQ("*="),

   218         SLASHEQ("/="),

   219         AMPEQ("&="),

   220         BAREQ("|="),

   221         CARETEQ("^="),

   222         PERCENTEQ("%="),

   223         LTLTEQ("<<="),

   224         GTGTEQ(">>="),

   225         GTGTGTEQ(">>>="),

   226         MONKEYS_AT("@"),

   227         CUSTOM;

   229         public final String name;

   230         public final Tag tag;

   232         TokenKind() {

   233             this(null, Tag.DEFAULT);

   234         }

   236         TokenKind(String name) {

   237             this(name, Tag.DEFAULT);

   238         }

   240         TokenKind(Tag tag) {

   241             this(null, tag);

   242         }

   244         TokenKind(String name, Tag tag) {

   245             this.name = name;

   246             this.tag = tag;

   247         }

   249         public String toString() {

   250             switch (this) {

   251             case IDENTIFIER:

   252                 return "token.identifier";

   253             case CHARLITERAL:

   254                 return "token.character";

   255             case STRINGLITERAL:

   256                 return "token.string";

   257             case INTLITERAL:

   258                 return "token.integer";

   259             case LONGLITERAL:

   260                 return "token.long-integer";

   261             case FLOATLITERAL:

   262                 return "token.float";

   263             case DOUBLELITERAL:

   264                 return "token.double";

   265             case ERROR:

   266                 return "token.bad-symbol";

   267             case EOF:

   268                 return "token.end-of-input";

   269             case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN:

   270             case LBRACKET: case RBRACKET: case LBRACE: case RBRACE:

   271                 return "'" + name + "'";

   272             default:

   273                 return name;

   274             }

   275         }

   277         public String getKind() {

   278             return "Token";

   279         }

   281         public String toString(Locale locale, Messages messages) {

   282             return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString());

   283         }

   284     }

   286     public interface Comment {

   288         enum CommentStyle {

   289             LINE,

   290             BLOCK,

   291             JAVADOC,

   292         }

   294         String getText();

   295         CommentStyle getStyle();

   296         boolean isDeprecated();

   297     }

   299     /**

   300      * This is the class representing a javac token. Each token has several fields

   301      * that are set by the javac lexer (i.e. start/end position, string value, etc).

   302      */

   303     public static class Token {

   305         /** tags constants **/

   306         enum Tag {

   307             DEFAULT,

   308             NAMED,

   309             STRING,

   310             NUMERIC;

   311         }

   313         /** The token kind */

   314         public final TokenKind kind;

   316         /** The start position of this token */

   317         public final int pos;

   319         /** The end position of this token */

   320         public final int endPos;

   322         /** Comment reader associated with this token */

   323         public final List<Comment> comments;

   325         Token(TokenKind kind, int pos, int endPos, List<Comment> comments) {

   326             this.kind = kind;

   327             this.pos = pos;

   328             this.endPos = endPos;

   329             this.comments = comments;

   330             checkKind();

   331         }

   333         Token[] split(Tokens tokens) {

   334             if (kind.name.length() < 2 || kind.tag != Tag.DEFAULT) {

   335                 throw new AssertionError("Cant split" + kind);

   336             }

   338             TokenKind t1 = tokens.lookupKind(kind.name.substring(0, 1));

   339             TokenKind t2 = tokens.lookupKind(kind.name.substring(1));

   341             if (t1 == null || t2 == null) {

   342                 throw new AssertionError("Cant split - bad subtokens");

   343             }

   344             return new Token[] {

   345                 new Token(t1, pos, pos + t1.name.length(), comments),

   346                 new Token(t2, pos + t1.name.length(), endPos, null)

   347             };

   348         }

   350         protected void checkKind() {

   351             if (kind.tag != Tag.DEFAULT) {

   352                 throw new AssertionError("Bad token kind - expected " + Tag.STRING);

   353             }

   354         }

   356         public Name name() {

   357             throw new UnsupportedOperationException();

   358         }

   360         public String stringVal() {

   361             throw new UnsupportedOperationException();

   362         }

   364         public int radix() {

   365             throw new UnsupportedOperationException();

   366         }

   368         /**

   369          * Preserve classic semantics - if multiple javadocs are found on the token

   370          * the last one is returned

   371          */

   372         public String comment(Comment.CommentStyle style) {

   373             List<Comment> readers = getReaders(Comment.CommentStyle.JAVADOC);

   374             return readers.isEmpty() ?

   375                     null :

   376                     readers.head.getText();

   377         }

   379         /**

   380          * Preserve classic semantics - deprecated should be set if at least one

   381          * javadoc comment attached to this token contains the '@deprecated' string

   382          */

   383         public boolean deprecatedFlag() {

   384             for (Comment r : getReaders(Comment.CommentStyle.JAVADOC)) {

   385                 if (r.isDeprecated()) {

   386                     return true;

   387                 }

   388             }

   389             return false;

   390         }

   392         private List<Comment> getReaders(Comment.CommentStyle style) {

   393             if (comments == null) {

   394                 return List.nil();

   395             } else {

   396                 ListBuffer<Comment> buf = ListBuffer.lb();

   397                 for (Comment r : comments) {

   398                     if (r.getStyle() == style) {

   399                         buf.add(r);

   400                     }

   401                 }

   402                 return buf.toList();

   403             }

   404         }

   405     }

   407     final static class NamedToken extends Token {

   408         /** The name of this token */

   409         public final Name name;

   411         public NamedToken(TokenKind kind, int pos, int endPos, Name name, List<Comment> comments) {

   412             super(kind, pos, endPos, comments);

   413             this.name = name;

   414         }

   416         protected void checkKind() {

   417             if (kind.tag != Tag.NAMED) {

   418                 throw new AssertionError("Bad token kind - expected " + Tag.NAMED);

   419             }

   420         }

   422         @Override

   423         public Name name() {

   424             return name;

   425         }

   426     }

   428     static class StringToken extends Token {

   429         /** The string value of this token */

   430         public final String stringVal;

   432         public StringToken(TokenKind kind, int pos, int endPos, String stringVal, List<Comment> comments) {

   433             super(kind, pos, endPos, comments);

   434             this.stringVal = stringVal;

   435         }

   437         protected void checkKind() {

   438             if (kind.tag != Tag.STRING) {

   439                 throw new AssertionError("Bad token kind - expected " + Tag.STRING);

   440             }

   441         }

   443         @Override

   444         public String stringVal() {

   445             return stringVal;

   446         }

   447     }

   449     final static class NumericToken extends StringToken {

   450         /** The 'radix' value of this token */

   451         public final int radix;

   453         public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, List<Comment> comments) {

   454             super(kind, pos, endPos, stringVal, comments);

   455             this.radix = radix;

   456         }

   458         protected void checkKind() {

   459             if (kind.tag != Tag.NUMERIC) {

   460                 throw new AssertionError("Bad token kind - expected " + Tag.NUMERIC);

   461             }

   462         }

   464         @Override

   465         public int radix() {

   466             return radix;

   467         }

   468     }

   470     public static final Token DUMMY =

   471                 new Token(TokenKind.ERROR, 0, 0, null);

   472 }

Mercurial > jdk8-mips64-public > langtools / file revision

src/share/classes/com/sun/tools/javac/parser/Tokens.java@56830d5cb5bb

src/share/classes/com/sun/tools/javac/parser/Tokens.java