jdk8-mips64-public/langtools: src/share/classes/com/sun/tools/javac/parser/Tokens.java@c46e0c9940d6

     1 /*

     2  * Copyright (c) 1999, 2012, Oracle and/or its affiliates. All rights reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.  Oracle designates this

     8  * particular file as subject to the "Classpath" exception as provided

     9  * by Oracle in the LICENSE file that accompanied this code.

    10  *

    11  * This code is distributed in the hope that it will be useful, but WITHOUT

    12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    14  * version 2 for more details (a copy is included in the LICENSE file that

    15  * accompanied this code).

    16  *

    17  * You should have received a copy of the GNU General Public License version

    18  * 2 along with this work; if not, write to the Free Software Foundation,

    19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    20  *

    21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    22  * or visit www.oracle.com if you need additional information or have any

    23  * questions.

    24  */

    26 package com.sun.tools.javac.parser;

    28 import java.util.Locale;

    30 import com.sun.tools.javac.api.Formattable;

    31 import com.sun.tools.javac.api.Messages;

    32 import com.sun.tools.javac.parser.Tokens.Token.Tag;

    33 import com.sun.tools.javac.util.List;

    34 import com.sun.tools.javac.util.Name;

    35 import com.sun.tools.javac.util.Context;

    36 import com.sun.tools.javac.util.ListBuffer;

    37 import com.sun.tools.javac.util.Names;

    39 /** A class that defines codes/utilities for Java source tokens

    40  *  returned from lexical analysis.

    41  *

    42  *  <p><b>This is NOT part of any supported API.

    43  *  If you write code that depends on this, you do so at your own risk.

    44  *  This code and its internal interfaces are subject to change or

    45  *  deletion without notice.</b>

    46  */

    47 public class Tokens {

    49     private final Names names;

    51     /**

    52      * Keyword array. Maps name indices to Token.

    53      */

    54     private final TokenKind[] key;

    56     /**  The number of the last entered keyword.

    57      */

    58     private int maxKey = 0;

    60     /** The names of all tokens.

    61      */

    62     private Name[] tokenName = new Name[TokenKind.values().length];

    64     public static final Context.Key<Tokens> tokensKey =

    65         new Context.Key<Tokens>();

    67     public static Tokens instance(Context context) {

    68         Tokens instance = context.get(tokensKey);

    69         if (instance == null)

    70             instance = new Tokens(context);

    71         return instance;

    72     }

    74     protected Tokens(Context context) {

    75         context.put(tokensKey, this);

    76         names = Names.instance(context);

    78         for (TokenKind t : TokenKind.values()) {

    79             if (t.name != null)

    80                 enterKeyword(t.name, t);

    81             else

    82                 tokenName[t.ordinal()] = null;

    83         }

    85         key = new TokenKind[maxKey+1];

    86         for (int i = 0; i <= maxKey; i++) key[i] = TokenKind.IDENTIFIER;

    87         for (TokenKind t : TokenKind.values()) {

    88             if (t.name != null)

    89             key[tokenName[t.ordinal()].getIndex()] = t;

    90         }

    91     }

    93     private void enterKeyword(String s, TokenKind token) {

    94         Name n = names.fromString(s);

    95         tokenName[token.ordinal()] = n;

    96         if (n.getIndex() > maxKey) maxKey = n.getIndex();

    97     }

    99     /**

   100      * Create a new token given a name; if the name corresponds to a token name,

   101      * a new token of the corresponding kind is returned; otherwise, an

   102      * identifier token is returned.

   103      */

   104     TokenKind lookupKind(Name name) {

   105         return (name.getIndex() > maxKey) ? TokenKind.IDENTIFIER : key[name.getIndex()];

   106     }

   108     TokenKind lookupKind(String name) {

   109         return lookupKind(names.fromString(name));

   110     }

   112     /**

   113      * This enum defines all tokens used by the javac scanner. A token is

   114      * optionally associated with a name.

   115      */

   116     public enum TokenKind implements Formattable {

   117         EOF(),

   118         ERROR(),

   119         IDENTIFIER(Tag.NAMED),

   120         ABSTRACT("abstract"),

   121         ASSERT("assert", Tag.NAMED),

   122         BOOLEAN("boolean", Tag.NAMED),

   123         BREAK("break"),

   124         BYTE("byte", Tag.NAMED),

   125         CASE("case"),

   126         CATCH("catch"),

   127         CHAR("char", Tag.NAMED),

   128         CLASS("class"),

   129         CONST("const"),

   130         CONTINUE("continue"),

   131         DEFAULT("default"),

   132         DO("do"),

   133         DOUBLE("double", Tag.NAMED),

   134         ELSE("else"),

   135         ENUM("enum", Tag.NAMED),

   136         EXTENDS("extends"),

   137         FINAL("final"),

   138         FINALLY("finally"),

   139         FLOAT("float", Tag.NAMED),

   140         FOR("for"),

   141         GOTO("goto"),

   142         IF("if"),

   143         IMPLEMENTS("implements"),

   144         IMPORT("import"),

   145         INSTANCEOF("instanceof"),

   146         INT("int", Tag.NAMED),

   147         INTERFACE("interface"),

   148         LONG("long", Tag.NAMED),

   149         NATIVE("native"),

   150         NEW("new"),

   151         PACKAGE("package"),

   152         PRIVATE("private"),

   153         PROTECTED("protected"),

   154         PUBLIC("public"),

   155         RETURN("return"),

   156         SHORT("short", Tag.NAMED),

   157         STATIC("static"),

   158         STRICTFP("strictfp"),

   159         SUPER("super", Tag.NAMED),

   160         SWITCH("switch"),

   161         SYNCHRONIZED("synchronized"),

   162         THIS("this", Tag.NAMED),

   163         THROW("throw"),

   164         THROWS("throws"),

   165         TRANSIENT("transient"),

   166         TRY("try"),

   167         VOID("void", Tag.NAMED),

   168         VOLATILE("volatile"),

   169         WHILE("while"),

   170         INTLITERAL(Tag.NUMERIC),

   171         LONGLITERAL(Tag.NUMERIC),

   172         FLOATLITERAL(Tag.NUMERIC),

   173         DOUBLELITERAL(Tag.NUMERIC),

   174         CHARLITERAL(Tag.NUMERIC),

   175         STRINGLITERAL(Tag.STRING),

   176         TRUE("true", Tag.NAMED),

   177         FALSE("false", Tag.NAMED),

   178         NULL("null", Tag.NAMED),

   179         ARROW("->"),

   180         COLCOL("::"),

   181         LPAREN("("),

   182         RPAREN(")"),

   183         LBRACE("{"),

   184         RBRACE("}"),

   185         LBRACKET("["),

   186         RBRACKET("]"),

   187         SEMI(";"),

   188         COMMA(","),

   189         DOT("."),

   190         ELLIPSIS("..."),

   191         EQ("="),

   192         GT(">"),

   193         LT("<"),

   194         BANG("!"),

   195         TILDE("~"),

   196         QUES("?"),

   197         COLON(":"),

   198         EQEQ("=="),

   199         LTEQ("<="),

   200         GTEQ(">="),

   201         BANGEQ("!="),

   202         AMPAMP("&&"),

   203         BARBAR("||"),

   204         PLUSPLUS("++"),

   205         SUBSUB("--"),

   206         PLUS("+"),

   207         SUB("-"),

   208         STAR("*"),

   209         SLASH("/"),

   210         AMP("&"),

   211         BAR("|"),

   212         CARET("^"),

   213         PERCENT("%"),

   214         LTLT("<<"),

   215         GTGT(">>"),

   216         GTGTGT(">>>"),

   217         PLUSEQ("+="),

   218         SUBEQ("-="),

   219         STAREQ("*="),

   220         SLASHEQ("/="),

   221         AMPEQ("&="),

   222         BAREQ("|="),

   223         CARETEQ("^="),

   224         PERCENTEQ("%="),

   225         LTLTEQ("<<="),

   226         GTGTEQ(">>="),

   227         GTGTGTEQ(">>>="),

   228         MONKEYS_AT("@"),

   229         CUSTOM;

   231         public final String name;

   232         public final Tag tag;

   234         TokenKind() {

   235             this(null, Tag.DEFAULT);

   236         }

   238         TokenKind(String name) {

   239             this(name, Tag.DEFAULT);

   240         }

   242         TokenKind(Tag tag) {

   243             this(null, tag);

   244         }

   246         TokenKind(String name, Tag tag) {

   247             this.name = name;

   248             this.tag = tag;

   249         }

   251         public String toString() {

   252             switch (this) {

   253             case IDENTIFIER:

   254                 return "token.identifier";

   255             case CHARLITERAL:

   256                 return "token.character";

   257             case STRINGLITERAL:

   258                 return "token.string";

   259             case INTLITERAL:

   260                 return "token.integer";

   261             case LONGLITERAL:

   262                 return "token.long-integer";

   263             case FLOATLITERAL:

   264                 return "token.float";

   265             case DOUBLELITERAL:

   266                 return "token.double";

   267             case ERROR:

   268                 return "token.bad-symbol";

   269             case EOF:

   270                 return "token.end-of-input";

   271             case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN:

   272             case LBRACKET: case RBRACKET: case LBRACE: case RBRACE:

   273                 return "'" + name + "'";

   274             default:

   275                 return name;

   276             }

   277         }

   279         public String getKind() {

   280             return "Token";

   281         }

   283         public String toString(Locale locale, Messages messages) {

   284             return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString());

   285         }

   286     }

   288     public interface Comment {

   290         enum CommentStyle {

   291             LINE,

   292             BLOCK,

   293             JAVADOC,

   294         }

   296         String getText();

   297         int getSourcePos(int index);

   298         CommentStyle getStyle();

   299         boolean isDeprecated();

   300     }

   302     /**

   303      * This is the class representing a javac token. Each token has several fields

   304      * that are set by the javac lexer (i.e. start/end position, string value, etc).

   305      */

   306     public static class Token {

   308         /** tags constants **/

   309         enum Tag {

   310             DEFAULT,

   311             NAMED,

   312             STRING,

   313             NUMERIC;

   314         }

   316         /** The token kind */

   317         public final TokenKind kind;

   319         /** The start position of this token */

   320         public final int pos;

   322         /** The end position of this token */

   323         public final int endPos;

   325         /** Comment reader associated with this token */

   326         public final List<Comment> comments;

   328         Token(TokenKind kind, int pos, int endPos, List<Comment> comments) {

   329             this.kind = kind;

   330             this.pos = pos;

   331             this.endPos = endPos;

   332             this.comments = comments;

   333             checkKind();

   334         }

   336         Token[] split(Tokens tokens) {

   337             if (kind.name.length() < 2 || kind.tag != Tag.DEFAULT) {

   338                 throw new AssertionError("Cant split" + kind);

   339             }

   341             TokenKind t1 = tokens.lookupKind(kind.name.substring(0, 1));

   342             TokenKind t2 = tokens.lookupKind(kind.name.substring(1));

   344             if (t1 == null || t2 == null) {

   345                 throw new AssertionError("Cant split - bad subtokens");

   346             }

   347             return new Token[] {

   348                 new Token(t1, pos, pos + t1.name.length(), comments),

   349                 new Token(t2, pos + t1.name.length(), endPos, null)

   350             };

   351         }

   353         protected void checkKind() {

   354             if (kind.tag != Tag.DEFAULT) {

   355                 throw new AssertionError("Bad token kind - expected " + Tag.STRING);

   356             }

   357         }

   359         public Name name() {

   360             throw new UnsupportedOperationException();

   361         }

   363         public String stringVal() {

   364             throw new UnsupportedOperationException();

   365         }

   367         public int radix() {

   368             throw new UnsupportedOperationException();

   369         }

   371         /**

   372          * Preserve classic semantics - if multiple javadocs are found on the token

   373          * the last one is returned

   374          */

   375         public Comment comment(Comment.CommentStyle style) {

   376             List<Comment> comments = getComments(Comment.CommentStyle.JAVADOC);

   377             return comments.isEmpty() ?

   378                     null :

   379                     comments.head;

   380         }

   382         /**

   383          * Preserve classic semantics - deprecated should be set if at least one

   384          * javadoc comment attached to this token contains the '@deprecated' string

   385          */

   386         public boolean deprecatedFlag() {

   387             for (Comment c : getComments(Comment.CommentStyle.JAVADOC)) {

   388                 if (c.isDeprecated()) {

   389                     return true;

   390                 }

   391             }

   392             return false;

   393         }

   395         private List<Comment> getComments(Comment.CommentStyle style) {

   396             if (comments == null) {

   397                 return List.nil();

   398             } else {

   399                 ListBuffer<Comment> buf = ListBuffer.lb();

   400                 for (Comment c : comments) {

   401                     if (c.getStyle() == style) {

   402                         buf.add(c);

   403                     }

   404                 }

   405                 return buf.toList();

   406             }

   407         }

   408     }

   410     final static class NamedToken extends Token {

   411         /** The name of this token */

   412         public final Name name;

   414         public NamedToken(TokenKind kind, int pos, int endPos, Name name, List<Comment> comments) {

   415             super(kind, pos, endPos, comments);

   416             this.name = name;

   417         }

   419         protected void checkKind() {

   420             if (kind.tag != Tag.NAMED) {

   421                 throw new AssertionError("Bad token kind - expected " + Tag.NAMED);

   422             }

   423         }

   425         @Override

   426         public Name name() {

   427             return name;

   428         }

   429     }

   431     static class StringToken extends Token {

   432         /** The string value of this token */

   433         public final String stringVal;

   435         public StringToken(TokenKind kind, int pos, int endPos, String stringVal, List<Comment> comments) {

   436             super(kind, pos, endPos, comments);

   437             this.stringVal = stringVal;

   438         }

   440         protected void checkKind() {

   441             if (kind.tag != Tag.STRING) {

   442                 throw new AssertionError("Bad token kind - expected " + Tag.STRING);

   443             }

   444         }

   446         @Override

   447         public String stringVal() {

   448             return stringVal;

   449         }

   450     }

   452     final static class NumericToken extends StringToken {

   453         /** The 'radix' value of this token */

   454         public final int radix;

   456         public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, List<Comment> comments) {

   457             super(kind, pos, endPos, stringVal, comments);

   458             this.radix = radix;

   459         }

   461         protected void checkKind() {

   462             if (kind.tag != Tag.NUMERIC) {

   463                 throw new AssertionError("Bad token kind - expected " + Tag.NUMERIC);

   464             }

   465         }

   467         @Override

   468         public int radix() {

   469             return radix;

   470         }

   471     }

   473     public static final Token DUMMY =

   474                 new Token(TokenKind.ERROR, 0, 0, null);

   475 }

Mercurial > jdk8-mips64-public > langtools / file revision

src/share/classes/com/sun/tools/javac/parser/Tokens.java@c46e0c9940d6

src/share/classes/com/sun/tools/javac/parser/Tokens.java