jdk8-mips64-public/langtools: src/share/classes/com/sun/tools/javac/parser/Tokens.java@af8417e590f4

     1 /*

     2  * Copyright (c) 1999, 2012, Oracle and/or its affiliates. All rights reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.  Oracle designates this

     8  * particular file as subject to the "Classpath" exception as provided

     9  * by Oracle in the LICENSE file that accompanied this code.

    10  *

    11  * This code is distributed in the hope that it will be useful, but WITHOUT

    12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    14  * version 2 for more details (a copy is included in the LICENSE file that

    15  * accompanied this code).

    16  *

    17  * You should have received a copy of the GNU General Public License version

    18  * 2 along with this work; if not, write to the Free Software Foundation,

    19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    20  *

    21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    22  * or visit www.oracle.com if you need additional information or have any

    23  * questions.

    24  */

    26 package com.sun.tools.javac.parser;

    28 import java.util.Locale;

    30 import com.sun.tools.javac.api.Formattable;

    31 import com.sun.tools.javac.api.Messages;

    32 import com.sun.tools.javac.parser.Tokens.Token.Tag;

    33 import com.sun.tools.javac.util.List;

    34 import com.sun.tools.javac.util.Name;

    35 import com.sun.tools.javac.util.Context;

    36 import com.sun.tools.javac.util.Filter;

    37 import com.sun.tools.javac.util.ListBuffer;

    38 import com.sun.tools.javac.util.Names;

    40 /** A class that defines codes/utilities for Java source tokens

    41  *  returned from lexical analysis.

    42  *

    43  *  <p><b>This is NOT part of any supported API.

    44  *  If you write code that depends on this, you do so at your own risk.

    45  *  This code and its internal interfaces are subject to change or

    46  *  deletion without notice.</b>

    47  */

    48 public class Tokens {

    50     private final Names names;

    52     /**

    53      * Keyword array. Maps name indices to Token.

    54      */

    55     private final TokenKind[] key;

    57     /**  The number of the last entered keyword.

    58      */

    59     private int maxKey = 0;

    61     /** The names of all tokens.

    62      */

    63     private Name[] tokenName = new Name[TokenKind.values().length];

    65     public static final Context.Key<Tokens> tokensKey =

    66         new Context.Key<Tokens>();

    68     public static Tokens instance(Context context) {

    69         Tokens instance = context.get(tokensKey);

    70         if (instance == null)

    71             instance = new Tokens(context);

    72         return instance;

    73     }

    75     protected Tokens(Context context) {

    76         context.put(tokensKey, this);

    77         names = Names.instance(context);

    78         for (TokenKind t : TokenKind.values()) {

    79             if (t.name != null)

    80                 enterKeyword(t.name, t);

    81             else

    82                 tokenName[t.ordinal()] = null;

    83         }

    85         key = new TokenKind[maxKey+1];

    86         for (int i = 0; i <= maxKey; i++) key[i] = TokenKind.IDENTIFIER;

    87         for (TokenKind t : TokenKind.values()) {

    88             if (t.name != null)

    89             key[tokenName[t.ordinal()].getIndex()] = t;

    90         }

    91     }

    93     private void enterKeyword(String s, TokenKind token) {

    94         Name n = names.fromString(s);

    95         tokenName[token.ordinal()] = n;

    96         if (n.getIndex() > maxKey) maxKey = n.getIndex();

    97     }

    99     /**

   100      * Create a new token given a name; if the name corresponds to a token name,

   101      * a new token of the corresponding kind is returned; otherwise, an

   102      * identifier token is returned.

   103      */

   104     TokenKind lookupKind(Name name) {

   105         return (name.getIndex() > maxKey) ? TokenKind.IDENTIFIER : key[name.getIndex()];

   106     }

   108     TokenKind lookupKind(String name) {

   109         return lookupKind(names.fromString(name));

   110     }

   112     /**

   113      * This enum defines all tokens used by the javac scanner. A token is

   114      * optionally associated with a name.

   115      */

   116     public enum TokenKind implements Formattable, Filter<TokenKind> {

   117         EOF(),

   118         ERROR(),

   119         IDENTIFIER(Tag.NAMED),

   120         ABSTRACT("abstract"),

   121         ASSERT("assert", Tag.NAMED),

   122         BOOLEAN("boolean", Tag.NAMED),

   123         BREAK("break"),

   124         BYTE("byte", Tag.NAMED),

   125         CASE("case"),

   126         CATCH("catch"),

   127         CHAR("char", Tag.NAMED),

   128         CLASS("class"),

   129         CONST("const"),

   130         CONTINUE("continue"),

   131         DEFAULT("default"),

   132         DO("do"),

   133         DOUBLE("double", Tag.NAMED),

   134         ELSE("else"),

   135         ENUM("enum", Tag.NAMED),

   136         EXTENDS("extends"),

   137         FINAL("final"),

   138         FINALLY("finally"),

   139         FLOAT("float", Tag.NAMED),

   140         FOR("for"),

   141         GOTO("goto"),

   142         IF("if"),

   143         IMPLEMENTS("implements"),

   144         IMPORT("import"),

   145         INSTANCEOF("instanceof"),

   146         INT("int", Tag.NAMED),

   147         INTERFACE("interface"),

   148         LONG("long", Tag.NAMED),

   149         NATIVE("native"),

   150         NEW("new"),

   151         PACKAGE("package"),

   152         PRIVATE("private"),

   153         PROTECTED("protected"),

   154         PUBLIC("public"),

   155         RETURN("return"),

   156         SHORT("short", Tag.NAMED),

   157         STATIC("static"),

   158         STRICTFP("strictfp"),

   159         SUPER("super", Tag.NAMED),

   160         SWITCH("switch"),

   161         SYNCHRONIZED("synchronized"),

   162         THIS("this", Tag.NAMED),

   163         THROW("throw"),

   164         THROWS("throws"),

   165         TRANSIENT("transient"),

   166         TRY("try"),

   167         VOID("void", Tag.NAMED),

   168         VOLATILE("volatile"),

   169         WHILE("while"),

   170         INTLITERAL(Tag.NUMERIC),

   171         LONGLITERAL(Tag.NUMERIC),

   172         FLOATLITERAL(Tag.NUMERIC),

   173         DOUBLELITERAL(Tag.NUMERIC),

   174         CHARLITERAL(Tag.NUMERIC),

   175         STRINGLITERAL(Tag.STRING),

   176         TRUE("true", Tag.NAMED),

   177         FALSE("false", Tag.NAMED),

   178         NULL("null", Tag.NAMED),

   179         UNDERSCORE("_", Tag.NAMED),

   180         ARROW("->"),

   181         COLCOL("::"),

   182         LPAREN("("),

   183         RPAREN(")"),

   184         LBRACE("{"),

   185         RBRACE("}"),

   186         LBRACKET("["),

   187         RBRACKET("]"),

   188         SEMI(";"),

   189         COMMA(","),

   190         DOT("."),

   191         ELLIPSIS("..."),

   192         EQ("="),

   193         GT(">"),

   194         LT("<"),

   195         BANG("!"),

   196         TILDE("~"),

   197         QUES("?"),

   198         COLON(":"),

   199         EQEQ("=="),

   200         LTEQ("<="),

   201         GTEQ(">="),

   202         BANGEQ("!="),

   203         AMPAMP("&&"),

   204         BARBAR("||"),

   205         PLUSPLUS("++"),

   206         SUBSUB("--"),

   207         PLUS("+"),

   208         SUB("-"),

   209         STAR("*"),

   210         SLASH("/"),

   211         AMP("&"),

   212         BAR("|"),

   213         CARET("^"),

   214         PERCENT("%"),

   215         LTLT("<<"),

   216         GTGT(">>"),

   217         GTGTGT(">>>"),

   218         PLUSEQ("+="),

   219         SUBEQ("-="),

   220         STAREQ("*="),

   221         SLASHEQ("/="),

   222         AMPEQ("&="),

   223         BAREQ("|="),

   224         CARETEQ("^="),

   225         PERCENTEQ("%="),

   226         LTLTEQ("<<="),

   227         GTGTEQ(">>="),

   228         GTGTGTEQ(">>>="),

   229         MONKEYS_AT("@"),

   230         CUSTOM;

   232         public final String name;

   233         public final Tag tag;

   235         TokenKind() {

   236             this(null, Tag.DEFAULT);

   237         }

   239         TokenKind(String name) {

   240             this(name, Tag.DEFAULT);

   241         }

   243         TokenKind(Tag tag) {

   244             this(null, tag);

   245         }

   247         TokenKind(String name, Tag tag) {

   248             this.name = name;

   249             this.tag = tag;

   250         }

   252         public String toString() {

   253             switch (this) {

   254             case IDENTIFIER:

   255                 return "token.identifier";

   256             case CHARLITERAL:

   257                 return "token.character";

   258             case STRINGLITERAL:

   259                 return "token.string";

   260             case INTLITERAL:

   261                 return "token.integer";

   262             case LONGLITERAL:

   263                 return "token.long-integer";

   264             case FLOATLITERAL:

   265                 return "token.float";

   266             case DOUBLELITERAL:

   267                 return "token.double";

   268             case ERROR:

   269                 return "token.bad-symbol";

   270             case EOF:

   271                 return "token.end-of-input";

   272             case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN:

   273             case LBRACKET: case RBRACKET: case LBRACE: case RBRACE:

   274                 return "'" + name + "'";

   275             default:

   276                 return name;

   277             }

   278         }

   280         public String getKind() {

   281             return "Token";

   282         }

   284         public String toString(Locale locale, Messages messages) {

   285             return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString());

   286         }

   288         @Override

   289         public boolean accepts(TokenKind that) {

   290             return this == that;

   291         }

   292     }

   294     public interface Comment {

   296         enum CommentStyle {

   297             LINE,

   298             BLOCK,

   299             JAVADOC,

   300         }

   302         String getText();

   303         int getSourcePos(int index);

   304         CommentStyle getStyle();

   305         boolean isDeprecated();

   306     }

   308     /**

   309      * This is the class representing a javac token. Each token has several fields

   310      * that are set by the javac lexer (i.e. start/end position, string value, etc).

   311      */

   312     public static class Token {

   314         /** tags constants **/

   315         enum Tag {

   316             DEFAULT,

   317             NAMED,

   318             STRING,

   319             NUMERIC;

   320         }

   322         /** The token kind */

   323         public final TokenKind kind;

   325         /** The start position of this token */

   326         public final int pos;

   328         /** The end position of this token */

   329         public final int endPos;

   331         /** Comment reader associated with this token */

   332         public final List<Comment> comments;

   334         Token(TokenKind kind, int pos, int endPos, List<Comment> comments) {

   335             this.kind = kind;

   336             this.pos = pos;

   337             this.endPos = endPos;

   338             this.comments = comments;

   339             checkKind();

   340         }

   342         Token[] split(Tokens tokens) {

   343             if (kind.name.length() < 2 || kind.tag != Tag.DEFAULT) {

   344                 throw new AssertionError("Cant split" + kind);

   345             }

   347             TokenKind t1 = tokens.lookupKind(kind.name.substring(0, 1));

   348             TokenKind t2 = tokens.lookupKind(kind.name.substring(1));

   350             if (t1 == null || t2 == null) {

   351                 throw new AssertionError("Cant split - bad subtokens");

   352             }

   353             return new Token[] {

   354                 new Token(t1, pos, pos + t1.name.length(), comments),

   355                 new Token(t2, pos + t1.name.length(), endPos, null)

   356             };

   357         }

   359         protected void checkKind() {

   360             if (kind.tag != Tag.DEFAULT) {

   361                 throw new AssertionError("Bad token kind - expected " + Tag.STRING);

   362             }

   363         }

   365         public Name name() {

   366             throw new UnsupportedOperationException();

   367         }

   369         public String stringVal() {

   370             throw new UnsupportedOperationException();

   371         }

   373         public int radix() {

   374             throw new UnsupportedOperationException();

   375         }

   377         /**

   378          * Preserve classic semantics - if multiple javadocs are found on the token

   379          * the last one is returned

   380          */

   381         public Comment comment(Comment.CommentStyle style) {

   382             List<Comment> comments = getComments(Comment.CommentStyle.JAVADOC);

   383             return comments.isEmpty() ?

   384                     null :

   385                     comments.head;

   386         }

   388         /**

   389          * Preserve classic semantics - deprecated should be set if at least one

   390          * javadoc comment attached to this token contains the '@deprecated' string

   391          */

   392         public boolean deprecatedFlag() {

   393             for (Comment c : getComments(Comment.CommentStyle.JAVADOC)) {

   394                 if (c.isDeprecated()) {

   395                     return true;

   396                 }

   397             }

   398             return false;

   399         }

   401         private List<Comment> getComments(Comment.CommentStyle style) {

   402             if (comments == null) {

   403                 return List.nil();

   404             } else {

   405                 ListBuffer<Comment> buf = ListBuffer.lb();

   406                 for (Comment c : comments) {

   407                     if (c.getStyle() == style) {

   408                         buf.add(c);

   409                     }

   410                 }

   411                 return buf.toList();

   412             }

   413         }

   414     }

   416     final static class NamedToken extends Token {

   417         /** The name of this token */

   418         public final Name name;

   420         public NamedToken(TokenKind kind, int pos, int endPos, Name name, List<Comment> comments) {

   421             super(kind, pos, endPos, comments);

   422             this.name = name;

   423         }

   425         protected void checkKind() {

   426             if (kind.tag != Tag.NAMED) {

   427                 throw new AssertionError("Bad token kind - expected " + Tag.NAMED);

   428             }

   429         }

   431         @Override

   432         public Name name() {

   433             return name;

   434         }

   435     }

   437     static class StringToken extends Token {

   438         /** The string value of this token */

   439         public final String stringVal;

   441         public StringToken(TokenKind kind, int pos, int endPos, String stringVal, List<Comment> comments) {

   442             super(kind, pos, endPos, comments);

   443             this.stringVal = stringVal;

   444         }

   446         protected void checkKind() {

   447             if (kind.tag != Tag.STRING) {

   448                 throw new AssertionError("Bad token kind - expected " + Tag.STRING);

   449             }

   450         }

   452         @Override

   453         public String stringVal() {

   454             return stringVal;

   455         }

   456     }

   458     final static class NumericToken extends StringToken {

   459         /** The 'radix' value of this token */

   460         public final int radix;

   462         public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, List<Comment> comments) {

   463             super(kind, pos, endPos, stringVal, comments);

   464             this.radix = radix;

   465         }

   467         protected void checkKind() {

   468             if (kind.tag != Tag.NUMERIC) {

   469                 throw new AssertionError("Bad token kind - expected " + Tag.NUMERIC);

   470             }

   471         }

   473         @Override

   474         public int radix() {

   475             return radix;

   476         }

   477     }

   479     public static final Token DUMMY =

   480                 new Token(TokenKind.ERROR, 0, 0, null);

   481 }

Mercurial > jdk8-mips64-public > langtools / file revision

src/share/classes/com/sun/tools/javac/parser/Tokens.java@af8417e590f4

src/share/classes/com/sun/tools/javac/parser/Tokens.java