aoqi@0: /*
aoqi@0:  * Copyright (c) 1999, 2013, Oracle and/or its affiliates. All rights reserved.
aoqi@0:  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
aoqi@0:  *
aoqi@0:  * This code is free software; you can redistribute it and/or modify it
aoqi@0:  * under the terms of the GNU General Public License version 2 only, as
aoqi@0:  * published by the Free Software Foundation.  Oracle designates this
aoqi@0:  * particular file as subject to the "Classpath" exception as provided
aoqi@0:  * by Oracle in the LICENSE file that accompanied this code.
aoqi@0:  *
aoqi@0:  * This code is distributed in the hope that it will be useful, but WITHOUT
aoqi@0:  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
aoqi@0:  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
aoqi@0:  * version 2 for more details (a copy is included in the LICENSE file that
aoqi@0:  * accompanied this code).
aoqi@0:  *
aoqi@0:  * You should have received a copy of the GNU General Public License version
aoqi@0:  * 2 along with this work; if not, write to the Free Software Foundation,
aoqi@0:  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
aoqi@0:  *
aoqi@0:  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
aoqi@0:  * or visit www.oracle.com if you need additional information or have any
aoqi@0:  * questions.
aoqi@0:  */
aoqi@0: 
aoqi@0: package com.sun.tools.javac.parser;
aoqi@0: 
aoqi@0: import java.util.Locale;
aoqi@0: 
aoqi@0: import com.sun.tools.javac.api.Formattable;
aoqi@0: import com.sun.tools.javac.api.Messages;
aoqi@0: import com.sun.tools.javac.parser.Tokens.Token.Tag;
aoqi@0: import com.sun.tools.javac.util.List;
aoqi@0: import com.sun.tools.javac.util.Name;
aoqi@0: import com.sun.tools.javac.util.Context;
aoqi@0: import com.sun.tools.javac.util.Filter;
aoqi@0: import com.sun.tools.javac.util.ListBuffer;
aoqi@0: import com.sun.tools.javac.util.Names;
aoqi@0: 
aoqi@0: /** A class that defines codes/utilities for Java source tokens
aoqi@0:  *  returned from lexical analysis.
aoqi@0:  *
aoqi@0:  *  <p><b>This is NOT part of any supported API.
aoqi@0:  *  If you write code that depends on this, you do so at your own risk.
aoqi@0:  *  This code and its internal interfaces are subject to change or
aoqi@0:  *  deletion without notice.</b>
aoqi@0:  */
aoqi@0: public class Tokens {
aoqi@0: 
aoqi@0:     private final Names names;
aoqi@0: 
aoqi@0:     /**
aoqi@0:      * Keyword array. Maps name indices to Token.
aoqi@0:      */
aoqi@0:     private final TokenKind[] key;
aoqi@0: 
aoqi@0:     /**  The number of the last entered keyword.
aoqi@0:      */
aoqi@0:     private int maxKey = 0;
aoqi@0: 
aoqi@0:     /** The names of all tokens.
aoqi@0:      */
aoqi@0:     private Name[] tokenName = new Name[TokenKind.values().length];
aoqi@0: 
aoqi@0:     public static final Context.Key<Tokens> tokensKey =
aoqi@0:         new Context.Key<Tokens>();
aoqi@0: 
aoqi@0:     public static Tokens instance(Context context) {
aoqi@0:         Tokens instance = context.get(tokensKey);
aoqi@0:         if (instance == null)
aoqi@0:             instance = new Tokens(context);
aoqi@0:         return instance;
aoqi@0:     }
aoqi@0: 
aoqi@0:     protected Tokens(Context context) {
aoqi@0:         context.put(tokensKey, this);
aoqi@0:         names = Names.instance(context);
aoqi@0:         for (TokenKind t : TokenKind.values()) {
aoqi@0:             if (t.name != null)
aoqi@0:                 enterKeyword(t.name, t);
aoqi@0:             else
aoqi@0:                 tokenName[t.ordinal()] = null;
aoqi@0:         }
aoqi@0: 
aoqi@0:         key = new TokenKind[maxKey+1];
aoqi@0:         for (int i = 0; i <= maxKey; i++) key[i] = TokenKind.IDENTIFIER;
aoqi@0:         for (TokenKind t : TokenKind.values()) {
aoqi@0:             if (t.name != null)
aoqi@0:             key[tokenName[t.ordinal()].getIndex()] = t;
aoqi@0:         }
aoqi@0:     }
aoqi@0: 
aoqi@0:     private void enterKeyword(String s, TokenKind token) {
aoqi@0:         Name n = names.fromString(s);
aoqi@0:         tokenName[token.ordinal()] = n;
aoqi@0:         if (n.getIndex() > maxKey) maxKey = n.getIndex();
aoqi@0:     }
aoqi@0: 
aoqi@0:     /**
aoqi@0:      * Create a new token given a name; if the name corresponds to a token name,
aoqi@0:      * a new token of the corresponding kind is returned; otherwise, an
aoqi@0:      * identifier token is returned.
aoqi@0:      */
aoqi@0:     TokenKind lookupKind(Name name) {
aoqi@0:         return (name.getIndex() > maxKey) ? TokenKind.IDENTIFIER : key[name.getIndex()];
aoqi@0:     }
aoqi@0: 
aoqi@0:     TokenKind lookupKind(String name) {
aoqi@0:         return lookupKind(names.fromString(name));
aoqi@0:     }
aoqi@0: 
aoqi@0:     /**
aoqi@0:      * This enum defines all tokens used by the javac scanner. A token is
aoqi@0:      * optionally associated with a name.
aoqi@0:      */
aoqi@0:     public enum TokenKind implements Formattable, Filter<TokenKind> {
aoqi@0:         EOF(),
aoqi@0:         ERROR(),
aoqi@0:         IDENTIFIER(Tag.NAMED),
aoqi@0:         ABSTRACT("abstract"),
aoqi@0:         ASSERT("assert", Tag.NAMED),
aoqi@0:         BOOLEAN("boolean", Tag.NAMED),
aoqi@0:         BREAK("break"),
aoqi@0:         BYTE("byte", Tag.NAMED),
aoqi@0:         CASE("case"),
aoqi@0:         CATCH("catch"),
aoqi@0:         CHAR("char", Tag.NAMED),
aoqi@0:         CLASS("class"),
aoqi@0:         CONST("const"),
aoqi@0:         CONTINUE("continue"),
aoqi@0:         DEFAULT("default"),
aoqi@0:         DO("do"),
aoqi@0:         DOUBLE("double", Tag.NAMED),
aoqi@0:         ELSE("else"),
aoqi@0:         ENUM("enum", Tag.NAMED),
aoqi@0:         EXTENDS("extends"),
aoqi@0:         FINAL("final"),
aoqi@0:         FINALLY("finally"),
aoqi@0:         FLOAT("float", Tag.NAMED),
aoqi@0:         FOR("for"),
aoqi@0:         GOTO("goto"),
aoqi@0:         IF("if"),
aoqi@0:         IMPLEMENTS("implements"),
aoqi@0:         IMPORT("import"),
aoqi@0:         INSTANCEOF("instanceof"),
aoqi@0:         INT("int", Tag.NAMED),
aoqi@0:         INTERFACE("interface"),
aoqi@0:         LONG("long", Tag.NAMED),
aoqi@0:         NATIVE("native"),
aoqi@0:         NEW("new"),
aoqi@0:         PACKAGE("package"),
aoqi@0:         PRIVATE("private"),
aoqi@0:         PROTECTED("protected"),
aoqi@0:         PUBLIC("public"),
aoqi@0:         RETURN("return"),
aoqi@0:         SHORT("short", Tag.NAMED),
aoqi@0:         STATIC("static"),
aoqi@0:         STRICTFP("strictfp"),
aoqi@0:         SUPER("super", Tag.NAMED),
aoqi@0:         SWITCH("switch"),
aoqi@0:         SYNCHRONIZED("synchronized"),
aoqi@0:         THIS("this", Tag.NAMED),
aoqi@0:         THROW("throw"),
aoqi@0:         THROWS("throws"),
aoqi@0:         TRANSIENT("transient"),
aoqi@0:         TRY("try"),
aoqi@0:         VOID("void", Tag.NAMED),
aoqi@0:         VOLATILE("volatile"),
aoqi@0:         WHILE("while"),
aoqi@0:         INTLITERAL(Tag.NUMERIC),
aoqi@0:         LONGLITERAL(Tag.NUMERIC),
aoqi@0:         FLOATLITERAL(Tag.NUMERIC),
aoqi@0:         DOUBLELITERAL(Tag.NUMERIC),
aoqi@0:         CHARLITERAL(Tag.NUMERIC),
aoqi@0:         STRINGLITERAL(Tag.STRING),
aoqi@0:         TRUE("true", Tag.NAMED),
aoqi@0:         FALSE("false", Tag.NAMED),
aoqi@0:         NULL("null", Tag.NAMED),
aoqi@0:         UNDERSCORE("_", Tag.NAMED),
aoqi@0:         ARROW("->"),
aoqi@0:         COLCOL("::"),
aoqi@0:         LPAREN("("),
aoqi@0:         RPAREN(")"),
aoqi@0:         LBRACE("{"),
aoqi@0:         RBRACE("}"),
aoqi@0:         LBRACKET("["),
aoqi@0:         RBRACKET("]"),
aoqi@0:         SEMI(";"),
aoqi@0:         COMMA(","),
aoqi@0:         DOT("."),
aoqi@0:         ELLIPSIS("..."),
aoqi@0:         EQ("="),
aoqi@0:         GT(">"),
aoqi@0:         LT("<"),
aoqi@0:         BANG("!"),
aoqi@0:         TILDE("~"),
aoqi@0:         QUES("?"),
aoqi@0:         COLON(":"),
aoqi@0:         EQEQ("=="),
aoqi@0:         LTEQ("<="),
aoqi@0:         GTEQ(">="),
aoqi@0:         BANGEQ("!="),
aoqi@0:         AMPAMP("&&"),
aoqi@0:         BARBAR("||"),
aoqi@0:         PLUSPLUS("++"),
aoqi@0:         SUBSUB("--"),
aoqi@0:         PLUS("+"),
aoqi@0:         SUB("-"),
aoqi@0:         STAR("*"),
aoqi@0:         SLASH("/"),
aoqi@0:         AMP("&"),
aoqi@0:         BAR("|"),
aoqi@0:         CARET("^"),
aoqi@0:         PERCENT("%"),
aoqi@0:         LTLT("<<"),
aoqi@0:         GTGT(">>"),
aoqi@0:         GTGTGT(">>>"),
aoqi@0:         PLUSEQ("+="),
aoqi@0:         SUBEQ("-="),
aoqi@0:         STAREQ("*="),
aoqi@0:         SLASHEQ("/="),
aoqi@0:         AMPEQ("&="),
aoqi@0:         BAREQ("|="),
aoqi@0:         CARETEQ("^="),
aoqi@0:         PERCENTEQ("%="),
aoqi@0:         LTLTEQ("<<="),
aoqi@0:         GTGTEQ(">>="),
aoqi@0:         GTGTGTEQ(">>>="),
aoqi@0:         MONKEYS_AT("@"),
aoqi@0:         CUSTOM;
aoqi@0: 
aoqi@0:         public final String name;
aoqi@0:         public final Tag tag;
aoqi@0: 
aoqi@0:         TokenKind() {
aoqi@0:             this(null, Tag.DEFAULT);
aoqi@0:         }
aoqi@0: 
aoqi@0:         TokenKind(String name) {
aoqi@0:             this(name, Tag.DEFAULT);
aoqi@0:         }
aoqi@0: 
aoqi@0:         TokenKind(Tag tag) {
aoqi@0:             this(null, tag);
aoqi@0:         }
aoqi@0: 
aoqi@0:         TokenKind(String name, Tag tag) {
aoqi@0:             this.name = name;
aoqi@0:             this.tag = tag;
aoqi@0:         }
aoqi@0: 
aoqi@0:         public String toString() {
aoqi@0:             switch (this) {
aoqi@0:             case IDENTIFIER:
aoqi@0:                 return "token.identifier";
aoqi@0:             case CHARLITERAL:
aoqi@0:                 return "token.character";
aoqi@0:             case STRINGLITERAL:
aoqi@0:                 return "token.string";
aoqi@0:             case INTLITERAL:
aoqi@0:                 return "token.integer";
aoqi@0:             case LONGLITERAL:
aoqi@0:                 return "token.long-integer";
aoqi@0:             case FLOATLITERAL:
aoqi@0:                 return "token.float";
aoqi@0:             case DOUBLELITERAL:
aoqi@0:                 return "token.double";
aoqi@0:             case ERROR:
aoqi@0:                 return "token.bad-symbol";
aoqi@0:             case EOF:
aoqi@0:                 return "token.end-of-input";
aoqi@0:             case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN:
aoqi@0:             case LBRACKET: case RBRACKET: case LBRACE: case RBRACE:
aoqi@0:                 return "'" + name + "'";
aoqi@0:             default:
aoqi@0:                 return name;
aoqi@0:             }
aoqi@0:         }
aoqi@0: 
aoqi@0:         public String getKind() {
aoqi@0:             return "Token";
aoqi@0:         }
aoqi@0: 
aoqi@0:         public String toString(Locale locale, Messages messages) {
aoqi@0:             return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString());
aoqi@0:         }
aoqi@0: 
aoqi@0:         @Override
aoqi@0:         public boolean accepts(TokenKind that) {
aoqi@0:             return this == that;
aoqi@0:         }
aoqi@0:     }
aoqi@0: 
aoqi@0:     public interface Comment {
aoqi@0: 
aoqi@0:         enum CommentStyle {
aoqi@0:             LINE,
aoqi@0:             BLOCK,
aoqi@0:             JAVADOC,
aoqi@0:         }
aoqi@0: 
aoqi@0:         String getText();
aoqi@0:         int getSourcePos(int index);
aoqi@0:         CommentStyle getStyle();
aoqi@0:         boolean isDeprecated();
aoqi@0:     }
aoqi@0: 
aoqi@0:     /**
aoqi@0:      * This is the class representing a javac token. Each token has several fields
aoqi@0:      * that are set by the javac lexer (i.e. start/end position, string value, etc).
aoqi@0:      */
aoqi@0:     public static class Token {
aoqi@0: 
aoqi@0:         /** tags constants **/
aoqi@0:         enum Tag {
aoqi@0:             DEFAULT,
aoqi@0:             NAMED,
aoqi@0:             STRING,
aoqi@0:             NUMERIC;
aoqi@0:         }
aoqi@0: 
aoqi@0:         /** The token kind */
aoqi@0:         public final TokenKind kind;
aoqi@0: 
aoqi@0:         /** The start position of this token */
aoqi@0:         public final int pos;
aoqi@0: 
aoqi@0:         /** The end position of this token */
aoqi@0:         public final int endPos;
aoqi@0: 
aoqi@0:         /** Comment reader associated with this token */
aoqi@0:         public final List<Comment> comments;
aoqi@0: 
aoqi@0:         Token(TokenKind kind, int pos, int endPos, List<Comment> comments) {
aoqi@0:             this.kind = kind;
aoqi@0:             this.pos = pos;
aoqi@0:             this.endPos = endPos;
aoqi@0:             this.comments = comments;
aoqi@0:             checkKind();
aoqi@0:         }
aoqi@0: 
aoqi@0:         Token[] split(Tokens tokens) {
aoqi@0:             if (kind.name.length() < 2 || kind.tag != Tag.DEFAULT) {
aoqi@0:                 throw new AssertionError("Cant split" + kind);
aoqi@0:             }
aoqi@0: 
aoqi@0:             TokenKind t1 = tokens.lookupKind(kind.name.substring(0, 1));
aoqi@0:             TokenKind t2 = tokens.lookupKind(kind.name.substring(1));
aoqi@0: 
aoqi@0:             if (t1 == null || t2 == null) {
aoqi@0:                 throw new AssertionError("Cant split - bad subtokens");
aoqi@0:             }
aoqi@0:             return new Token[] {
aoqi@0:                 new Token(t1, pos, pos + t1.name.length(), comments),
aoqi@0:                 new Token(t2, pos + t1.name.length(), endPos, null)
aoqi@0:             };
aoqi@0:         }
aoqi@0: 
aoqi@0:         protected void checkKind() {
aoqi@0:             if (kind.tag != Tag.DEFAULT) {
aoqi@0:                 throw new AssertionError("Bad token kind - expected " + Tag.STRING);
aoqi@0:             }
aoqi@0:         }
aoqi@0: 
aoqi@0:         public Name name() {
aoqi@0:             throw new UnsupportedOperationException();
aoqi@0:         }
aoqi@0: 
aoqi@0:         public String stringVal() {
aoqi@0:             throw new UnsupportedOperationException();
aoqi@0:         }
aoqi@0: 
aoqi@0:         public int radix() {
aoqi@0:             throw new UnsupportedOperationException();
aoqi@0:         }
aoqi@0: 
aoqi@0:         /**
aoqi@0:          * Preserve classic semantics - if multiple javadocs are found on the token
aoqi@0:          * the last one is returned
aoqi@0:          */
aoqi@0:         public Comment comment(Comment.CommentStyle style) {
aoqi@0:             List<Comment> comments = getComments(Comment.CommentStyle.JAVADOC);
aoqi@0:             return comments.isEmpty() ?
aoqi@0:                     null :
aoqi@0:                     comments.head;
aoqi@0:         }
aoqi@0: 
aoqi@0:         /**
aoqi@0:          * Preserve classic semantics - deprecated should be set if at least one
aoqi@0:          * javadoc comment attached to this token contains the '@deprecated' string
aoqi@0:          */
aoqi@0:         public boolean deprecatedFlag() {
aoqi@0:             for (Comment c : getComments(Comment.CommentStyle.JAVADOC)) {
aoqi@0:                 if (c.isDeprecated()) {
aoqi@0:                     return true;
aoqi@0:                 }
aoqi@0:             }
aoqi@0:             return false;
aoqi@0:         }
aoqi@0: 
aoqi@0:         private List<Comment> getComments(Comment.CommentStyle style) {
aoqi@0:             if (comments == null) {
aoqi@0:                 return List.nil();
aoqi@0:             } else {
aoqi@0:                 ListBuffer<Comment> buf = new ListBuffer<>();
aoqi@0:                 for (Comment c : comments) {
aoqi@0:                     if (c.getStyle() == style) {
aoqi@0:                         buf.add(c);
aoqi@0:                     }
aoqi@0:                 }
aoqi@0:                 return buf.toList();
aoqi@0:             }
aoqi@0:         }
aoqi@0:     }
aoqi@0: 
aoqi@0:     final static class NamedToken extends Token {
aoqi@0:         /** The name of this token */
aoqi@0:         public final Name name;
aoqi@0: 
aoqi@0:         public NamedToken(TokenKind kind, int pos, int endPos, Name name, List<Comment> comments) {
aoqi@0:             super(kind, pos, endPos, comments);
aoqi@0:             this.name = name;
aoqi@0:         }
aoqi@0: 
aoqi@0:         protected void checkKind() {
aoqi@0:             if (kind.tag != Tag.NAMED) {
aoqi@0:                 throw new AssertionError("Bad token kind - expected " + Tag.NAMED);
aoqi@0:             }
aoqi@0:         }
aoqi@0: 
aoqi@0:         @Override
aoqi@0:         public Name name() {
aoqi@0:             return name;
aoqi@0:         }
aoqi@0:     }
aoqi@0: 
aoqi@0:     static class StringToken extends Token {
aoqi@0:         /** The string value of this token */
aoqi@0:         public final String stringVal;
aoqi@0: 
aoqi@0:         public StringToken(TokenKind kind, int pos, int endPos, String stringVal, List<Comment> comments) {
aoqi@0:             super(kind, pos, endPos, comments);
aoqi@0:             this.stringVal = stringVal;
aoqi@0:         }
aoqi@0: 
aoqi@0:         protected void checkKind() {
aoqi@0:             if (kind.tag != Tag.STRING) {
aoqi@0:                 throw new AssertionError("Bad token kind - expected " + Tag.STRING);
aoqi@0:             }
aoqi@0:         }
aoqi@0: 
aoqi@0:         @Override
aoqi@0:         public String stringVal() {
aoqi@0:             return stringVal;
aoqi@0:         }
aoqi@0:     }
aoqi@0: 
aoqi@0:     final static class NumericToken extends StringToken {
aoqi@0:         /** The 'radix' value of this token */
aoqi@0:         public final int radix;
aoqi@0: 
aoqi@0:         public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, List<Comment> comments) {
aoqi@0:             super(kind, pos, endPos, stringVal, comments);
aoqi@0:             this.radix = radix;
aoqi@0:         }
aoqi@0: 
aoqi@0:         protected void checkKind() {
aoqi@0:             if (kind.tag != Tag.NUMERIC) {
aoqi@0:                 throw new AssertionError("Bad token kind - expected " + Tag.NUMERIC);
aoqi@0:             }
aoqi@0:         }
aoqi@0: 
aoqi@0:         @Override
aoqi@0:         public int radix() {
aoqi@0:             return radix;
aoqi@0:         }
aoqi@0:     }
aoqi@0: 
aoqi@0:     public static final Token DUMMY =
aoqi@0:                 new Token(TokenKind.ERROR, 0, 0, null);
aoqi@0: }