jdk8-mips64-public/langtools: src/share/classes/com/sun/tools/javac/parser/Tokens.java@d346ab55031b (annotated)

src/share/classes/com/sun/tools/javac/parser/Tokens.java@d346ab55031b (annotated)

src/share/classes/com/sun/tools/javac/parser/Tokens.java

Mon, 24 Oct 2011 13:00:20 +0100

author: mcimadamore
date: Mon, 24 Oct 2011 13:00:20 +0100
changeset 1113: d346ab55031b
child 1125: 56830d5cb5bb
permissions: -rw-r--r--

7096014: Javac tokens should retain state
Summary: Refactor javac tokens from enum constants to stateful instances (to keep track of position, comments, etc.)
Reviewed-by: jjg

 /*
  * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 only, as
  * published by the Free Software Foundation.  Oracle designates this
  * particular file as subject to the "Classpath" exception as provided
  * by Oracle in the LICENSE file that accompanied this code.
  *
  * This code is distributed in the hope that it will be useful, but WITHOUT
  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  * version 2 for more details (a copy is included in the LICENSE file that
  * accompanied this code).
  *
  * You should have received a copy of the GNU General Public License version
  * 2 along with this work; if not, write to the Free Software Foundation,
  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  *
  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  * or visit www.oracle.com if you need additional information or have any
  * questions.
  */
 package com.sun.tools.javac.parser;
 import java.util.Locale;
 import com.sun.tools.javac.api.Formattable;
 import com.sun.tools.javac.api.Messages;
 import com.sun.tools.javac.parser.Tokens.Token.Tag;
 import com.sun.tools.javac.util.Name;
 import com.sun.tools.javac.util.Context;
 import com.sun.tools.javac.util.Names;
 /** A class that defines codes/utilities for Java source tokens
  *  returned from lexical analysis.
  *
  *  <p><b>This is NOT part of any supported API.
  *  If you write code that depends on this, you do so at your own risk.
  *  This code and its internal interfaces are subject to change or
  *  deletion without notice.</b>
  */
 public class Tokens {
     private final Names names;
     /**
      * Keyword array. Maps name indices to Token.
      */
     private final TokenKind[] key;
     /**  The number of the last entered keyword.
      */
     private int maxKey = 0;
     /** The names of all tokens.
      */
     private Name[] tokenName = new Name[TokenKind.values().length];
     public static final Context.Key<Tokens> tokensKey =
         new Context.Key<Tokens>();
     public static Tokens instance(Context context) {
         Tokens instance = context.get(tokensKey);
         if (instance == null)
             instance = new Tokens(context);
         return instance;
     }
     protected Tokens(Context context) {
         context.put(tokensKey, this);
         names = Names.instance(context);
         for (TokenKind t : TokenKind.values()) {
             if (t.name != null)
                 enterKeyword(t.name, t);
             else
                 tokenName[t.ordinal()] = null;
         }
         key = new TokenKind[maxKey+1];
         for (int i = 0; i <= maxKey; i++) key[i] = TokenKind.IDENTIFIER;
         for (TokenKind t : TokenKind.values()) {
             if (t.name != null)
             key[tokenName[t.ordinal()].getIndex()] = t;
         }
     }
     private void enterKeyword(String s, TokenKind token) {
         Name n = names.fromString(s);
         tokenName[token.ordinal()] = n;
         if (n.getIndex() > maxKey) maxKey = n.getIndex();
     }
     /**
      * Create a new token given a name; if the name corresponds to a token name,
      * a new token of the corresponding kind is returned; otherwise, an
      * identifier token is returned.
      */
     TokenKind lookupKind(Name name) {
         return (name.getIndex() > maxKey) ? TokenKind.IDENTIFIER : key[name.getIndex()];
     }
     TokenKind lookupKind(String name) {
         return lookupKind(names.fromString(name));
     }
     /**
      * This enum defines all tokens used by the javac scanner. A token is
      * optionally associated with a name.
      */
     public enum TokenKind implements Formattable {
         EOF(),
         ERROR(),
         IDENTIFIER(Tag.NAMED),
         ABSTRACT("abstract"),
         ASSERT("assert", Tag.NAMED),
         BOOLEAN("boolean", Tag.NAMED),
         BREAK("break"),
         BYTE("byte", Tag.NAMED),
         CASE("case"),
         CATCH("catch"),
         CHAR("char", Tag.NAMED),
         CLASS("class"),
         CONST("const"),
         CONTINUE("continue"),
         DEFAULT("default"),
         DO("do"),
         DOUBLE("double", Tag.NAMED),
         ELSE("else"),
         ENUM("enum", Tag.NAMED),
         EXTENDS("extends"),
         FINAL("final"),
         FINALLY("finally"),
         FLOAT("float", Tag.NAMED),
         FOR("for"),
         GOTO("goto"),
         IF("if"),
         IMPLEMENTS("implements"),
         IMPORT("import"),
         INSTANCEOF("instanceof"),
         INT("int", Tag.NAMED),
         INTERFACE("interface"),
         LONG("long", Tag.NAMED),
         NATIVE("native"),
         NEW("new"),
         PACKAGE("package"),
         PRIVATE("private"),
         PROTECTED("protected"),
         PUBLIC("public"),
         RETURN("return"),
         SHORT("short", Tag.NAMED),
         STATIC("static"),
         STRICTFP("strictfp"),
         SUPER("super", Tag.NAMED),
         SWITCH("switch"),
         SYNCHRONIZED("synchronized"),
         THIS("this", Tag.NAMED),
         THROW("throw"),
         THROWS("throws"),
         TRANSIENT("transient"),
         TRY("try"),
         VOID("void", Tag.NAMED),
         VOLATILE("volatile"),
         WHILE("while"),
         INTLITERAL(Tag.NUMERIC),
         LONGLITERAL(Tag.NUMERIC),
         FLOATLITERAL(Tag.NUMERIC),
         DOUBLELITERAL(Tag.NUMERIC),
         CHARLITERAL(Tag.NUMERIC),
         STRINGLITERAL(Tag.STRING),
         TRUE("true", Tag.NAMED),
         FALSE("false", Tag.NAMED),
         NULL("null", Tag.NAMED),
         LPAREN("("),
         RPAREN(")"),
         LBRACE("{"),
         RBRACE("}"),
         LBRACKET("["),
         RBRACKET("]"),
         SEMI(";"),
         COMMA(","),
         DOT("."),
         ELLIPSIS("..."),
         EQ("="),
         GT(">"),
         LT("<"),
         BANG("!"),
         TILDE("~"),
         QUES("?"),
         COLON(":"),
         EQEQ("=="),
         LTEQ("<="),
         GTEQ(">="),
         BANGEQ("!="),
         AMPAMP("&&"),
         BARBAR("||"),
         PLUSPLUS("++"),
         SUBSUB("--"),
         PLUS("+"),
         SUB("-"),
         STAR("*"),
         SLASH("/"),
         AMP("&"),
         BAR("|"),
         CARET("^"),
         PERCENT("%"),
         LTLT("<<"),
         GTGT(">>"),
         GTGTGT(">>>"),
         PLUSEQ("+="),
         SUBEQ("-="),
         STAREQ("*="),
         SLASHEQ("/="),
         AMPEQ("&="),
         BAREQ("|="),
         CARETEQ("^="),
         PERCENTEQ("%="),
         LTLTEQ("<<="),
         GTGTEQ(">>="),
         GTGTGTEQ(">>>="),
         MONKEYS_AT("@"),
         CUSTOM;
         public final String name;
         public final Tag tag;
         TokenKind() {
             this(null, Tag.DEFAULT);
         }
         TokenKind(String name) {
             this(name, Tag.DEFAULT);
         }
         TokenKind(Tag tag) {
             this(null, tag);
         }
         TokenKind(String name, Tag tag) {
             this.name = name;
             this.tag = tag;
         }
         public String toString() {
             switch (this) {
             case IDENTIFIER:
                 return "token.identifier";
             case CHARLITERAL:
                 return "token.character";
             case STRINGLITERAL:
                 return "token.string";
             case INTLITERAL:
                 return "token.integer";
             case LONGLITERAL:
                 return "token.long-integer";
             case FLOATLITERAL:
                 return "token.float";
             case DOUBLELITERAL:
                 return "token.double";
             case ERROR:
                 return "token.bad-symbol";
             case EOF:
                 return "token.end-of-input";
             case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN:
             case LBRACKET: case RBRACKET: case LBRACE: case RBRACE:
                 return "'" + name + "'";
             default:
                 return name;
             }
         }
         public String getKind() {
             return "Token";
         }
         public String toString(Locale locale, Messages messages) {
             return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString());
         }
     }
     /**
      * This is the class representing a javac token. Each token has several fields
      * that are set by the javac lexer (i.e. start/end position, string value, etc).
      */
     public static class Token {
         /** tags constants **/
         enum Tag {
             DEFAULT,
             NAMED,
             STRING,
             NUMERIC;
         }
         /** The token kind */
         public final TokenKind kind;
         /** The start position of this token */
         public final int pos;
         /** The end position of this token */
         public final int endPos;
         /** Is this token preceeded by a deprecated comment? */
         public final boolean deprecatedFlag;
         /** Is this token preceeded by a deprecated comment? */
         public String docComment;
         Token(TokenKind kind, int pos, int endPos,
                 boolean deprecatedFlag) {
             this.kind = kind;
             this.pos = pos;
             this.endPos = endPos;
             this.deprecatedFlag = deprecatedFlag;
             checkKind();
         }
         Token[] split(Tokens tokens) {
             if (kind.name.length() < 2 || kind.tag != Tag.DEFAULT) {
                 throw new AssertionError("Cant split" + kind);
             }
             TokenKind t1 = tokens.lookupKind(kind.name.substring(0, 1));
             TokenKind t2 = tokens.lookupKind(kind.name.substring(1));
             if (t1 == null || t2 == null) {
                 throw new AssertionError("Cant split - bad subtokens");
             }
             return new Token[] {
                 new Token(t1, pos, pos + t1.name.length(), deprecatedFlag),
                 new Token(t2, pos + t1.name.length(), endPos, false)
             };
         }
         protected void checkKind() {
             if (kind.tag != Tag.DEFAULT) {
                 throw new AssertionError("Bad token kind - expected " + Tag.STRING);
             }
         }
         public Name name() {
             throw new UnsupportedOperationException();
         }
         public String stringVal() {
             throw new UnsupportedOperationException();
         }
         public int radix() {
             throw new UnsupportedOperationException();
         }
     }
     final static class NamedToken extends Token {
         /** The name of this token */
         public final Name name;
         public NamedToken(TokenKind kind, int pos, int endPos, Name name, boolean deprecatedFlag) {
             super(kind, pos, endPos, deprecatedFlag);
             this.name = name;
         }
         protected void checkKind() {
             if (kind.tag != Tag.NAMED) {
                 throw new AssertionError("Bad token kind - expected " + Tag.NAMED);
             }
         }
         @Override
         public Name name() {
             return name;
         }
     }
     static class StringToken extends Token {
         /** The string value of this token */
         public final String stringVal;
         public StringToken(TokenKind kind, int pos, int endPos, String stringVal, boolean deprecatedFlag) {
             super(kind, pos, endPos, deprecatedFlag);
             this.stringVal = stringVal;
         }
         protected void checkKind() {
             if (kind.tag != Tag.STRING) {
                 throw new AssertionError("Bad token kind - expected " + Tag.STRING);
             }
         }
         @Override
         public String stringVal() {
             return stringVal;
         }
     }
     final static class NumericToken extends StringToken {
         /** The 'radix' value of this token */
         public final int radix;
         public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, boolean deprecatedFlag) {
             super(kind, pos, endPos, stringVal, deprecatedFlag);
             this.radix = radix;
         }
         protected void checkKind() {
             if (kind.tag != Tag.NUMERIC) {
                 throw new AssertionError("Bad token kind - expected " + Tag.NUMERIC);
             }
         }
         @Override
         public int radix() {
             return radix;
         }
     }
     public static final Token DUMMY =
                 new Token(TokenKind.ERROR, 0, 0, false);
 }

Mercurial > jdk8-mips64-public > langtools / annotate

src/share/classes/com/sun/tools/javac/parser/Tokens.java@d346ab55031b (annotated)

src/share/classes/com/sun/tools/javac/parser/Tokens.java