1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/src/share/classes/com/sun/tools/javac/parser/Tokens.java Mon Oct 24 13:00:20 2011 +0100
1.3 @@ -0,0 +1,423 @@
1.4 +/*
1.5 + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved.
1.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
1.7 + *
1.8 + * This code is free software; you can redistribute it and/or modify it
1.9 + * under the terms of the GNU General Public License version 2 only, as
1.10 + * published by the Free Software Foundation. Oracle designates this
1.11 + * particular file as subject to the "Classpath" exception as provided
1.12 + * by Oracle in the LICENSE file that accompanied this code.
1.13 + *
1.14 + * This code is distributed in the hope that it will be useful, but WITHOUT
1.15 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
1.16 + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
1.17 + * version 2 for more details (a copy is included in the LICENSE file that
1.18 + * accompanied this code).
1.19 + *
1.20 + * You should have received a copy of the GNU General Public License version
1.21 + * 2 along with this work; if not, write to the Free Software Foundation,
1.22 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
1.23 + *
1.24 + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
1.25 + * or visit www.oracle.com if you need additional information or have any
1.26 + * questions.
1.27 + */
1.28 +
1.29 +package com.sun.tools.javac.parser;
1.30 +
1.31 +import java.util.Locale;
1.32 +
1.33 +import com.sun.tools.javac.api.Formattable;
1.34 +import com.sun.tools.javac.api.Messages;
1.35 +import com.sun.tools.javac.parser.Tokens.Token.Tag;
1.36 +import com.sun.tools.javac.util.Name;
1.37 +import com.sun.tools.javac.util.Context;
1.38 +import com.sun.tools.javac.util.Names;
1.39 +
1.40 +/** A class that defines codes/utilities for Java source tokens
1.41 + * returned from lexical analysis.
1.42 + *
1.43 + * <p><b>This is NOT part of any supported API.
1.44 + * If you write code that depends on this, you do so at your own risk.
1.45 + * This code and its internal interfaces are subject to change or
1.46 + * deletion without notice.</b>
1.47 + */
1.48 +public class Tokens {
1.49 +
1.50 + private final Names names;
1.51 +
1.52 + /**
1.53 + * Keyword array. Maps name indices to Token.
1.54 + */
1.55 + private final TokenKind[] key;
1.56 +
1.57 + /** The number of the last entered keyword.
1.58 + */
1.59 + private int maxKey = 0;
1.60 +
1.61 + /** The names of all tokens.
1.62 + */
1.63 + private Name[] tokenName = new Name[TokenKind.values().length];
1.64 +
1.65 + public static final Context.Key<Tokens> tokensKey =
1.66 + new Context.Key<Tokens>();
1.67 +
1.68 + public static Tokens instance(Context context) {
1.69 + Tokens instance = context.get(tokensKey);
1.70 + if (instance == null)
1.71 + instance = new Tokens(context);
1.72 + return instance;
1.73 + }
1.74 +
1.75 + protected Tokens(Context context) {
1.76 + context.put(tokensKey, this);
1.77 + names = Names.instance(context);
1.78 +
1.79 + for (TokenKind t : TokenKind.values()) {
1.80 + if (t.name != null)
1.81 + enterKeyword(t.name, t);
1.82 + else
1.83 + tokenName[t.ordinal()] = null;
1.84 + }
1.85 +
1.86 + key = new TokenKind[maxKey+1];
1.87 + for (int i = 0; i <= maxKey; i++) key[i] = TokenKind.IDENTIFIER;
1.88 + for (TokenKind t : TokenKind.values()) {
1.89 + if (t.name != null)
1.90 + key[tokenName[t.ordinal()].getIndex()] = t;
1.91 + }
1.92 + }
1.93 +
1.94 + private void enterKeyword(String s, TokenKind token) {
1.95 + Name n = names.fromString(s);
1.96 + tokenName[token.ordinal()] = n;
1.97 + if (n.getIndex() > maxKey) maxKey = n.getIndex();
1.98 + }
1.99 +
1.100 + /**
1.101 + * Create a new token given a name; if the name corresponds to a token name,
1.102 + * a new token of the corresponding kind is returned; otherwise, an
1.103 + * identifier token is returned.
1.104 + */
1.105 + TokenKind lookupKind(Name name) {
1.106 + return (name.getIndex() > maxKey) ? TokenKind.IDENTIFIER : key[name.getIndex()];
1.107 + }
1.108 +
1.109 + TokenKind lookupKind(String name) {
1.110 + return lookupKind(names.fromString(name));
1.111 + }
1.112 +
1.113 + /**
1.114 + * This enum defines all tokens used by the javac scanner. A token is
1.115 + * optionally associated with a name.
1.116 + */
1.117 + public enum TokenKind implements Formattable {
1.118 + EOF(),
1.119 + ERROR(),
1.120 + IDENTIFIER(Tag.NAMED),
1.121 + ABSTRACT("abstract"),
1.122 + ASSERT("assert", Tag.NAMED),
1.123 + BOOLEAN("boolean", Tag.NAMED),
1.124 + BREAK("break"),
1.125 + BYTE("byte", Tag.NAMED),
1.126 + CASE("case"),
1.127 + CATCH("catch"),
1.128 + CHAR("char", Tag.NAMED),
1.129 + CLASS("class"),
1.130 + CONST("const"),
1.131 + CONTINUE("continue"),
1.132 + DEFAULT("default"),
1.133 + DO("do"),
1.134 + DOUBLE("double", Tag.NAMED),
1.135 + ELSE("else"),
1.136 + ENUM("enum", Tag.NAMED),
1.137 + EXTENDS("extends"),
1.138 + FINAL("final"),
1.139 + FINALLY("finally"),
1.140 + FLOAT("float", Tag.NAMED),
1.141 + FOR("for"),
1.142 + GOTO("goto"),
1.143 + IF("if"),
1.144 + IMPLEMENTS("implements"),
1.145 + IMPORT("import"),
1.146 + INSTANCEOF("instanceof"),
1.147 + INT("int", Tag.NAMED),
1.148 + INTERFACE("interface"),
1.149 + LONG("long", Tag.NAMED),
1.150 + NATIVE("native"),
1.151 + NEW("new"),
1.152 + PACKAGE("package"),
1.153 + PRIVATE("private"),
1.154 + PROTECTED("protected"),
1.155 + PUBLIC("public"),
1.156 + RETURN("return"),
1.157 + SHORT("short", Tag.NAMED),
1.158 + STATIC("static"),
1.159 + STRICTFP("strictfp"),
1.160 + SUPER("super", Tag.NAMED),
1.161 + SWITCH("switch"),
1.162 + SYNCHRONIZED("synchronized"),
1.163 + THIS("this", Tag.NAMED),
1.164 + THROW("throw"),
1.165 + THROWS("throws"),
1.166 + TRANSIENT("transient"),
1.167 + TRY("try"),
1.168 + VOID("void", Tag.NAMED),
1.169 + VOLATILE("volatile"),
1.170 + WHILE("while"),
1.171 + INTLITERAL(Tag.NUMERIC),
1.172 + LONGLITERAL(Tag.NUMERIC),
1.173 + FLOATLITERAL(Tag.NUMERIC),
1.174 + DOUBLELITERAL(Tag.NUMERIC),
1.175 + CHARLITERAL(Tag.NUMERIC),
1.176 + STRINGLITERAL(Tag.STRING),
1.177 + TRUE("true", Tag.NAMED),
1.178 + FALSE("false", Tag.NAMED),
1.179 + NULL("null", Tag.NAMED),
1.180 + LPAREN("("),
1.181 + RPAREN(")"),
1.182 + LBRACE("{"),
1.183 + RBRACE("}"),
1.184 + LBRACKET("["),
1.185 + RBRACKET("]"),
1.186 + SEMI(";"),
1.187 + COMMA(","),
1.188 + DOT("."),
1.189 + ELLIPSIS("..."),
1.190 + EQ("="),
1.191 + GT(">"),
1.192 + LT("<"),
1.193 + BANG("!"),
1.194 + TILDE("~"),
1.195 + QUES("?"),
1.196 + COLON(":"),
1.197 + EQEQ("=="),
1.198 + LTEQ("<="),
1.199 + GTEQ(">="),
1.200 + BANGEQ("!="),
1.201 + AMPAMP("&&"),
1.202 + BARBAR("||"),
1.203 + PLUSPLUS("++"),
1.204 + SUBSUB("--"),
1.205 + PLUS("+"),
1.206 + SUB("-"),
1.207 + STAR("*"),
1.208 + SLASH("/"),
1.209 + AMP("&"),
1.210 + BAR("|"),
1.211 + CARET("^"),
1.212 + PERCENT("%"),
1.213 + LTLT("<<"),
1.214 + GTGT(">>"),
1.215 + GTGTGT(">>>"),
1.216 + PLUSEQ("+="),
1.217 + SUBEQ("-="),
1.218 + STAREQ("*="),
1.219 + SLASHEQ("/="),
1.220 + AMPEQ("&="),
1.221 + BAREQ("|="),
1.222 + CARETEQ("^="),
1.223 + PERCENTEQ("%="),
1.224 + LTLTEQ("<<="),
1.225 + GTGTEQ(">>="),
1.226 + GTGTGTEQ(">>>="),
1.227 + MONKEYS_AT("@"),
1.228 + CUSTOM;
1.229 +
1.230 + public final String name;
1.231 + public final Tag tag;
1.232 +
1.233 + TokenKind() {
1.234 + this(null, Tag.DEFAULT);
1.235 + }
1.236 +
1.237 + TokenKind(String name) {
1.238 + this(name, Tag.DEFAULT);
1.239 + }
1.240 +
1.241 + TokenKind(Tag tag) {
1.242 + this(null, tag);
1.243 + }
1.244 +
1.245 + TokenKind(String name, Tag tag) {
1.246 + this.name = name;
1.247 + this.tag = tag;
1.248 + }
1.249 +
1.250 + public String toString() {
1.251 + switch (this) {
1.252 + case IDENTIFIER:
1.253 + return "token.identifier";
1.254 + case CHARLITERAL:
1.255 + return "token.character";
1.256 + case STRINGLITERAL:
1.257 + return "token.string";
1.258 + case INTLITERAL:
1.259 + return "token.integer";
1.260 + case LONGLITERAL:
1.261 + return "token.long-integer";
1.262 + case FLOATLITERAL:
1.263 + return "token.float";
1.264 + case DOUBLELITERAL:
1.265 + return "token.double";
1.266 + case ERROR:
1.267 + return "token.bad-symbol";
1.268 + case EOF:
1.269 + return "token.end-of-input";
1.270 + case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN:
1.271 + case LBRACKET: case RBRACKET: case LBRACE: case RBRACE:
1.272 + return "'" + name + "'";
1.273 + default:
1.274 + return name;
1.275 + }
1.276 + }
1.277 +
1.278 + public String getKind() {
1.279 + return "Token";
1.280 + }
1.281 +
1.282 + public String toString(Locale locale, Messages messages) {
1.283 + return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString());
1.284 + }
1.285 + }
1.286 +
1.287 + /**
1.288 + * This is the class representing a javac token. Each token has several fields
1.289 + * that are set by the javac lexer (i.e. start/end position, string value, etc).
1.290 + */
1.291 + public static class Token {
1.292 +
1.293 + /** tags constants **/
1.294 + enum Tag {
1.295 + DEFAULT,
1.296 + NAMED,
1.297 + STRING,
1.298 + NUMERIC;
1.299 + }
1.300 +
1.301 + /** The token kind */
1.302 + public final TokenKind kind;
1.303 +
1.304 + /** The start position of this token */
1.305 + public final int pos;
1.306 +
1.307 + /** The end position of this token */
1.308 + public final int endPos;
1.309 +
1.310 + /** Is this token preceeded by a deprecated comment? */
1.311 + public final boolean deprecatedFlag;
1.312 +
1.313 + /** Is this token preceeded by a deprecated comment? */
1.314 + public String docComment;
1.315 +
1.316 + Token(TokenKind kind, int pos, int endPos,
1.317 + boolean deprecatedFlag) {
1.318 + this.kind = kind;
1.319 + this.pos = pos;
1.320 + this.endPos = endPos;
1.321 + this.deprecatedFlag = deprecatedFlag;
1.322 + checkKind();
1.323 + }
1.324 +
1.325 + Token[] split(Tokens tokens) {
1.326 + if (kind.name.length() < 2 || kind.tag != Tag.DEFAULT) {
1.327 + throw new AssertionError("Cant split" + kind);
1.328 + }
1.329 +
1.330 + TokenKind t1 = tokens.lookupKind(kind.name.substring(0, 1));
1.331 + TokenKind t2 = tokens.lookupKind(kind.name.substring(1));
1.332 +
1.333 + if (t1 == null || t2 == null) {
1.334 + throw new AssertionError("Cant split - bad subtokens");
1.335 + }
1.336 + return new Token[] {
1.337 + new Token(t1, pos, pos + t1.name.length(), deprecatedFlag),
1.338 + new Token(t2, pos + t1.name.length(), endPos, false)
1.339 + };
1.340 + }
1.341 +
1.342 + protected void checkKind() {
1.343 + if (kind.tag != Tag.DEFAULT) {
1.344 + throw new AssertionError("Bad token kind - expected " + Tag.STRING);
1.345 + }
1.346 + }
1.347 +
1.348 + public Name name() {
1.349 + throw new UnsupportedOperationException();
1.350 + }
1.351 +
1.352 + public String stringVal() {
1.353 + throw new UnsupportedOperationException();
1.354 + }
1.355 +
1.356 + public int radix() {
1.357 + throw new UnsupportedOperationException();
1.358 + }
1.359 + }
1.360 +
1.361 + final static class NamedToken extends Token {
1.362 + /** The name of this token */
1.363 + public final Name name;
1.364 +
1.365 + public NamedToken(TokenKind kind, int pos, int endPos, Name name, boolean deprecatedFlag) {
1.366 + super(kind, pos, endPos, deprecatedFlag);
1.367 + this.name = name;
1.368 + }
1.369 +
1.370 + protected void checkKind() {
1.371 + if (kind.tag != Tag.NAMED) {
1.372 + throw new AssertionError("Bad token kind - expected " + Tag.NAMED);
1.373 + }
1.374 + }
1.375 +
1.376 + @Override
1.377 + public Name name() {
1.378 + return name;
1.379 + }
1.380 + }
1.381 +
1.382 + static class StringToken extends Token {
1.383 + /** The string value of this token */
1.384 + public final String stringVal;
1.385 +
1.386 + public StringToken(TokenKind kind, int pos, int endPos, String stringVal, boolean deprecatedFlag) {
1.387 + super(kind, pos, endPos, deprecatedFlag);
1.388 + this.stringVal = stringVal;
1.389 + }
1.390 +
1.391 + protected void checkKind() {
1.392 + if (kind.tag != Tag.STRING) {
1.393 + throw new AssertionError("Bad token kind - expected " + Tag.STRING);
1.394 + }
1.395 + }
1.396 +
1.397 + @Override
1.398 + public String stringVal() {
1.399 + return stringVal;
1.400 + }
1.401 + }
1.402 +
1.403 + final static class NumericToken extends StringToken {
1.404 + /** The 'radix' value of this token */
1.405 + public final int radix;
1.406 +
1.407 + public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, boolean deprecatedFlag) {
1.408 + super(kind, pos, endPos, stringVal, deprecatedFlag);
1.409 + this.radix = radix;
1.410 + }
1.411 +
1.412 + protected void checkKind() {
1.413 + if (kind.tag != Tag.NUMERIC) {
1.414 + throw new AssertionError("Bad token kind - expected " + Tag.NUMERIC);
1.415 + }
1.416 + }
1.417 +
1.418 + @Override
1.419 + public int radix() {
1.420 + return radix;
1.421 + }
1.422 + }
1.423 +
1.424 + public static final Token DUMMY =
1.425 + new Token(TokenKind.ERROR, 0, 0, false);
1.426 +}
