Mercurial > jdk8-mips64-public > langtools / file comparison

comparison: src/share/classes/com/sun/tools/javac/parser/Tokens.java

src/share/classes/com/sun/tools/javac/parser/Tokens.java

changeset 0
959103a6100f
child 2525
2eb010b6cb22
equal deleted inserted replaced
-1:000000000000 0:959103a6100f
1 /*
2 * Copyright (c) 1999, 2013, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25
26 package com.sun.tools.javac.parser;
27
28 import java.util.Locale;
29
30 import com.sun.tools.javac.api.Formattable;
31 import com.sun.tools.javac.api.Messages;
32 import com.sun.tools.javac.parser.Tokens.Token.Tag;
33 import com.sun.tools.javac.util.List;
34 import com.sun.tools.javac.util.Name;
35 import com.sun.tools.javac.util.Context;
36 import com.sun.tools.javac.util.Filter;
37 import com.sun.tools.javac.util.ListBuffer;
38 import com.sun.tools.javac.util.Names;
39
40 /** A class that defines codes/utilities for Java source tokens
41 * returned from lexical analysis.
42 *
43 * <p><b>This is NOT part of any supported API.
44 * If you write code that depends on this, you do so at your own risk.
45 * This code and its internal interfaces are subject to change or
46 * deletion without notice.</b>
47 */
48 public class Tokens {
49
50 private final Names names;
51
52 /**
53 * Keyword array. Maps name indices to Token.
54 */
55 private final TokenKind[] key;
56
57 /** The number of the last entered keyword.
58 */
59 private int maxKey = 0;
60
61 /** The names of all tokens.
62 */
63 private Name[] tokenName = new Name[TokenKind.values().length];
64
65 public static final Context.Key<Tokens> tokensKey =
66 new Context.Key<Tokens>();
67
68 public static Tokens instance(Context context) {
69 Tokens instance = context.get(tokensKey);
70 if (instance == null)
71 instance = new Tokens(context);
72 return instance;
73 }
74
75 protected Tokens(Context context) {
76 context.put(tokensKey, this);
77 names = Names.instance(context);
78 for (TokenKind t : TokenKind.values()) {
79 if (t.name != null)
80 enterKeyword(t.name, t);
81 else
82 tokenName[t.ordinal()] = null;
83 }
84
85 key = new TokenKind[maxKey+1];
86 for (int i = 0; i <= maxKey; i++) key[i] = TokenKind.IDENTIFIER;
87 for (TokenKind t : TokenKind.values()) {
88 if (t.name != null)
89 key[tokenName[t.ordinal()].getIndex()] = t;
90 }
91 }
92
93 private void enterKeyword(String s, TokenKind token) {
94 Name n = names.fromString(s);
95 tokenName[token.ordinal()] = n;
96 if (n.getIndex() > maxKey) maxKey = n.getIndex();
97 }
98
99 /**
100 * Create a new token given a name; if the name corresponds to a token name,
101 * a new token of the corresponding kind is returned; otherwise, an
102 * identifier token is returned.
103 */
104 TokenKind lookupKind(Name name) {
105 return (name.getIndex() > maxKey) ? TokenKind.IDENTIFIER : key[name.getIndex()];
106 }
107
108 TokenKind lookupKind(String name) {
109 return lookupKind(names.fromString(name));
110 }
111
112 /**
113 * This enum defines all tokens used by the javac scanner. A token is
114 * optionally associated with a name.
115 */
116 public enum TokenKind implements Formattable, Filter<TokenKind> {
117 EOF(),
118 ERROR(),
119 IDENTIFIER(Tag.NAMED),
120 ABSTRACT("abstract"),
121 ASSERT("assert", Tag.NAMED),
122 BOOLEAN("boolean", Tag.NAMED),
123 BREAK("break"),
124 BYTE("byte", Tag.NAMED),
125 CASE("case"),
126 CATCH("catch"),
127 CHAR("char", Tag.NAMED),
128 CLASS("class"),
129 CONST("const"),
130 CONTINUE("continue"),
131 DEFAULT("default"),
132 DO("do"),
133 DOUBLE("double", Tag.NAMED),
134 ELSE("else"),
135 ENUM("enum", Tag.NAMED),
136 EXTENDS("extends"),
137 FINAL("final"),
138 FINALLY("finally"),
139 FLOAT("float", Tag.NAMED),
140 FOR("for"),
141 GOTO("goto"),
142 IF("if"),
143 IMPLEMENTS("implements"),
144 IMPORT("import"),
145 INSTANCEOF("instanceof"),
146 INT("int", Tag.NAMED),
147 INTERFACE("interface"),
148 LONG("long", Tag.NAMED),
149 NATIVE("native"),
150 NEW("new"),
151 PACKAGE("package"),
152 PRIVATE("private"),
153 PROTECTED("protected"),
154 PUBLIC("public"),
155 RETURN("return"),
156 SHORT("short", Tag.NAMED),
157 STATIC("static"),
158 STRICTFP("strictfp"),
159 SUPER("super", Tag.NAMED),
160 SWITCH("switch"),
161 SYNCHRONIZED("synchronized"),
162 THIS("this", Tag.NAMED),
163 THROW("throw"),
164 THROWS("throws"),
165 TRANSIENT("transient"),
166 TRY("try"),
167 VOID("void", Tag.NAMED),
168 VOLATILE("volatile"),
169 WHILE("while"),
170 INTLITERAL(Tag.NUMERIC),
171 LONGLITERAL(Tag.NUMERIC),
172 FLOATLITERAL(Tag.NUMERIC),
173 DOUBLELITERAL(Tag.NUMERIC),
174 CHARLITERAL(Tag.NUMERIC),
175 STRINGLITERAL(Tag.STRING),
176 TRUE("true", Tag.NAMED),
177 FALSE("false", Tag.NAMED),
178 NULL("null", Tag.NAMED),
179 UNDERSCORE("_", Tag.NAMED),
180 ARROW("->"),
181 COLCOL("::"),
182 LPAREN("("),
183 RPAREN(")"),
184 LBRACE("{"),
185 RBRACE("}"),
186 LBRACKET("["),
187 RBRACKET("]"),
188 SEMI(";"),
189 COMMA(","),
190 DOT("."),
191 ELLIPSIS("..."),
192 EQ("="),
193 GT(">"),
194 LT("<"),
195 BANG("!"),
196 TILDE("~"),
197 QUES("?"),
198 COLON(":"),
199 EQEQ("=="),
200 LTEQ("<="),
201 GTEQ(">="),
202 BANGEQ("!="),
203 AMPAMP("&&"),
204 BARBAR("||"),
205 PLUSPLUS("++"),
206 SUBSUB("--"),
207 PLUS("+"),
208 SUB("-"),
209 STAR("*"),
210 SLASH("/"),
211 AMP("&"),
212 BAR("|"),
213 CARET("^"),
214 PERCENT("%"),
215 LTLT("<<"),
216 GTGT(">>"),
217 GTGTGT(">>>"),
218 PLUSEQ("+="),
219 SUBEQ("-="),
220 STAREQ("*="),
221 SLASHEQ("/="),
222 AMPEQ("&="),
223 BAREQ("|="),
224 CARETEQ("^="),
225 PERCENTEQ("%="),
226 LTLTEQ("<<="),
227 GTGTEQ(">>="),
228 GTGTGTEQ(">>>="),
229 MONKEYS_AT("@"),
230 CUSTOM;
231
232 public final String name;
233 public final Tag tag;
234
235 TokenKind() {
236 this(null, Tag.DEFAULT);
237 }
238
239 TokenKind(String name) {
240 this(name, Tag.DEFAULT);
241 }
242
243 TokenKind(Tag tag) {
244 this(null, tag);
245 }
246
247 TokenKind(String name, Tag tag) {
248 this.name = name;
249 this.tag = tag;
250 }
251
252 public String toString() {
253 switch (this) {
254 case IDENTIFIER:
255 return "token.identifier";
256 case CHARLITERAL:
257 return "token.character";
258 case STRINGLITERAL:
259 return "token.string";
260 case INTLITERAL:
261 return "token.integer";
262 case LONGLITERAL:
263 return "token.long-integer";
264 case FLOATLITERAL:
265 return "token.float";
266 case DOUBLELITERAL:
267 return "token.double";
268 case ERROR:
269 return "token.bad-symbol";
270 case EOF:
271 return "token.end-of-input";
272 case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN:
273 case LBRACKET: case RBRACKET: case LBRACE: case RBRACE:
274 return "'" + name + "'";
275 default:
276 return name;
277 }
278 }
279
280 public String getKind() {
281 return "Token";
282 }
283
284 public String toString(Locale locale, Messages messages) {
285 return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString());
286 }
287
288 @Override
289 public boolean accepts(TokenKind that) {
290 return this == that;
291 }
292 }
293
294 public interface Comment {
295
296 enum CommentStyle {
297 LINE,
298 BLOCK,
299 JAVADOC,
300 }
301
302 String getText();
303 int getSourcePos(int index);
304 CommentStyle getStyle();
305 boolean isDeprecated();
306 }
307
308 /**
309 * This is the class representing a javac token. Each token has several fields
310 * that are set by the javac lexer (i.e. start/end position, string value, etc).
311 */
312 public static class Token {
313
314 /** tags constants **/
315 enum Tag {
316 DEFAULT,
317 NAMED,
318 STRING,
319 NUMERIC;
320 }
321
322 /** The token kind */
323 public final TokenKind kind;
324
325 /** The start position of this token */
326 public final int pos;
327
328 /** The end position of this token */
329 public final int endPos;
330
331 /** Comment reader associated with this token */
332 public final List<Comment> comments;
333
334 Token(TokenKind kind, int pos, int endPos, List<Comment> comments) {
335 this.kind = kind;
336 this.pos = pos;
337 this.endPos = endPos;
338 this.comments = comments;
339 checkKind();
340 }
341
342 Token[] split(Tokens tokens) {
343 if (kind.name.length() < 2 || kind.tag != Tag.DEFAULT) {
344 throw new AssertionError("Cant split" + kind);
345 }
346
347 TokenKind t1 = tokens.lookupKind(kind.name.substring(0, 1));
348 TokenKind t2 = tokens.lookupKind(kind.name.substring(1));
349
350 if (t1 == null || t2 == null) {
351 throw new AssertionError("Cant split - bad subtokens");
352 }
353 return new Token[] {
354 new Token(t1, pos, pos + t1.name.length(), comments),
355 new Token(t2, pos + t1.name.length(), endPos, null)
356 };
357 }
358
359 protected void checkKind() {
360 if (kind.tag != Tag.DEFAULT) {
361 throw new AssertionError("Bad token kind - expected " + Tag.STRING);
362 }
363 }
364
365 public Name name() {
366 throw new UnsupportedOperationException();
367 }
368
369 public String stringVal() {
370 throw new UnsupportedOperationException();
371 }
372
373 public int radix() {
374 throw new UnsupportedOperationException();
375 }
376
377 /**
378 * Preserve classic semantics - if multiple javadocs are found on the token
379 * the last one is returned
380 */
381 public Comment comment(Comment.CommentStyle style) {
382 List<Comment> comments = getComments(Comment.CommentStyle.JAVADOC);
383 return comments.isEmpty() ?
384 null :
385 comments.head;
386 }
387
388 /**
389 * Preserve classic semantics - deprecated should be set if at least one
390 * javadoc comment attached to this token contains the '@deprecated' string
391 */
392 public boolean deprecatedFlag() {
393 for (Comment c : getComments(Comment.CommentStyle.JAVADOC)) {
394 if (c.isDeprecated()) {
395 return true;
396 }
397 }
398 return false;
399 }
400
401 private List<Comment> getComments(Comment.CommentStyle style) {
402 if (comments == null) {
403 return List.nil();
404 } else {
405 ListBuffer<Comment> buf = new ListBuffer<>();
406 for (Comment c : comments) {
407 if (c.getStyle() == style) {
408 buf.add(c);
409 }
410 }
411 return buf.toList();
412 }
413 }
414 }
415
416 final static class NamedToken extends Token {
417 /** The name of this token */
418 public final Name name;
419
420 public NamedToken(TokenKind kind, int pos, int endPos, Name name, List<Comment> comments) {
421 super(kind, pos, endPos, comments);
422 this.name = name;
423 }
424
425 protected void checkKind() {
426 if (kind.tag != Tag.NAMED) {
427 throw new AssertionError("Bad token kind - expected " + Tag.NAMED);
428 }
429 }
430
431 @Override
432 public Name name() {
433 return name;
434 }
435 }
436
437 static class StringToken extends Token {
438 /** The string value of this token */
439 public final String stringVal;
440
441 public StringToken(TokenKind kind, int pos, int endPos, String stringVal, List<Comment> comments) {
442 super(kind, pos, endPos, comments);
443 this.stringVal = stringVal;
444 }
445
446 protected void checkKind() {
447 if (kind.tag != Tag.STRING) {
448 throw new AssertionError("Bad token kind - expected " + Tag.STRING);
449 }
450 }
451
452 @Override
453 public String stringVal() {
454 return stringVal;
455 }
456 }
457
458 final static class NumericToken extends StringToken {
459 /** The 'radix' value of this token */
460 public final int radix;
461
462 public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, List<Comment> comments) {
463 super(kind, pos, endPos, stringVal, comments);
464 this.radix = radix;
465 }
466
467 protected void checkKind() {
468 if (kind.tag != Tag.NUMERIC) {
469 throw new AssertionError("Bad token kind - expected " + Tag.NUMERIC);
470 }
471 }
472
473 @Override
474 public int radix() {
475 return radix;
476 }
477 }
478
479 public static final Token DUMMY =
480 new Token(TokenKind.ERROR, 0, 0, null);
481 }

Mercurial Repository: jdk8-mips64-public/langtools

mercurial