OLD | NEW |
1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 library analyzer.src.dart.ast.token; | 5 library analyzer.src.dart.ast.token; |
6 | 6 |
7 import 'package:analyzer/dart/ast/token.dart'; | 7 import 'package:front_end/src/scanner/token.dart'; |
8 import 'package:analyzer/src/generated/java_engine.dart'; | |
9 | 8 |
10 /** | 9 export 'package:front_end/src/scanner/token.dart' |
11 * The opening half of a grouping pair of tokens. This is used for curly | 10 show |
12 * brackets ('{'), parentheses ('('), and square brackets ('['). | 11 BeginToken, |
13 */ | 12 BeginTokenWithComment, |
14 class BeginToken extends SimpleToken { | 13 CommentToken, |
15 /** | 14 DocumentationCommentToken, |
16 * The token that corresponds to this token. | 15 KeywordToken, |
17 */ | 16 KeywordTokenWithComment, |
18 Token endToken; | 17 SimpleToken, |
19 | 18 StringToken, |
20 /** | 19 StringTokenWithComment, |
21 * Initialize a newly created token to have the given [type] at the given | 20 TokenClass, |
22 * [offset]. | 21 TokenWithComment; |
23 */ | |
24 BeginToken(TokenType type, int offset) : super(type, offset) { | |
25 assert(type == TokenType.OPEN_CURLY_BRACKET || | |
26 type == TokenType.OPEN_PAREN || | |
27 type == TokenType.OPEN_SQUARE_BRACKET || | |
28 type == TokenType.STRING_INTERPOLATION_EXPRESSION); | |
29 } | |
30 | |
31 @override | |
32 Token copy() => new BeginToken(type, offset); | |
33 } | |
34 | |
35 /** | |
36 * A begin token that is preceded by comments. | |
37 */ | |
38 class BeginTokenWithComment extends BeginToken implements TokenWithComment { | |
39 /** | |
40 * The first comment in the list of comments that precede this token. | |
41 */ | |
42 @override | |
43 CommentToken _precedingComment; | |
44 | |
45 /** | |
46 * Initialize a newly created token to have the given [type] at the given | |
47 * [offset] and to be preceded by the comments reachable from the given | |
48 * [_precedingComment]. | |
49 */ | |
50 BeginTokenWithComment(TokenType type, int offset, this._precedingComment) | |
51 : super(type, offset) { | |
52 _setCommentParent(_precedingComment); | |
53 } | |
54 | |
55 @override | |
56 CommentToken get precedingComments => _precedingComment; | |
57 | |
58 @override | |
59 void set precedingComments(CommentToken comment) { | |
60 _precedingComment = comment; | |
61 _setCommentParent(_precedingComment); | |
62 } | |
63 | |
64 @override | |
65 void applyDelta(int delta) { | |
66 super.applyDelta(delta); | |
67 Token token = precedingComments; | |
68 while (token != null) { | |
69 token.applyDelta(delta); | |
70 token = token.next; | |
71 } | |
72 } | |
73 | |
74 @override | |
75 Token copy() => | |
76 new BeginTokenWithComment(type, offset, copyComments(precedingComments)); | |
77 } | |
78 | |
79 /** | |
80 * A token representing a comment. | |
81 */ | |
82 class CommentToken extends StringToken { | |
83 /** | |
84 * The token that contains this comment. | |
85 */ | |
86 TokenWithComment parent; | |
87 | |
88 /** | |
89 * Initialize a newly created token to represent a token of the given [type] | |
90 * with the given [value] at the given [offset]. | |
91 */ | |
92 CommentToken(TokenType type, String value, int offset) | |
93 : super(type, value, offset); | |
94 | |
95 @override | |
96 CommentToken copy() => new CommentToken(type, _value, offset); | |
97 | |
98 /** | |
99 * Remove this comment token from the list. | |
100 * | |
101 * This is used when we decide to interpret the comment as syntax. | |
102 */ | |
103 void remove() { | |
104 if (previous != null) { | |
105 previous.setNextWithoutSettingPrevious(next); | |
106 next?.previous = previous; | |
107 } else { | |
108 assert(parent.precedingComments == this); | |
109 parent.precedingComments = next; | |
110 } | |
111 } | |
112 } | |
113 | |
114 /** | |
115 * A documentation comment token. | |
116 */ | |
117 class DocumentationCommentToken extends CommentToken { | |
118 /** | |
119 * The references embedded within the documentation comment. | |
120 * This list will be empty unless this is a documentation comment that has | |
121 * references embedded within it. | |
122 */ | |
123 final List<Token> references = <Token>[]; | |
124 | |
125 /** | |
126 * Initialize a newly created token to represent a token of the given [type] | |
127 * with the given [value] at the given [offset]. | |
128 */ | |
129 DocumentationCommentToken(TokenType type, String value, int offset) | |
130 : super(type, value, offset); | |
131 | |
132 @override | |
133 CommentToken copy() { | |
134 DocumentationCommentToken copy = | |
135 new DocumentationCommentToken(type, _value, offset); | |
136 references.forEach((ref) => copy.references.add(ref.copy())); | |
137 return copy; | |
138 } | |
139 } | |
140 | |
141 /** | |
142 * A token representing a keyword in the language. | |
143 */ | |
144 class KeywordToken extends SimpleToken { | |
145 @override | |
146 final Keyword keyword; | |
147 | |
148 /** | |
149 * Initialize a newly created token to represent the given [keyword] at the | |
150 * given [offset]. | |
151 */ | |
152 KeywordToken(this.keyword, int offset) : super(TokenType.KEYWORD, offset); | |
153 | |
154 @override | |
155 String get lexeme => keyword.syntax; | |
156 | |
157 @override | |
158 Token copy() => new KeywordToken(keyword, offset); | |
159 | |
160 @override | |
161 Keyword value() => keyword; | |
162 } | |
163 | |
164 /** | |
165 * A keyword token that is preceded by comments. | |
166 */ | |
167 class KeywordTokenWithComment extends KeywordToken implements TokenWithComment { | |
168 /** | |
169 * The first comment in the list of comments that precede this token. | |
170 */ | |
171 @override | |
172 CommentToken _precedingComment; | |
173 | |
174 /** | |
175 * Initialize a newly created token to to represent the given [keyword] at the | |
176 * given [offset] and to be preceded by the comments reachable from the given | |
177 * [_precedingComment]. | |
178 */ | |
179 KeywordTokenWithComment(Keyword keyword, int offset, this._precedingComment) | |
180 : super(keyword, offset) { | |
181 _setCommentParent(_precedingComment); | |
182 } | |
183 | |
184 @override | |
185 CommentToken get precedingComments => _precedingComment; | |
186 | |
187 void set precedingComments(CommentToken comment) { | |
188 _precedingComment = comment; | |
189 _setCommentParent(_precedingComment); | |
190 } | |
191 | |
192 @override | |
193 void applyDelta(int delta) { | |
194 super.applyDelta(delta); | |
195 Token token = precedingComments; | |
196 while (token != null) { | |
197 token.applyDelta(delta); | |
198 token = token.next; | |
199 } | |
200 } | |
201 | |
202 @override | |
203 Token copy() => new KeywordTokenWithComment( | |
204 keyword, offset, copyComments(precedingComments)); | |
205 } | |
206 | |
207 /** | |
208 * A token that was scanned from the input. Each token knows which tokens | |
209 * precede and follow it, acting as a link in a doubly linked list of tokens. | |
210 */ | |
211 class SimpleToken implements Token { | |
212 /** | |
213 * The type of the token. | |
214 */ | |
215 @override | |
216 final TokenType type; | |
217 | |
218 /** | |
219 * The offset from the beginning of the file to the first character in the | |
220 * token. | |
221 */ | |
222 @override | |
223 int offset = 0; | |
224 | |
225 /** | |
226 * The previous token in the token stream. | |
227 */ | |
228 @override | |
229 Token previous; | |
230 | |
231 /** | |
232 * The next token in the token stream. | |
233 */ | |
234 Token _next; | |
235 | |
236 /** | |
237 * Initialize a newly created token to have the given [type] and [offset]. | |
238 */ | |
239 SimpleToken(this.type, this.offset); | |
240 | |
241 @override | |
242 int get end => offset + length; | |
243 | |
244 @override | |
245 bool get isOperator => type.isOperator; | |
246 | |
247 @override | |
248 bool get isSynthetic => length == 0; | |
249 | |
250 @override | |
251 bool get isUserDefinableOperator => type.isUserDefinableOperator; | |
252 | |
253 @override | |
254 Keyword get keyword => null; | |
255 | |
256 @override | |
257 int get length => lexeme.length; | |
258 | |
259 @override | |
260 String get lexeme => type.lexeme; | |
261 | |
262 @override | |
263 Token get next => _next; | |
264 | |
265 @override | |
266 CommentToken get precedingComments => null; | |
267 | |
268 @override | |
269 void applyDelta(int delta) { | |
270 offset += delta; | |
271 } | |
272 | |
273 @override | |
274 Token copy() => new Token(type, offset); | |
275 | |
276 @override | |
277 Token copyComments(Token token) { | |
278 if (token == null) { | |
279 return null; | |
280 } | |
281 Token head = token.copy(); | |
282 Token tail = head; | |
283 token = token.next; | |
284 while (token != null) { | |
285 tail = tail.setNext(token.copy()); | |
286 token = token.next; | |
287 } | |
288 return head; | |
289 } | |
290 | |
291 @override | |
292 bool matchesAny(List<TokenType> types) { | |
293 for (TokenType type in types) { | |
294 if (this.type == type) { | |
295 return true; | |
296 } | |
297 } | |
298 return false; | |
299 } | |
300 | |
301 @override | |
302 Token setNext(Token token) { | |
303 _next = token; | |
304 token.previous = this; | |
305 return token; | |
306 } | |
307 | |
308 @override | |
309 Token setNextWithoutSettingPrevious(Token token) { | |
310 _next = token; | |
311 return token; | |
312 } | |
313 | |
314 @override | |
315 String toString() => lexeme; | |
316 | |
317 @override | |
318 Object value() => type.lexeme; | |
319 | |
320 /** | |
321 * Sets the `parent` property to `this` for the given [comment] and all the | |
322 * next tokens. | |
323 */ | |
324 void _setCommentParent(CommentToken comment) { | |
325 while (comment != null) { | |
326 comment.parent = this; | |
327 comment = comment.next; | |
328 } | |
329 } | |
330 } | |
331 | 22 |
332 /** | 23 /** |
333 * A token whose value is independent of it's type. | 24 * A token whose value is independent of it's type. |
334 */ | |
335 class StringToken extends SimpleToken { | |
336 /** | |
337 * The lexeme represented by this token. | |
338 */ | |
339 String _value; | |
340 | |
341 /** | |
342 * Initialize a newly created token to represent a token of the given [type] | |
343 * with the given [value] at the given [offset]. | |
344 */ | |
345 StringToken(TokenType type, String value, int offset) : super(type, offset) { | |
346 this._value = StringUtilities.intern(value); | |
347 } | |
348 | |
349 @override | |
350 String get lexeme => _value; | |
351 | |
352 @override | |
353 Token copy() => new StringToken(type, _value, offset); | |
354 | |
355 @override | |
356 String value() => _value; | |
357 } | |
358 | |
359 /** | |
360 * A string token that is preceded by comments. | |
361 */ | |
362 class StringTokenWithComment extends StringToken implements TokenWithComment { | |
363 /** | |
364 * The first comment in the list of comments that precede this token. | |
365 */ | |
366 CommentToken _precedingComment; | |
367 | |
368 /** | |
369 * Initialize a newly created token to have the given [type] at the given | |
370 * [offset] and to be preceded by the comments reachable from the given | |
371 * [comment]. | |
372 */ | |
373 StringTokenWithComment( | |
374 TokenType type, String value, int offset, this._precedingComment) | |
375 : super(type, value, offset) { | |
376 _setCommentParent(_precedingComment); | |
377 } | |
378 | |
379 @override | |
380 CommentToken get precedingComments => _precedingComment; | |
381 | |
382 void set precedingComments(CommentToken comment) { | |
383 _precedingComment = comment; | |
384 _setCommentParent(_precedingComment); | |
385 } | |
386 | |
387 @override | |
388 void applyDelta(int delta) { | |
389 super.applyDelta(delta); | |
390 Token token = precedingComments; | |
391 while (token != null) { | |
392 token.applyDelta(delta); | |
393 token = token.next; | |
394 } | |
395 } | |
396 | |
397 @override | |
398 Token copy() => new StringTokenWithComment( | |
399 type, lexeme, offset, copyComments(precedingComments)); | |
400 } | |
401 | |
402 /** | |
403 * A token whose value is independent of it's type. | |
404 */ | 25 */ |
405 class SyntheticStringToken extends StringToken { | 26 class SyntheticStringToken extends StringToken { |
406 /** | 27 /** |
407 * Initialize a newly created token to represent a token of the given [type] | 28 * Initialize a newly created token to represent a token of the given [type] |
408 * with the given [value] at the given [offset]. | 29 * with the given [value] at the given [offset]. |
409 */ | 30 */ |
410 SyntheticStringToken(TokenType type, String value, int offset) | 31 SyntheticStringToken(TokenType type, String value, int offset) |
411 : super(type, value, offset); | 32 : super(type, value, offset); |
412 | 33 |
413 @override | 34 @override |
414 bool get isSynthetic => true; | 35 bool get isSynthetic => true; |
415 } | 36 } |
416 | |
417 /** | |
418 * The classes (or groups) of tokens with a similar use. | |
419 */ | |
420 class TokenClass { | |
421 /** | |
422 * A value used to indicate that the token type is not part of any specific | |
423 * class of token. | |
424 */ | |
425 static const TokenClass NO_CLASS = const TokenClass('NO_CLASS'); | |
426 | |
427 /** | |
428 * A value used to indicate that the token type is an additive operator. | |
429 */ | |
430 static const TokenClass ADDITIVE_OPERATOR = | |
431 const TokenClass('ADDITIVE_OPERATOR', 13); | |
432 | |
433 /** | |
434 * A value used to indicate that the token type is an assignment operator. | |
435 */ | |
436 static const TokenClass ASSIGNMENT_OPERATOR = | |
437 const TokenClass('ASSIGNMENT_OPERATOR', 1); | |
438 | |
439 /** | |
440 * A value used to indicate that the token type is a bitwise-and operator. | |
441 */ | |
442 static const TokenClass BITWISE_AND_OPERATOR = | |
443 const TokenClass('BITWISE_AND_OPERATOR', 11); | |
444 | |
445 /** | |
446 * A value used to indicate that the token type is a bitwise-or operator. | |
447 */ | |
448 static const TokenClass BITWISE_OR_OPERATOR = | |
449 const TokenClass('BITWISE_OR_OPERATOR', 9); | |
450 | |
451 /** | |
452 * A value used to indicate that the token type is a bitwise-xor operator. | |
453 */ | |
454 static const TokenClass BITWISE_XOR_OPERATOR = | |
455 const TokenClass('BITWISE_XOR_OPERATOR', 10); | |
456 | |
457 /** | |
458 * A value used to indicate that the token type is a cascade operator. | |
459 */ | |
460 static const TokenClass CASCADE_OPERATOR = | |
461 const TokenClass('CASCADE_OPERATOR', 2); | |
462 | |
463 /** | |
464 * A value used to indicate that the token type is a conditional operator. | |
465 */ | |
466 static const TokenClass CONDITIONAL_OPERATOR = | |
467 const TokenClass('CONDITIONAL_OPERATOR', 3); | |
468 | |
469 /** | |
470 * A value used to indicate that the token type is an equality operator. | |
471 */ | |
472 static const TokenClass EQUALITY_OPERATOR = | |
473 const TokenClass('EQUALITY_OPERATOR', 7); | |
474 | |
475 /** | |
476 * A value used to indicate that the token type is an if-null operator. | |
477 */ | |
478 static const TokenClass IF_NULL_OPERATOR = | |
479 const TokenClass('IF_NULL_OPERATOR', 4); | |
480 | |
481 /** | |
482 * A value used to indicate that the token type is a logical-and operator. | |
483 */ | |
484 static const TokenClass LOGICAL_AND_OPERATOR = | |
485 const TokenClass('LOGICAL_AND_OPERATOR', 6); | |
486 | |
487 /** | |
488 * A value used to indicate that the token type is a logical-or operator. | |
489 */ | |
490 static const TokenClass LOGICAL_OR_OPERATOR = | |
491 const TokenClass('LOGICAL_OR_OPERATOR', 5); | |
492 | |
493 /** | |
494 * A value used to indicate that the token type is a multiplicative operator. | |
495 */ | |
496 static const TokenClass MULTIPLICATIVE_OPERATOR = | |
497 const TokenClass('MULTIPLICATIVE_OPERATOR', 14); | |
498 | |
499 /** | |
500 * A value used to indicate that the token type is a relational operator. | |
501 */ | |
502 static const TokenClass RELATIONAL_OPERATOR = | |
503 const TokenClass('RELATIONAL_OPERATOR', 8); | |
504 | |
505 /** | |
506 * A value used to indicate that the token type is a shift operator. | |
507 */ | |
508 static const TokenClass SHIFT_OPERATOR = | |
509 const TokenClass('SHIFT_OPERATOR', 12); | |
510 | |
511 /** | |
512 * A value used to indicate that the token type is a unary operator. | |
513 */ | |
514 static const TokenClass UNARY_POSTFIX_OPERATOR = | |
515 const TokenClass('UNARY_POSTFIX_OPERATOR', 16); | |
516 | |
517 /** | |
518 * A value used to indicate that the token type is a unary operator. | |
519 */ | |
520 static const TokenClass UNARY_PREFIX_OPERATOR = | |
521 const TokenClass('UNARY_PREFIX_OPERATOR', 15); | |
522 | |
523 /** | |
524 * The name of the token class. | |
525 */ | |
526 final String name; | |
527 | |
528 /** | |
529 * The precedence of tokens of this class, or `0` if the such tokens do not | |
530 * represent an operator. | |
531 */ | |
532 final int precedence; | |
533 | |
534 /** | |
535 * Initialize a newly created class of tokens to have the given [name] and | |
536 * [precedence]. | |
537 */ | |
538 const TokenClass(this.name, [this.precedence = 0]); | |
539 | |
540 @override | |
541 String toString() => name; | |
542 } | |
543 | |
544 /** | |
545 * A normal token that is preceded by comments. | |
546 */ | |
547 class TokenWithComment extends SimpleToken { | |
548 /** | |
549 * The first comment in the list of comments that precede this token. | |
550 */ | |
551 CommentToken _precedingComment; | |
552 | |
553 /** | |
554 * Initialize a newly created token to have the given [type] at the given | |
555 * [offset] and to be preceded by the comments reachable from the given | |
556 * [comment]. | |
557 */ | |
558 TokenWithComment(TokenType type, int offset, this._precedingComment) | |
559 : super(type, offset) { | |
560 _setCommentParent(_precedingComment); | |
561 } | |
562 | |
563 @override | |
564 CommentToken get precedingComments => _precedingComment; | |
565 | |
566 void set precedingComments(CommentToken comment) { | |
567 _precedingComment = comment; | |
568 _setCommentParent(_precedingComment); | |
569 } | |
570 | |
571 @override | |
572 Token copy() => | |
573 new TokenWithComment(type, offset, copyComments(precedingComments)); | |
574 } | |
OLD | NEW |