OLD | NEW |
(Empty) | |
| 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. |
| 4 |
| 5 library analyzer.src.dart.ast.token; |
| 6 |
| 7 import 'package:analyzer/dart/ast/token.dart'; |
| 8 import 'package:analyzer/src/generated/java_engine.dart'; |
| 9 |
| 10 /** |
| 11 * The opening half of a grouping pair of tokens. This is used for curly |
| 12 * brackets ('{'), parentheses ('('), and square brackets ('['). |
| 13 */ |
| 14 class BeginToken extends SimpleToken { |
| 15 /** |
| 16 * The token that corresponds to this token. |
| 17 */ |
| 18 Token endToken; |
| 19 |
| 20 /** |
| 21 * Initialize a newly created token to have the given [type] at the given |
| 22 * [offset]. |
| 23 */ |
| 24 BeginToken(TokenType type, int offset) : super(type, offset) { |
| 25 assert(type == TokenType.OPEN_CURLY_BRACKET || |
| 26 type == TokenType.OPEN_PAREN || |
| 27 type == TokenType.OPEN_SQUARE_BRACKET || |
| 28 type == TokenType.STRING_INTERPOLATION_EXPRESSION); |
| 29 } |
| 30 |
| 31 @override |
| 32 Token copy() => new BeginToken(type, offset); |
| 33 } |
| 34 |
| 35 /** |
| 36 * A begin token that is preceded by comments. |
| 37 */ |
| 38 class BeginTokenWithComment extends BeginToken { |
| 39 /** |
| 40 * The first comment in the list of comments that precede this token. |
| 41 */ |
| 42 CommentToken _precedingComment; |
| 43 |
| 44 /** |
| 45 * Initialize a newly created token to have the given [type] at the given |
| 46 * [offset] and to be preceded by the comments reachable from the given |
| 47 * [comment]. |
| 48 */ |
| 49 BeginTokenWithComment(TokenType type, int offset, this._precedingComment) |
| 50 : super(type, offset) { |
| 51 _setCommentParent(_precedingComment); |
| 52 } |
| 53 |
| 54 CommentToken get precedingComments => _precedingComment; |
| 55 |
| 56 void set precedingComments(CommentToken comment) { |
| 57 _precedingComment = comment; |
| 58 _setCommentParent(_precedingComment); |
| 59 } |
| 60 |
| 61 @override |
| 62 void applyDelta(int delta) { |
| 63 super.applyDelta(delta); |
| 64 Token token = precedingComments; |
| 65 while (token != null) { |
| 66 token.applyDelta(delta); |
| 67 token = token.next; |
| 68 } |
| 69 } |
| 70 |
| 71 @override |
| 72 Token copy() => |
| 73 new BeginTokenWithComment(type, offset, copyComments(precedingComments)); |
| 74 } |
| 75 |
| 76 /** |
| 77 * A token representing a comment. |
| 78 */ |
| 79 class CommentToken extends StringToken { |
| 80 /** |
| 81 * The [Token] that contains this comment. |
| 82 */ |
| 83 Token parent; |
| 84 |
| 85 /** |
| 86 * Initialize a newly created token to represent a token of the given [type] |
| 87 * with the given [value] at the given [offset]. |
| 88 */ |
| 89 CommentToken(TokenType type, String value, int offset) |
| 90 : super(type, value, offset); |
| 91 |
| 92 @override |
| 93 CommentToken copy() => new CommentToken(type, _value, offset); |
| 94 } |
| 95 |
| 96 /** |
| 97 * A documentation comment token. |
| 98 */ |
| 99 class DocumentationCommentToken extends CommentToken { |
| 100 /** |
| 101 * The references embedded within the documentation comment. |
| 102 * This list will be empty unless this is a documentation comment that has |
| 103 * references embedded within it. |
| 104 */ |
| 105 final List<Token> references = <Token>[]; |
| 106 |
| 107 /** |
| 108 * Initialize a newly created token to represent a token of the given [type] |
| 109 * with the given [value] at the given [offset]. |
| 110 */ |
| 111 DocumentationCommentToken(TokenType type, String value, int offset) |
| 112 : super(type, value, offset); |
| 113 |
| 114 @override |
| 115 CommentToken copy() { |
| 116 DocumentationCommentToken copy = |
| 117 new DocumentationCommentToken(type, _value, offset); |
| 118 references.forEach((ref) => copy.references.add(ref.copy())); |
| 119 return copy; |
| 120 } |
| 121 } |
| 122 |
| 123 /** |
| 124 * A token representing a keyword in the language. |
| 125 */ |
| 126 class KeywordToken extends SimpleToken { |
| 127 /** |
| 128 * The keyword being represented by this token. |
| 129 */ |
| 130 final Keyword keyword; |
| 131 |
| 132 /** |
| 133 * Initialize a newly created token to represent the given [keyword] at the |
| 134 * given [offset]. |
| 135 */ |
| 136 KeywordToken(this.keyword, int offset) : super(TokenType.KEYWORD, offset); |
| 137 |
| 138 @override |
| 139 String get lexeme => keyword.syntax; |
| 140 |
| 141 @override |
| 142 Token copy() => new KeywordToken(keyword, offset); |
| 143 |
| 144 @override |
| 145 Keyword value() => keyword; |
| 146 } |
| 147 |
| 148 /** |
| 149 * A keyword token that is preceded by comments. |
| 150 */ |
| 151 class KeywordTokenWithComment extends KeywordToken { |
| 152 /** |
| 153 * The first comment in the list of comments that precede this token. |
| 154 */ |
| 155 CommentToken _precedingComment; |
| 156 |
| 157 /** |
| 158 * Initialize a newly created token to to represent the given [keyword] at the |
| 159 * given [offset] and to be preceded by the comments reachable from the given |
| 160 * [comment]. |
| 161 */ |
| 162 KeywordTokenWithComment(Keyword keyword, int offset, this._precedingComment) |
| 163 : super(keyword, offset) { |
| 164 _setCommentParent(_precedingComment); |
| 165 } |
| 166 |
| 167 CommentToken get precedingComments => _precedingComment; |
| 168 |
| 169 void set precedingComments(CommentToken comment) { |
| 170 _precedingComment = comment; |
| 171 _setCommentParent(_precedingComment); |
| 172 } |
| 173 |
| 174 @override |
| 175 void applyDelta(int delta) { |
| 176 super.applyDelta(delta); |
| 177 Token token = precedingComments; |
| 178 while (token != null) { |
| 179 token.applyDelta(delta); |
| 180 token = token.next; |
| 181 } |
| 182 } |
| 183 |
| 184 @override |
| 185 Token copy() => new KeywordTokenWithComment( |
| 186 keyword, offset, copyComments(precedingComments)); |
| 187 } |
| 188 |
| 189 /** |
| 190 * A token that was scanned from the input. Each token knows which tokens |
| 191 * precede and follow it, acting as a link in a doubly linked list of tokens. |
| 192 */ |
| 193 class SimpleToken implements Token { |
| 194 /** |
| 195 * The type of the token. |
| 196 */ |
| 197 final TokenType type; |
| 198 |
| 199 /** |
| 200 * The offset from the beginning of the file to the first character in the |
| 201 * token. |
| 202 */ |
| 203 int offset = 0; |
| 204 |
| 205 /** |
| 206 * The previous token in the token stream. |
| 207 */ |
| 208 Token previous; |
| 209 |
| 210 /** |
| 211 * The next token in the token stream. |
| 212 */ |
| 213 Token _next; |
| 214 |
| 215 /** |
| 216 * Initialize a newly created token to have the given [type] and [offset]. |
| 217 */ |
| 218 SimpleToken(this.type, this.offset); |
| 219 |
| 220 /** |
| 221 * Return the offset from the beginning of the file to the character after the |
| 222 * last character of the token. |
| 223 */ |
| 224 int get end => offset + length; |
| 225 |
| 226 /** |
| 227 * Return `true` if this token represents an operator. |
| 228 */ |
| 229 bool get isOperator => type.isOperator; |
| 230 |
| 231 /** |
| 232 * Return `true` if this token is a synthetic token. A synthetic token is a |
| 233 * token that was introduced by the parser in order to recover from an error |
| 234 * in the code. |
| 235 */ |
| 236 bool get isSynthetic => length == 0; |
| 237 |
| 238 /** |
| 239 * Return `true` if this token represents an operator that can be defined by |
| 240 * users. |
| 241 */ |
| 242 bool get isUserDefinableOperator => type.isUserDefinableOperator; |
| 243 |
| 244 /** |
| 245 * Return the number of characters in the node's source range. |
| 246 */ |
| 247 int get length => lexeme.length; |
| 248 |
| 249 /** |
| 250 * Return the lexeme that represents this token. |
| 251 */ |
| 252 String get lexeme => type.lexeme; |
| 253 |
| 254 /** |
| 255 * Return the next token in the token stream. |
| 256 */ |
| 257 Token get next => _next; |
| 258 |
| 259 /** |
| 260 * Return the first comment in the list of comments that precede this token, |
| 261 * or `null` if there are no comments preceding this token. Additional |
| 262 * comments can be reached by following the token stream using [next] until |
| 263 * `null` is returned. |
| 264 * |
| 265 * For example, if the original contents were `/* one */ /* two */ id`, then |
| 266 * the first preceding comment token will have a lexeme of `/* one */` and |
| 267 * the next comment token will have a lexeme of `/* two */`. |
| 268 */ |
| 269 CommentToken get precedingComments => null; |
| 270 |
| 271 /** |
| 272 * Apply (add) the given [delta] to this token's offset. |
| 273 */ |
| 274 void applyDelta(int delta) { |
| 275 offset += delta; |
| 276 } |
| 277 |
| 278 /** |
| 279 * Return a newly created token that is a copy of this token but that is not a |
| 280 * part of any token stream. |
| 281 */ |
| 282 Token copy() => new Token(type, offset); |
| 283 |
| 284 /** |
| 285 * Copy a linked list of comment tokens identical to the given comment tokens. |
| 286 */ |
| 287 Token copyComments(Token token) { |
| 288 if (token == null) { |
| 289 return null; |
| 290 } |
| 291 Token head = token.copy(); |
| 292 Token tail = head; |
| 293 token = token.next; |
| 294 while (token != null) { |
| 295 tail = tail.setNext(token.copy()); |
| 296 token = token.next; |
| 297 } |
| 298 return head; |
| 299 } |
| 300 |
| 301 /** |
| 302 * Return `true` if this token has any one of the given [types]. |
| 303 */ |
| 304 bool matchesAny(List<TokenType> types) { |
| 305 for (TokenType type in types) { |
| 306 if (this.type == type) { |
| 307 return true; |
| 308 } |
| 309 } |
| 310 return false; |
| 311 } |
| 312 |
| 313 /** |
| 314 * Set the next token in the token stream to the given [token]. This has the |
| 315 * side-effect of setting this token to be the previous token for the given |
| 316 * token. Return the token that was passed in. |
| 317 */ |
| 318 Token setNext(Token token) { |
| 319 _next = token; |
| 320 token.previous = this; |
| 321 return token; |
| 322 } |
| 323 |
| 324 /** |
| 325 * Set the next token in the token stream to the given token without changing |
| 326 * which token is the previous token for the given token. Return the token |
| 327 * that was passed in. |
| 328 */ |
| 329 Token setNextWithoutSettingPrevious(Token token) { |
| 330 _next = token; |
| 331 return token; |
| 332 } |
| 333 |
| 334 @override |
| 335 String toString() => lexeme; |
| 336 |
| 337 /** |
| 338 * Return the value of this token. For keyword tokens, this is the keyword |
| 339 * associated with the token, for other tokens it is the lexeme associated |
| 340 * with the token. |
| 341 */ |
| 342 Object value() => type.lexeme; |
| 343 |
| 344 /** |
| 345 * Sets the `parent` property to `this` for the given [comment] and all the |
| 346 * next tokens. |
| 347 */ |
| 348 void _setCommentParent(CommentToken comment) { |
| 349 while (comment != null) { |
| 350 comment.parent = this; |
| 351 comment = comment.next; |
| 352 } |
| 353 } |
| 354 } |
| 355 |
| 356 /** |
| 357 * A token whose value is independent of it's type. |
| 358 */ |
| 359 class StringToken extends SimpleToken { |
| 360 /** |
| 361 * The lexeme represented by this token. |
| 362 */ |
| 363 String _value; |
| 364 |
| 365 /** |
| 366 * Initialize a newly created token to represent a token of the given [type] |
| 367 * with the given [value] at the given [offset]. |
| 368 */ |
| 369 StringToken(TokenType type, String value, int offset) : super(type, offset) { |
| 370 this._value = StringUtilities.intern(value); |
| 371 } |
| 372 |
| 373 @override |
| 374 String get lexeme => _value; |
| 375 |
| 376 @override |
| 377 Token copy() => new StringToken(type, _value, offset); |
| 378 |
| 379 @override |
| 380 String value() => _value; |
| 381 } |
| 382 |
| 383 /** |
| 384 * A string token that is preceded by comments. |
| 385 */ |
| 386 class StringTokenWithComment extends StringToken { |
| 387 /** |
| 388 * The first comment in the list of comments that precede this token. |
| 389 */ |
| 390 CommentToken _precedingComment; |
| 391 |
| 392 /** |
| 393 * Initialize a newly created token to have the given [type] at the given |
| 394 * [offset] and to be preceded by the comments reachable from the given |
| 395 * [comment]. |
| 396 */ |
| 397 StringTokenWithComment( |
| 398 TokenType type, String value, int offset, this._precedingComment) |
| 399 : super(type, value, offset) { |
| 400 _setCommentParent(_precedingComment); |
| 401 } |
| 402 |
| 403 CommentToken get precedingComments => _precedingComment; |
| 404 |
| 405 void set precedingComments(CommentToken comment) { |
| 406 _precedingComment = comment; |
| 407 _setCommentParent(_precedingComment); |
| 408 } |
| 409 |
| 410 @override |
| 411 void applyDelta(int delta) { |
| 412 super.applyDelta(delta); |
| 413 Token token = precedingComments; |
| 414 while (token != null) { |
| 415 token.applyDelta(delta); |
| 416 token = token.next; |
| 417 } |
| 418 } |
| 419 |
| 420 @override |
| 421 Token copy() => new StringTokenWithComment( |
| 422 type, lexeme, offset, copyComments(precedingComments)); |
| 423 } |
| 424 |
| 425 /** |
| 426 * A token whose value is independent of it's type. |
| 427 */ |
| 428 class SyntheticStringToken extends StringToken { |
| 429 /** |
| 430 * Initialize a newly created token to represent a token of the given [type] |
| 431 * with the given [value] at the given [offset]. |
| 432 */ |
| 433 SyntheticStringToken(TokenType type, String value, int offset) |
| 434 : super(type, value, offset); |
| 435 |
| 436 @override |
| 437 bool get isSynthetic => true; |
| 438 } |
| 439 |
| 440 /** |
| 441 * The classes (or groups) of tokens with a similar use. |
| 442 */ |
| 443 class TokenClass { |
| 444 /** |
| 445 * A value used to indicate that the token type is not part of any specific |
| 446 * class of token. |
| 447 */ |
| 448 static const TokenClass NO_CLASS = const TokenClass('NO_CLASS'); |
| 449 |
| 450 /** |
| 451 * A value used to indicate that the token type is an additive operator. |
| 452 */ |
| 453 static const TokenClass ADDITIVE_OPERATOR = |
| 454 const TokenClass('ADDITIVE_OPERATOR', 13); |
| 455 |
| 456 /** |
| 457 * A value used to indicate that the token type is an assignment operator. |
| 458 */ |
| 459 static const TokenClass ASSIGNMENT_OPERATOR = |
| 460 const TokenClass('ASSIGNMENT_OPERATOR', 1); |
| 461 |
| 462 /** |
| 463 * A value used to indicate that the token type is a bitwise-and operator. |
| 464 */ |
| 465 static const TokenClass BITWISE_AND_OPERATOR = |
| 466 const TokenClass('BITWISE_AND_OPERATOR', 11); |
| 467 |
| 468 /** |
| 469 * A value used to indicate that the token type is a bitwise-or operator. |
| 470 */ |
| 471 static const TokenClass BITWISE_OR_OPERATOR = |
| 472 const TokenClass('BITWISE_OR_OPERATOR', 9); |
| 473 |
| 474 /** |
| 475 * A value used to indicate that the token type is a bitwise-xor operator. |
| 476 */ |
| 477 static const TokenClass BITWISE_XOR_OPERATOR = |
| 478 const TokenClass('BITWISE_XOR_OPERATOR', 10); |
| 479 |
| 480 /** |
| 481 * A value used to indicate that the token type is a cascade operator. |
| 482 */ |
| 483 static const TokenClass CASCADE_OPERATOR = |
| 484 const TokenClass('CASCADE_OPERATOR', 2); |
| 485 |
| 486 /** |
| 487 * A value used to indicate that the token type is a conditional operator. |
| 488 */ |
| 489 static const TokenClass CONDITIONAL_OPERATOR = |
| 490 const TokenClass('CONDITIONAL_OPERATOR', 3); |
| 491 |
| 492 /** |
| 493 * A value used to indicate that the token type is an equality operator. |
| 494 */ |
| 495 static const TokenClass EQUALITY_OPERATOR = |
| 496 const TokenClass('EQUALITY_OPERATOR', 7); |
| 497 |
| 498 /** |
| 499 * A value used to indicate that the token type is an if-null operator. |
| 500 */ |
| 501 static const TokenClass IF_NULL_OPERATOR = |
| 502 const TokenClass('IF_NULL_OPERATOR', 4); |
| 503 |
| 504 /** |
| 505 * A value used to indicate that the token type is a logical-and operator. |
| 506 */ |
| 507 static const TokenClass LOGICAL_AND_OPERATOR = |
| 508 const TokenClass('LOGICAL_AND_OPERATOR', 6); |
| 509 |
| 510 /** |
| 511 * A value used to indicate that the token type is a logical-or operator. |
| 512 */ |
| 513 static const TokenClass LOGICAL_OR_OPERATOR = |
| 514 const TokenClass('LOGICAL_OR_OPERATOR', 5); |
| 515 |
| 516 /** |
| 517 * A value used to indicate that the token type is a multiplicative operator. |
| 518 */ |
| 519 static const TokenClass MULTIPLICATIVE_OPERATOR = |
| 520 const TokenClass('MULTIPLICATIVE_OPERATOR', 14); |
| 521 |
| 522 /** |
| 523 * A value used to indicate that the token type is a relational operator. |
| 524 */ |
| 525 static const TokenClass RELATIONAL_OPERATOR = |
| 526 const TokenClass('RELATIONAL_OPERATOR', 8); |
| 527 |
| 528 /** |
| 529 * A value used to indicate that the token type is a shift operator. |
| 530 */ |
| 531 static const TokenClass SHIFT_OPERATOR = |
| 532 const TokenClass('SHIFT_OPERATOR', 12); |
| 533 |
| 534 /** |
| 535 * A value used to indicate that the token type is a unary operator. |
| 536 */ |
| 537 static const TokenClass UNARY_POSTFIX_OPERATOR = |
| 538 const TokenClass('UNARY_POSTFIX_OPERATOR', 16); |
| 539 |
| 540 /** |
| 541 * A value used to indicate that the token type is a unary operator. |
| 542 */ |
| 543 static const TokenClass UNARY_PREFIX_OPERATOR = |
| 544 const TokenClass('UNARY_PREFIX_OPERATOR', 15); |
| 545 |
| 546 /** |
| 547 * The name of the token class. |
| 548 */ |
| 549 final String name; |
| 550 |
| 551 /** |
| 552 * The precedence of tokens of this class, or `0` if the such tokens do not |
| 553 * represent an operator. |
| 554 */ |
| 555 final int precedence; |
| 556 |
| 557 /** |
| 558 * Initialize a newly created class of tokens to have the given [name] and |
| 559 * [precedence]. |
| 560 */ |
| 561 const TokenClass(this.name, [this.precedence = 0]); |
| 562 |
| 563 @override |
| 564 String toString() => name; |
| 565 } |
| 566 |
| 567 /** |
| 568 * A normal token that is preceded by comments. |
| 569 */ |
| 570 class TokenWithComment extends SimpleToken { |
| 571 /** |
| 572 * The first comment in the list of comments that precede this token. |
| 573 */ |
| 574 CommentToken _precedingComment; |
| 575 |
| 576 /** |
| 577 * Initialize a newly created token to have the given [type] at the given |
| 578 * [offset] and to be preceded by the comments reachable from the given |
| 579 * [comment]. |
| 580 */ |
| 581 TokenWithComment(TokenType type, int offset, this._precedingComment) |
| 582 : super(type, offset) { |
| 583 _setCommentParent(_precedingComment); |
| 584 } |
| 585 |
| 586 CommentToken get precedingComments => _precedingComment; |
| 587 |
| 588 void set precedingComments(CommentToken comment) { |
| 589 _precedingComment = comment; |
| 590 _setCommentParent(_precedingComment); |
| 591 } |
| 592 |
| 593 @override |
| 594 Token copy() => new TokenWithComment(type, offset, precedingComments); |
| 595 } |
OLD | NEW |