| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 part of scanner; | 5 part of scanner; |
| 6 | 6 |
| 7 abstract class Scanner { | 7 abstract class Scanner { |
| 8 Token tokenize(); | 8 Token tokenize(); |
| 9 | 9 |
| 10 factory Scanner(SourceFile file, {bool includeComments: false}) { | 10 factory Scanner(SourceFile file, |
| 11 {bool includeComments: false, bool enableNullAwareOperators: false}) { |
| 11 if (file is Utf8BytesSourceFile) { | 12 if (file is Utf8BytesSourceFile) { |
| 12 return new Utf8BytesScanner(file, includeComments: includeComments); | 13 return new Utf8BytesScanner(file, includeComments: includeComments, |
| 14 enableNullAwareOperators: enableNullAwareOperators); |
| 13 } else { | 15 } else { |
| 14 return new StringScanner(file, includeComments: includeComments); | 16 return new StringScanner(file, includeComments: includeComments, |
| 17 enableNullAwareOperators: enableNullAwareOperators); |
| 15 } | 18 } |
| 16 } | 19 } |
| 17 } | 20 } |
| 18 | 21 |
| 19 abstract class AbstractScanner implements Scanner { | 22 abstract class AbstractScanner implements Scanner { |
| 20 // TODO(ahe): Move this class to implementation. | 23 // TODO(ahe): Move this class to implementation. |
| 21 | 24 |
| 22 final bool includeComments; | 25 final bool includeComments; |
| 26 final bool enableNullAwareOperators; |
| 23 | 27 |
| 24 /** | 28 /** |
| 25 * The string offset for the next token that will be created. | 29 * The string offset for the next token that will be created. |
| 26 * | 30 * |
| 27 * Note that in the [Utf8BytesScanner], [stringOffset] and [scanOffset] values | 31 * Note that in the [Utf8BytesScanner], [stringOffset] and [scanOffset] values |
| 28 * are different. One string character can be encoded using multiple UTF-8 | 32 * are different. One string character can be encoded using multiple UTF-8 |
| 29 * bytes. | 33 * bytes. |
| 30 */ | 34 */ |
| 31 int tokenStart = -1; | 35 int tokenStart = -1; |
| 32 | 36 |
| (...skipping 13 matching lines...) Expand all Loading... |
| 46 | 50 |
| 47 /** | 51 /** |
| 48 * The source file that is being scanned. This field can be [:null:]. | 52 * The source file that is being scanned. This field can be [:null:]. |
| 49 * If the source file is available, the scanner assigns its [:lineStarts:] and | 53 * If the source file is available, the scanner assigns its [:lineStarts:] and |
| 50 * [:length:] fields at the end of [tokenize]. | 54 * [:length:] fields at the end of [tokenize]. |
| 51 */ | 55 */ |
| 52 final SourceFile file; | 56 final SourceFile file; |
| 53 | 57 |
| 54 final List<int> lineStarts = <int>[0]; | 58 final List<int> lineStarts = <int>[0]; |
| 55 | 59 |
| 56 AbstractScanner(this.file, this.includeComments) { | 60 AbstractScanner( |
| 61 this.file, this.includeComments, this.enableNullAwareOperators) { |
| 57 this.tail = this.tokens; | 62 this.tail = this.tokens; |
| 58 } | 63 } |
| 59 | 64 |
| 60 /** | 65 /** |
| 61 * Advances and returns the next character. | 66 * Advances and returns the next character. |
| 62 * | 67 * |
| 63 * If the next character is non-ASCII, then the returned value depends on the | 68 * If the next character is non-ASCII, then the returned value depends on the |
| 64 * scanner implementation. The [Utf8BytesScanner] returns a UTF-8 byte, while | 69 * scanner implementation. The [Utf8BytesScanner] returns a UTF-8 byte, while |
| 65 * the [StringScanner] returns a UTF-16 code unit. | 70 * the [StringScanner] returns a UTF-16 code unit. |
| 66 * | 71 * |
| (...skipping 256 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 323 } | 328 } |
| 324 | 329 |
| 325 if (identical(next, $SEMICOLON)) { | 330 if (identical(next, $SEMICOLON)) { |
| 326 appendPrecedenceToken(SEMICOLON_INFO); | 331 appendPrecedenceToken(SEMICOLON_INFO); |
| 327 // Type parameters and arguments cannot contain semicolon. | 332 // Type parameters and arguments cannot contain semicolon. |
| 328 discardOpenLt(); | 333 discardOpenLt(); |
| 329 return advance(); | 334 return advance(); |
| 330 } | 335 } |
| 331 | 336 |
| 332 if (identical(next, $QUESTION)) { | 337 if (identical(next, $QUESTION)) { |
| 333 appendPrecedenceToken(QUESTION_INFO); | 338 return tokenizeQuestion(next); |
| 334 return advance(); | |
| 335 } | 339 } |
| 336 | 340 |
| 337 if (identical(next, $CLOSE_SQUARE_BRACKET)) { | 341 if (identical(next, $CLOSE_SQUARE_BRACKET)) { |
| 338 return appendEndGroup(CLOSE_SQUARE_BRACKET_INFO, | 342 return appendEndGroup(CLOSE_SQUARE_BRACKET_INFO, |
| 339 OPEN_SQUARE_BRACKET_TOKEN); | 343 OPEN_SQUARE_BRACKET_TOKEN); |
| 340 } | 344 } |
| 341 | 345 |
| 342 if (identical(next, $BACKPING)) { | 346 if (identical(next, $BACKPING)) { |
| 343 appendPrecedenceToken(BACKPING_INFO); | 347 appendPrecedenceToken(BACKPING_INFO); |
| 344 return advance(); | 348 return advance(); |
| (...skipping 98 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 443 } | 447 } |
| 444 appendBeginGroup(OPEN_SQUARE_BRACKET_INFO); | 448 appendBeginGroup(OPEN_SQUARE_BRACKET_INFO); |
| 445 return next; | 449 return next; |
| 446 } | 450 } |
| 447 | 451 |
| 448 int tokenizeCaret(int next) { | 452 int tokenizeCaret(int next) { |
| 449 // ^ ^= | 453 // ^ ^= |
| 450 return select($EQ, CARET_EQ_INFO, CARET_INFO); | 454 return select($EQ, CARET_EQ_INFO, CARET_INFO); |
| 451 } | 455 } |
| 452 | 456 |
| 457 int tokenizeQuestion(int next) { |
| 458 // ? ?. ?? ??= |
| 459 next = advance(); |
| 460 if (identical(next, $QUESTION)) { |
| 461 if (enableNullAwareOperators) { |
| 462 return select($EQ, QUESTION_QUESTION_EQ_INFO, QUESTION_QUESTION_INFO); |
| 463 } else { |
| 464 next = advance(); |
| 465 PrecedenceInfo info; |
| 466 if (identical(next, $EQ)) { |
| 467 info = QUESTION_QUESTION_EQ_INFO; |
| 468 next = advance(); |
| 469 } else { |
| 470 info = QUESTION_QUESTION_INFO; |
| 471 } |
| 472 appendErrorToken(new UnsupportedNullAwareToken(info.value, tokenStart)); |
| 473 return next; |
| 474 } |
| 475 } else if (identical(next, $PERIOD)) { |
| 476 if (enableNullAwareOperators) { |
| 477 appendPrecedenceToken(QUESTION_PERIOD_INFO); |
| 478 } else { |
| 479 appendErrorToken(new UnsupportedNullAwareToken( |
| 480 QUESTION_PERIOD_INFO.value, tokenStart)); |
| 481 } |
| 482 return advance(); |
| 483 } else { |
| 484 appendPrecedenceToken(QUESTION_INFO); |
| 485 return next; |
| 486 } |
| 487 } |
| 488 |
| 453 int tokenizeBar(int next) { | 489 int tokenizeBar(int next) { |
| 454 // | || |= | 490 // | || |= |
| 455 next = advance(); | 491 next = advance(); |
| 456 if (identical(next, $BAR)) { | 492 if (identical(next, $BAR)) { |
| 457 appendPrecedenceToken(BAR_BAR_INFO); | 493 appendPrecedenceToken(BAR_BAR_INFO); |
| 458 return advance(); | 494 return advance(); |
| 459 } else if (identical(next, $EQ)) { | 495 } else if (identical(next, $EQ)) { |
| 460 appendPrecedenceToken(BAR_EQ_INFO); | 496 appendPrecedenceToken(BAR_EQ_INFO); |
| 461 return advance(); | 497 return advance(); |
| 462 } else { | 498 } else { |
| (...skipping 684 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1147 | 1183 |
| 1148 PrecedenceInfo closeBraceInfoFor(BeginGroupToken begin) { | 1184 PrecedenceInfo closeBraceInfoFor(BeginGroupToken begin) { |
| 1149 return const { | 1185 return const { |
| 1150 '(': CLOSE_PAREN_INFO, | 1186 '(': CLOSE_PAREN_INFO, |
| 1151 '[': CLOSE_SQUARE_BRACKET_INFO, | 1187 '[': CLOSE_SQUARE_BRACKET_INFO, |
| 1152 '{': CLOSE_CURLY_BRACKET_INFO, | 1188 '{': CLOSE_CURLY_BRACKET_INFO, |
| 1153 '<': GT_INFO, | 1189 '<': GT_INFO, |
| 1154 r'${': CLOSE_CURLY_BRACKET_INFO, | 1190 r'${': CLOSE_CURLY_BRACKET_INFO, |
| 1155 }[begin.value]; | 1191 }[begin.value]; |
| 1156 } | 1192 } |
| OLD | NEW |