Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(993)

Side by Side Diff: pkg/analyzer/lib/src/dart/scanner/scanner.dart

Issue 2199323002: Initial implementation for lazy compound assignment operators (Closed) Base URL: https://github.com/dart-lang/sdk.git@master
Patch Set: Created 4 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 library analyzer.src.dart.scanner.scanner; 5 library analyzer.src.dart.scanner.scanner;
6 6
7 import 'package:analyzer/dart/ast/token.dart'; 7 import 'package:analyzer/dart/ast/token.dart';
8 import 'package:analyzer/src/dart/ast/token.dart'; 8 import 'package:analyzer/src/dart/ast/token.dart';
9 import 'package:analyzer/src/dart/scanner/reader.dart'; 9 import 'package:analyzer/src/dart/scanner/reader.dart';
10 import 'package:analyzer/src/generated/error.dart'; 10 import 'package:analyzer/src/generated/error.dart';
11 import 'package:analyzer/src/generated/java_engine.dart'; 11 import 'package:analyzer/src/generated/java_engine.dart';
12 import 'package:analyzer/src/generated/source.dart'; 12 import 'package:analyzer/src/generated/source.dart';
13 import 'package:charcode/ascii.dart';
13 14
14 /** 15 /**
15 * A state in a state machine used to scan keywords. 16 * A state in a state machine used to scan keywords.
16 */ 17 */
17 class KeywordState { 18 class KeywordState {
18 /** 19 /**
19 * An empty transition table used by leaf states. 20 * An empty transition table used by leaf states.
20 */ 21 */
21 static List<KeywordState> _EMPTY_TABLE = new List<KeywordState>(26); 22 static List<KeywordState> _EMPTY_TABLE = new List<KeywordState>(26);
22 23
(...skipping 27 matching lines...) Expand all
50 * Return the keyword that was recognized by this state, or `null` if this 51 * Return the keyword that was recognized by this state, or `null` if this
51 * state does not recognized a keyword. 52 * state does not recognized a keyword.
52 */ 53 */
53 Keyword keyword() => _keyword; 54 Keyword keyword() => _keyword;
54 55
55 /** 56 /**
56 * Return the state that follows this state on a transition of the given 57 * Return the state that follows this state on a transition of the given
57 * [character], or `null` if there is no valid state reachable from this state 58 * [character], or `null` if there is no valid state reachable from this state
58 * with such a transition. 59 * with such a transition.
59 */ 60 */
60 KeywordState next(int character) => _table[character - 0x61]; 61 KeywordState next(int character) => _table[character - $a];
61 62
62 /** 63 /**
63 * Create the next state in the state machine where we have already recognized 64 * Create the next state in the state machine where we have already recognized
64 * the subset of strings in the given array of [strings] starting at the given 65 * the subset of strings in the given array of [strings] starting at the given
65 * [offset] and having the given [length]. All of these strings have a common 66 * [offset] and having the given [length]. All of these strings have a common
66 * prefix and the next character is at the given [start] index. 67 * prefix and the next character is at the given [start] index.
67 */ 68 */
68 static KeywordState _computeKeywordStateTable( 69 static KeywordState _computeKeywordStateTable(
69 int start, List<String> strings, int offset, int length) { 70 int start, List<String> strings, int offset, int length) {
70 List<KeywordState> result = new List<KeywordState>(26); 71 List<KeywordState> result = new List<KeywordState>(26);
71 assert(length != 0); 72 assert(length != 0);
72 int chunk = 0x0; 73 int chunk = $nul;
73 int chunkStart = -1; 74 int chunkStart = -1;
74 bool isLeaf = false; 75 bool isLeaf = false;
75 for (int i = offset; i < offset + length; i++) { 76 for (int i = offset; i < offset + length; i++) {
76 if (strings[i].length == start) { 77 if (strings[i].length == start) {
77 isLeaf = true; 78 isLeaf = true;
78 } 79 }
79 if (strings[i].length > start) { 80 if (strings[i].length > start) {
80 int c = strings[i].codeUnitAt(start); 81 int c = strings[i].codeUnitAt(start);
81 if (chunk != c) { 82 if (chunk != c) {
82 if (chunkStart != -1) { 83 if (chunkStart != -1) {
83 result[chunk - 0x61] = _computeKeywordStateTable( 84 result[chunk - $a] = _computeKeywordStateTable(
84 start + 1, strings, chunkStart, i - chunkStart); 85 start + 1, strings, chunkStart, i - chunkStart);
85 } 86 }
86 chunkStart = i; 87 chunkStart = i;
87 chunk = c; 88 chunk = c;
88 } 89 }
89 } 90 }
90 } 91 }
91 if (chunkStart != -1) { 92 if (chunkStart != -1) {
92 assert(result[chunk - 0x61] == null); 93 assert(result[chunk - $a] == null);
93 result[chunk - 0x61] = _computeKeywordStateTable( 94 result[chunk - $a] = _computeKeywordStateTable(
94 start + 1, strings, chunkStart, offset + length - chunkStart); 95 start + 1, strings, chunkStart, offset + length - chunkStart);
95 } else { 96 } else {
96 assert(length == 1); 97 assert(length == 1);
97 return new KeywordState(_EMPTY_TABLE, strings[offset]); 98 return new KeywordState(_EMPTY_TABLE, strings[offset]);
98 } 99 }
99 if (isLeaf) { 100 if (isLeaf) {
100 return new KeywordState(result, strings[offset]); 101 return new KeywordState(result, strings[offset]);
101 } else { 102 } else {
102 return new KeywordState(result, null); 103 return new KeywordState(result, null);
103 } 104 }
(...skipping 96 matching lines...) Expand 10 before | Expand all | Expand 10 after
200 */ 201 */
201 bool _hasUnmatchedGroups = false; 202 bool _hasUnmatchedGroups = false;
202 203
203 /** 204 /**
204 * A flag indicating whether to parse generic method comments, of the form 205 * A flag indicating whether to parse generic method comments, of the form
205 * `/*=T*/` and `/*<T>*/`. 206 * `/*=T*/` and `/*<T>*/`.
206 */ 207 */
207 bool scanGenericMethodComments = false; 208 bool scanGenericMethodComments = false;
208 209
209 /** 210 /**
211 * A flag indicating whether the lazy compound assignment operators '&&=' and
212 * '||=' are enabled.
213 */
214 bool scanLazyAssignmentOperators = false;
215
216 /**
210 * Initialize a newly created scanner to scan characters from the given 217 * Initialize a newly created scanner to scan characters from the given
211 * [source]. The given character [_reader] will be used to read the characters 218 * [source]. The given character [_reader] will be used to read the characters
212 * in the source. The given [_errorListener] will be informed of any errors 219 * in the source. The given [_errorListener] will be informed of any errors
213 * that are found. 220 * that are found.
214 */ 221 */
215 Scanner(this.source, this._reader, this._errorListener) { 222 Scanner(this.source, this._reader, this._errorListener) {
216 _tokens = new Token(TokenType.EOF, -1); 223 _tokens = new Token(TokenType.EOF, -1);
217 _tokens.setNext(_tokens); 224 _tokens.setNext(_tokens);
218 _tail = _tokens; 225 _tail = _tokens;
219 _tokenStart = -1; 226 _tokenStart = -1;
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after
253 * method is intended to be used by subclasses that copy existing tokens and 260 * method is intended to be used by subclasses that copy existing tokens and
254 * should not normally be used because it will fail to correctly associate any 261 * should not normally be used because it will fail to correctly associate any
255 * comments with the token being passed in. 262 * comments with the token being passed in.
256 */ 263 */
257 void appendToken(Token token) { 264 void appendToken(Token token) {
258 _tail = _tail.setNext(token); 265 _tail = _tail.setNext(token);
259 } 266 }
260 267
261 int bigSwitch(int next) { 268 int bigSwitch(int next) {
262 _beginToken(); 269 _beginToken();
263 if (next == 0xD) { 270 if (next == $cr) {
264 // '\r' 271 // '\r'
265 next = _reader.advance(); 272 next = _reader.advance();
266 if (next == 0xA) { 273 if (next == $lf) {
267 // '\n' 274 // '\n'
268 next = _reader.advance(); 275 next = _reader.advance();
269 } 276 }
270 recordStartOfLine(); 277 recordStartOfLine();
271 return next; 278 return next;
272 } else if (next == 0xA) { 279 } else if (next == $lf) {
273 // '\n' 280 // '\n'
274 next = _reader.advance(); 281 next = _reader.advance();
275 recordStartOfLine(); 282 recordStartOfLine();
276 return next; 283 return next;
277 } else if (next == 0x9 || next == 0x20) { 284 } else if (next == $tab || next == $space) {
278 // '\t' || ' ' 285 // '\t' || ' '
279 return _reader.advance(); 286 return _reader.advance();
280 } 287 }
281 if (next == 0x72) { 288 if (next == $r) {
282 // 'r' 289 // 'r'
283 int peek = _reader.peek(); 290 int peek = _reader.peek();
284 if (peek == 0x22 || peek == 0x27) { 291 if (peek == $double_quote || peek == $single_quote) {
285 // '"' || "'" 292 // '"' || "'"
286 int start = _reader.offset; 293 int start = _reader.offset;
287 return _tokenizeString(_reader.advance(), start, true); 294 return _tokenizeString(_reader.advance(), start, true);
288 } 295 }
289 } 296 }
290 if (0x61 <= next && next <= 0x7A) { 297 if ($a <= next && next <= $z) {
291 // 'a'-'z' 298 // 'a'-'z'
292 return _tokenizeKeywordOrIdentifier(next, true); 299 return _tokenizeKeywordOrIdentifier(next, true);
293 } 300 }
294 if ((0x41 <= next && next <= 0x5A) || next == 0x5F || next == 0x24) { 301 if (($A <= next && next <= $Z) || next == $_ || next == $$) {
295 // 'A'-'Z' || '_' || '$' 302 // 'A'-'Z' || '_' || '$'
296 return _tokenizeIdentifier(next, _reader.offset, true); 303 return _tokenizeIdentifier(next, _reader.offset, true);
297 } 304 }
298 if (next == 0x3C) { 305 if (next == $lt) {
299 // '<' 306 // '<'
300 return _tokenizeLessThan(next); 307 return _tokenizeLessThan(next);
301 } 308 }
302 if (next == 0x3E) { 309 if (next == $gt) {
303 // '>' 310 // '>'
304 return _tokenizeGreaterThan(next); 311 return _tokenizeGreaterThan(next);
305 } 312 }
306 if (next == 0x3D) { 313 if (next == $equal) {
307 // '=' 314 // '='
308 return _tokenizeEquals(next); 315 return _tokenizeEquals(next);
309 } 316 }
310 if (next == 0x21) { 317 if (next == $exclamation) {
311 // '!' 318 // '!'
312 return _tokenizeExclamation(next); 319 return _tokenizeExclamation(next);
313 } 320 }
314 if (next == 0x2B) { 321 if (next == $plus) {
315 // '+' 322 // '+'
316 return _tokenizePlus(next); 323 return _tokenizePlus(next);
317 } 324 }
318 if (next == 0x2D) { 325 if (next == $minus) {
319 // '-' 326 // '-'
320 return _tokenizeMinus(next); 327 return _tokenizeMinus(next);
321 } 328 }
322 if (next == 0x2A) { 329 if (next == $asterisk) {
323 // '*' 330 // '*'
324 return _tokenizeMultiply(next); 331 return _tokenizeMultiply(next);
325 } 332 }
326 if (next == 0x25) { 333 if (next == $percent) {
327 // '%' 334 // '%'
328 return _tokenizePercent(next); 335 return _tokenizePercent(next);
329 } 336 }
330 if (next == 0x26) { 337 if (next == $ampersand) {
331 // '&' 338 // '&'
332 return _tokenizeAmpersand(next); 339 return _tokenizeAmpersand(next);
333 } 340 }
334 if (next == 0x7C) { 341 if (next == $bar) {
335 // '|' 342 // '|'
336 return _tokenizeBar(next); 343 return _tokenizeBar(next);
337 } 344 }
338 if (next == 0x5E) { 345 if (next == $caret) {
339 // '^' 346 // '^'
340 return _tokenizeCaret(next); 347 return _tokenizeCaret(next);
341 } 348 }
342 if (next == 0x5B) { 349 if (next == $open_bracket) {
343 // '[' 350 // '['
344 return _tokenizeOpenSquareBracket(next); 351 return _tokenizeOpenSquareBracket(next);
345 } 352 }
346 if (next == 0x7E) { 353 if (next == $tilde) {
347 // '~' 354 // '~'
348 return _tokenizeTilde(next); 355 return _tokenizeTilde(next);
349 } 356 }
350 if (next == 0x5C) { 357 if (next == $backslash) {
351 // '\\' 358 // '\\'
352 _appendTokenOfType(TokenType.BACKSLASH); 359 _appendTokenOfType(TokenType.BACKSLASH);
353 return _reader.advance(); 360 return _reader.advance();
354 } 361 }
355 if (next == 0x23) { 362 if (next == $hash) {
356 // '#' 363 // '#'
357 return _tokenizeTag(next); 364 return _tokenizeTag(next);
358 } 365 }
359 if (next == 0x28) { 366 if (next == $open_paren) {
360 // '(' 367 // '('
361 _appendBeginToken(TokenType.OPEN_PAREN); 368 _appendBeginToken(TokenType.OPEN_PAREN);
362 return _reader.advance(); 369 return _reader.advance();
363 } 370 }
364 if (next == 0x29) { 371 if (next == $close_paren) {
365 // ')' 372 // ')'
366 _appendEndToken(TokenType.CLOSE_PAREN, TokenType.OPEN_PAREN); 373 _appendEndToken(TokenType.CLOSE_PAREN, TokenType.OPEN_PAREN);
367 return _reader.advance(); 374 return _reader.advance();
368 } 375 }
369 if (next == 0x2C) { 376 if (next == $comma) {
370 // ',' 377 // ','
371 _appendTokenOfType(TokenType.COMMA); 378 _appendTokenOfType(TokenType.COMMA);
372 return _reader.advance(); 379 return _reader.advance();
373 } 380 }
374 if (next == 0x3A) { 381 if (next == $colon) {
375 // ':' 382 // ':'
376 _appendTokenOfType(TokenType.COLON); 383 _appendTokenOfType(TokenType.COLON);
377 return _reader.advance(); 384 return _reader.advance();
378 } 385 }
379 if (next == 0x3B) { 386 if (next == $semicolon) {
380 // ';' 387 // ';'
381 _appendTokenOfType(TokenType.SEMICOLON); 388 _appendTokenOfType(TokenType.SEMICOLON);
382 return _reader.advance(); 389 return _reader.advance();
383 } 390 }
384 if (next == 0x3F) { 391 if (next == $question) {
385 // '?' 392 // '?'
386 return _tokenizeQuestion(); 393 return _tokenizeQuestion();
387 } 394 }
388 if (next == 0x5D) { 395 if (next == $close_bracket) {
389 // ']' 396 // ']'
390 _appendEndToken( 397 _appendEndToken(
391 TokenType.CLOSE_SQUARE_BRACKET, TokenType.OPEN_SQUARE_BRACKET); 398 TokenType.CLOSE_SQUARE_BRACKET, TokenType.OPEN_SQUARE_BRACKET);
392 return _reader.advance(); 399 return _reader.advance();
393 } 400 }
394 if (next == 0x60) { 401 if (next == $backquote) {
395 // '`' 402 // '`'
396 _appendTokenOfType(TokenType.BACKPING); 403 _appendTokenOfType(TokenType.BACKPING);
397 return _reader.advance(); 404 return _reader.advance();
398 } 405 }
399 if (next == 0x7B) { 406 if (next == $lbrace) {
400 // '{' 407 // '{'
401 _appendBeginToken(TokenType.OPEN_CURLY_BRACKET); 408 _appendBeginToken(TokenType.OPEN_CURLY_BRACKET);
402 return _reader.advance(); 409 return _reader.advance();
403 } 410 }
404 if (next == 0x7D) { 411 if (next == $rbrace) {
405 // '}' 412 // '}'
406 _appendEndToken( 413 _appendEndToken(
407 TokenType.CLOSE_CURLY_BRACKET, TokenType.OPEN_CURLY_BRACKET); 414 TokenType.CLOSE_CURLY_BRACKET, TokenType.OPEN_CURLY_BRACKET);
408 return _reader.advance(); 415 return _reader.advance();
409 } 416 }
410 if (next == 0x2F) { 417 if (next == $slash) {
411 // '/' 418 // '/'
412 return _tokenizeSlashOrComment(next); 419 return _tokenizeSlashOrComment(next);
413 } 420 }
414 if (next == 0x40) { 421 if (next == $at) {
415 // '@' 422 // '@'
416 _appendTokenOfType(TokenType.AT); 423 _appendTokenOfType(TokenType.AT);
417 return _reader.advance(); 424 return _reader.advance();
418 } 425 }
419 if (next == 0x22 || next == 0x27) { 426 if (next == $double_quote || next == $single_quote) {
420 // '"' || "'" 427 // '"' || "'"
421 return _tokenizeString(next, _reader.offset, false); 428 return _tokenizeString(next, _reader.offset, false);
422 } 429 }
423 if (next == 0x2E) { 430 if (next == $dot) {
424 // '.' 431 // '.'
425 return _tokenizeDotOrNumber(next); 432 return _tokenizeDotOrNumber(next);
426 } 433 }
427 if (next == 0x30) { 434 if (next == $0) {
428 // '0' 435 // '0'
429 return _tokenizeHexOrNumber(next); 436 return _tokenizeHexOrNumber(next);
430 } 437 }
431 if (0x31 <= next && next <= 0x39) { 438 if ($1 <= next && next <= $9) {
432 // '1'-'9' 439 // '1'-'9'
433 return _tokenizeNumber(next); 440 return _tokenizeNumber(next);
434 } 441 }
435 if (next == -1) { 442 if (next == -1) {
436 // EOF 443 // EOF
437 return -1; 444 return -1;
438 } 445 }
439 _reportError(ScannerErrorCode.ILLEGAL_CHARACTER, [next]); 446 _reportError(ScannerErrorCode.ILLEGAL_CHARACTER, [next]);
440 return _reader.advance(); 447 return _reader.advance();
441 } 448 }
(...skipping 199 matching lines...) Expand 10 before | Expand all | Expand 10 after
641 648
642 /** 649 /**
643 * Checks if [value] is the start of a generic method type annotation comment. 650 * Checks if [value] is the start of a generic method type annotation comment.
644 * 651 *
645 * This can either be of the form `/*<T>*/` or `/*=T*/`. The token type is 652 * This can either be of the form `/*<T>*/` or `/*=T*/`. The token type is
646 * returned, or null if it was not a generic method comment. 653 * returned, or null if it was not a generic method comment.
647 */ 654 */
648 TokenType _matchGenericMethodCommentType(String value) { 655 TokenType _matchGenericMethodCommentType(String value) {
649 if (scanGenericMethodComments) { 656 if (scanGenericMethodComments) {
650 // Match /*< and >*/ 657 // Match /*< and >*/
651 if (StringUtilities.startsWith3(value, 0, 0x2F, 0x2A, 0x3C) && 658 if (StringUtilities.startsWith3(value, 0, $slash, $asterisk, $lt) &&
652 StringUtilities.endsWith3(value, 0x3E, 0x2A, 0x2F)) { 659 StringUtilities.endsWith3(value, $gt, $asterisk, $slash)) {
653 return TokenType.GENERIC_METHOD_TYPE_LIST; 660 return TokenType.GENERIC_METHOD_TYPE_LIST;
654 } 661 }
655 // Match /*= 662 // Match /*=
656 if (StringUtilities.startsWith3(value, 0, 0x2F, 0x2A, 0x3D)) { 663 if (StringUtilities.startsWith3(value, 0, $slash, $asterisk, $equal)) {
657 return TokenType.GENERIC_METHOD_TYPE_ASSIGN; 664 return TokenType.GENERIC_METHOD_TYPE_ASSIGN;
658 } 665 }
659 } 666 }
660 return null; 667 return null;
661 } 668 }
662 669
663 /** 670 /**
664 * Report an error at the current offset. The [errorCode] is the error code 671 * Report an error at the current offset. The [errorCode] is the error code
665 * indicating the nature of the error. The [arguments] are any arguments 672 * indicating the nature of the error. The [arguments] are any arguments
666 * needed to complete the error message 673 * needed to complete the error message
(...skipping 20 matching lines...) Expand all
687 if (next == choice) { 694 if (next == choice) {
688 _appendTokenOfTypeWithOffset(yesType, offset); 695 _appendTokenOfTypeWithOffset(yesType, offset);
689 return _reader.advance(); 696 return _reader.advance();
690 } else { 697 } else {
691 _appendTokenOfTypeWithOffset(noType, offset); 698 _appendTokenOfTypeWithOffset(noType, offset);
692 return next; 699 return next;
693 } 700 }
694 } 701 }
695 702
696 int _tokenizeAmpersand(int next) { 703 int _tokenizeAmpersand(int next) {
697 // && &= & 704 // &&= && &= &
698 next = _reader.advance(); 705 next = _reader.advance();
699 if (next == 0x26) { 706 if (next == $ampersand) {
707 next = _reader.advance();
708 if (scanLazyAssignmentOperators && next == $equal) {
709 _appendTokenOfType(TokenType.AMPERSAND_AMPERSAND_EQ);
710 return _reader.advance();
711 }
700 _appendTokenOfType(TokenType.AMPERSAND_AMPERSAND); 712 _appendTokenOfType(TokenType.AMPERSAND_AMPERSAND);
701 return _reader.advance(); 713 return next;
702 } else if (next == 0x3D) { 714 } else if (next == $equal) {
703 _appendTokenOfType(TokenType.AMPERSAND_EQ); 715 _appendTokenOfType(TokenType.AMPERSAND_EQ);
704 return _reader.advance(); 716 return _reader.advance();
705 } else { 717 } else {
706 _appendTokenOfType(TokenType.AMPERSAND); 718 _appendTokenOfType(TokenType.AMPERSAND);
707 return next; 719 return next;
708 } 720 }
709 } 721 }
710 722
711 int _tokenizeBar(int next) { 723 int _tokenizeBar(int next) {
712 // | || |= 724 // ||= || |= |
713 next = _reader.advance(); 725 next = _reader.advance();
714 if (next == 0x7C) { 726 if (next == $bar) {
727 next = _reader.advance();
728 if (scanLazyAssignmentOperators && next == $equal) {
729 _appendTokenOfType(TokenType.BAR_BAR_EQ);
730 return _reader.advance();
731 }
715 _appendTokenOfType(TokenType.BAR_BAR); 732 _appendTokenOfType(TokenType.BAR_BAR);
716 return _reader.advance(); 733 return next;
717 } else if (next == 0x3D) { 734 } else if (next == $equal) {
718 _appendTokenOfType(TokenType.BAR_EQ); 735 _appendTokenOfType(TokenType.BAR_EQ);
719 return _reader.advance(); 736 return _reader.advance();
720 } else { 737 } else {
721 _appendTokenOfType(TokenType.BAR); 738 _appendTokenOfType(TokenType.BAR);
722 return next; 739 return next;
723 } 740 }
724 } 741 }
725 742
726 int _tokenizeCaret(int next) => 743 int _tokenizeCaret(int next) =>
727 _select(0x3D, TokenType.CARET_EQ, TokenType.CARET); 744 _select($equal, TokenType.CARET_EQ, TokenType.CARET);
728 745
729 int _tokenizeDotOrNumber(int next) { 746 int _tokenizeDotOrNumber(int next) {
730 int start = _reader.offset; 747 int start = _reader.offset;
731 next = _reader.advance(); 748 next = _reader.advance();
732 if (0x30 <= next && next <= 0x39) { 749 if ($0 <= next && next <= $9) {
733 return _tokenizeFractionPart(next, start); 750 return _tokenizeFractionPart(next, start);
734 } else if (0x2E == next) { 751 } else if ($dot == next) {
735 return _select( 752 return _select(
736 0x2E, TokenType.PERIOD_PERIOD_PERIOD, TokenType.PERIOD_PERIOD); 753 $dot, TokenType.PERIOD_PERIOD_PERIOD, TokenType.PERIOD_PERIOD);
737 } else { 754 } else {
738 _appendTokenOfType(TokenType.PERIOD); 755 _appendTokenOfType(TokenType.PERIOD);
739 return next; 756 return next;
740 } 757 }
741 } 758 }
742 759
743 int _tokenizeEquals(int next) { 760 int _tokenizeEquals(int next) {
744 // = == => 761 // = == =>
745 next = _reader.advance(); 762 next = _reader.advance();
746 if (next == 0x3D) { 763 if (next == $equal) {
747 _appendTokenOfType(TokenType.EQ_EQ); 764 _appendTokenOfType(TokenType.EQ_EQ);
748 return _reader.advance(); 765 return _reader.advance();
749 } else if (next == 0x3E) { 766 } else if (next == $gt) {
750 _appendTokenOfType(TokenType.FUNCTION); 767 _appendTokenOfType(TokenType.FUNCTION);
751 return _reader.advance(); 768 return _reader.advance();
752 } 769 }
753 _appendTokenOfType(TokenType.EQ); 770 _appendTokenOfType(TokenType.EQ);
754 return next; 771 return next;
755 } 772 }
756 773
757 int _tokenizeExclamation(int next) { 774 int _tokenizeExclamation(int next) {
758 // ! != 775 // ! !=
759 next = _reader.advance(); 776 next = _reader.advance();
760 if (next == 0x3D) { 777 if (next == $equal) {
761 _appendTokenOfType(TokenType.BANG_EQ); 778 _appendTokenOfType(TokenType.BANG_EQ);
762 return _reader.advance(); 779 return _reader.advance();
763 } 780 }
764 _appendTokenOfType(TokenType.BANG); 781 _appendTokenOfType(TokenType.BANG);
765 return next; 782 return next;
766 } 783 }
767 784
768 int _tokenizeExponent(int next) { 785 int _tokenizeExponent(int next) {
769 if (next == 0x2B || next == 0x2D) { 786 if (next == $plus || next == $minus) {
770 next = _reader.advance(); 787 next = _reader.advance();
771 } 788 }
772 bool hasDigits = false; 789 bool hasDigits = false;
773 while (true) { 790 while (true) {
774 if (0x30 <= next && next <= 0x39) { 791 if ($0 <= next && next <= $9) {
775 hasDigits = true; 792 hasDigits = true;
776 } else { 793 } else {
777 if (!hasDigits) { 794 if (!hasDigits) {
778 _reportError(ScannerErrorCode.MISSING_DIGIT); 795 _reportError(ScannerErrorCode.MISSING_DIGIT);
779 } 796 }
780 return next; 797 return next;
781 } 798 }
782 next = _reader.advance(); 799 next = _reader.advance();
783 } 800 }
784 } 801 }
785 802
786 int _tokenizeFractionPart(int next, int start) { 803 int _tokenizeFractionPart(int next, int start) {
787 bool done = false; 804 bool done = false;
788 bool hasDigit = false; 805 bool hasDigit = false;
789 LOOP: while (!done) { 806 LOOP: while (!done) {
790 if (0x30 <= next && next <= 0x39) { 807 if ($0 <= next && next <= $9) {
791 hasDigit = true; 808 hasDigit = true;
792 } else if (0x65 == next || 0x45 == next) { 809 } else if ($e == next || $E == next) {
793 hasDigit = true; 810 hasDigit = true;
794 next = _tokenizeExponent(_reader.advance()); 811 next = _tokenizeExponent(_reader.advance());
795 done = true; 812 done = true;
796 continue LOOP; 813 continue LOOP;
797 } else { 814 } else {
798 done = true; 815 done = true;
799 continue LOOP; 816 continue LOOP;
800 } 817 }
801 next = _reader.advance(); 818 next = _reader.advance();
802 } 819 }
803 if (!hasDigit) { 820 if (!hasDigit) {
804 _appendStringToken(TokenType.INT, _reader.getString(start, -2)); 821 _appendStringToken(TokenType.INT, _reader.getString(start, -2));
805 if (0x2E == next) { 822 if ($dot == next) {
806 return _selectWithOffset(0x2E, TokenType.PERIOD_PERIOD_PERIOD, 823 return _selectWithOffset($dot, TokenType.PERIOD_PERIOD_PERIOD,
807 TokenType.PERIOD_PERIOD, _reader.offset - 1); 824 TokenType.PERIOD_PERIOD, _reader.offset - 1);
808 } 825 }
809 _appendTokenOfTypeWithOffset(TokenType.PERIOD, _reader.offset - 1); 826 _appendTokenOfTypeWithOffset(TokenType.PERIOD, _reader.offset - 1);
810 return bigSwitch(next); 827 return bigSwitch(next);
811 } 828 }
812 _appendStringToken( 829 _appendStringToken(
813 TokenType.DOUBLE, _reader.getString(start, next < 0 ? 0 : -1)); 830 TokenType.DOUBLE, _reader.getString(start, next < 0 ? 0 : -1));
814 return next; 831 return next;
815 } 832 }
816 833
817 int _tokenizeGreaterThan(int next) { 834 int _tokenizeGreaterThan(int next) {
818 // > >= >> >>= 835 // > >= >> >>=
819 next = _reader.advance(); 836 next = _reader.advance();
820 if (0x3D == next) { 837 if ($equal == next) {
821 _appendTokenOfType(TokenType.GT_EQ); 838 _appendTokenOfType(TokenType.GT_EQ);
822 return _reader.advance(); 839 return _reader.advance();
823 } else if (0x3E == next) { 840 } else if ($gt == next) {
824 next = _reader.advance(); 841 next = _reader.advance();
825 if (0x3D == next) { 842 if ($equal == next) {
826 _appendTokenOfType(TokenType.GT_GT_EQ); 843 _appendTokenOfType(TokenType.GT_GT_EQ);
827 return _reader.advance(); 844 return _reader.advance();
828 } else { 845 } else {
829 _appendTokenOfType(TokenType.GT_GT); 846 _appendTokenOfType(TokenType.GT_GT);
830 return next; 847 return next;
831 } 848 }
832 } else { 849 } else {
833 _appendTokenOfType(TokenType.GT); 850 _appendTokenOfType(TokenType.GT);
834 return next; 851 return next;
835 } 852 }
836 } 853 }
837 854
838 int _tokenizeHex(int next) { 855 int _tokenizeHex(int next) {
839 int start = _reader.offset - 1; 856 int start = _reader.offset - 1;
840 bool hasDigits = false; 857 bool hasDigits = false;
841 while (true) { 858 while (true) {
842 next = _reader.advance(); 859 next = _reader.advance();
843 if ((0x30 <= next && next <= 0x39) || 860 if (($0 <= next && next <= $9) ||
844 (0x41 <= next && next <= 0x46) || 861 ($A <= next && next <= $F) ||
845 (0x61 <= next && next <= 0x66)) { 862 ($a <= next && next <= $f)) {
846 hasDigits = true; 863 hasDigits = true;
847 } else { 864 } else {
848 if (!hasDigits) { 865 if (!hasDigits) {
849 _reportError(ScannerErrorCode.MISSING_HEX_DIGIT); 866 _reportError(ScannerErrorCode.MISSING_HEX_DIGIT);
850 } 867 }
851 _appendStringToken( 868 _appendStringToken(
852 TokenType.HEXADECIMAL, _reader.getString(start, next < 0 ? 0 : -1)); 869 TokenType.HEXADECIMAL, _reader.getString(start, next < 0 ? 0 : -1));
853 return next; 870 return next;
854 } 871 }
855 } 872 }
856 } 873 }
857 874
858 int _tokenizeHexOrNumber(int next) { 875 int _tokenizeHexOrNumber(int next) {
859 int x = _reader.peek(); 876 int x = _reader.peek();
860 if (x == 0x78 || x == 0x58) { 877 if (x == $x || x == $X) {
861 _reader.advance(); 878 _reader.advance();
862 return _tokenizeHex(x); 879 return _tokenizeHex(x);
863 } 880 }
864 return _tokenizeNumber(next); 881 return _tokenizeNumber(next);
865 } 882 }
866 883
867 int _tokenizeIdentifier(int next, int start, bool allowDollar) { 884 int _tokenizeIdentifier(int next, int start, bool allowDollar) {
868 while ((0x61 <= next && next <= 0x7A) || 885 while (($a <= next && next <= $z) ||
869 (0x41 <= next && next <= 0x5A) || 886 ($A <= next && next <= $Z) ||
870 (0x30 <= next && next <= 0x39) || 887 ($0 <= next && next <= $9) ||
871 next == 0x5F || 888 next == $_ ||
872 (next == 0x24 && allowDollar)) { 889 (next == $$ && allowDollar)) {
873 next = _reader.advance(); 890 next = _reader.advance();
874 } 891 }
875 _appendStringToken( 892 _appendStringToken(
876 TokenType.IDENTIFIER, _reader.getString(start, next < 0 ? 0 : -1)); 893 TokenType.IDENTIFIER, _reader.getString(start, next < 0 ? 0 : -1));
877 return next; 894 return next;
878 } 895 }
879 896
880 int _tokenizeInterpolatedExpression(int next, int start) { 897 int _tokenizeInterpolatedExpression(int next, int start) {
881 _appendBeginToken(TokenType.STRING_INTERPOLATION_EXPRESSION); 898 _appendBeginToken(TokenType.STRING_INTERPOLATION_EXPRESSION);
882 next = _reader.advance(); 899 next = _reader.advance();
883 while (next != -1) { 900 while (next != -1) {
884 if (next == 0x7D) { 901 if (next == $rbrace) {
885 BeginToken begin = 902 BeginToken begin =
886 _findTokenMatchingClosingBraceInInterpolationExpression(); 903 _findTokenMatchingClosingBraceInInterpolationExpression();
887 if (begin == null) { 904 if (begin == null) {
888 _beginToken(); 905 _beginToken();
889 _appendTokenOfType(TokenType.CLOSE_CURLY_BRACKET); 906 _appendTokenOfType(TokenType.CLOSE_CURLY_BRACKET);
890 next = _reader.advance(); 907 next = _reader.advance();
891 _beginToken(); 908 _beginToken();
892 return next; 909 return next;
893 } else if (begin.type == TokenType.OPEN_CURLY_BRACKET) { 910 } else if (begin.type == TokenType.OPEN_CURLY_BRACKET) {
894 _beginToken(); 911 _beginToken();
(...skipping 12 matching lines...) Expand all
907 } else { 924 } else {
908 next = bigSwitch(next); 925 next = bigSwitch(next);
909 } 926 }
910 } 927 }
911 return next; 928 return next;
912 } 929 }
913 930
914 int _tokenizeInterpolatedIdentifier(int next, int start) { 931 int _tokenizeInterpolatedIdentifier(int next, int start) {
915 _appendStringTokenWithOffset( 932 _appendStringTokenWithOffset(
916 TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 0); 933 TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 0);
917 if ((0x41 <= next && next <= 0x5A) || 934 if (($A <= next && next <= $Z) ||
918 (0x61 <= next && next <= 0x7A) || 935 ($a <= next && next <= $z) ||
919 next == 0x5F) { 936 next == $_) {
920 _beginToken(); 937 _beginToken();
921 next = _tokenizeKeywordOrIdentifier(next, false); 938 next = _tokenizeKeywordOrIdentifier(next, false);
922 } 939 }
923 _beginToken(); 940 _beginToken();
924 return next; 941 return next;
925 } 942 }
926 943
927 int _tokenizeKeywordOrIdentifier(int next, bool allowDollar) { 944 int _tokenizeKeywordOrIdentifier(int next, bool allowDollar) {
928 KeywordState state = KeywordState.KEYWORD_STATE; 945 KeywordState state = KeywordState.KEYWORD_STATE;
929 int start = _reader.offset; 946 int start = _reader.offset;
930 while (state != null && 0x61 <= next && next <= 0x7A) { 947 while (state != null && $a <= next && next <= $z) {
931 state = state.next(next); 948 state = state.next(next);
932 next = _reader.advance(); 949 next = _reader.advance();
933 } 950 }
934 if (state == null || state.keyword() == null) { 951 if (state == null || state.keyword() == null) {
935 return _tokenizeIdentifier(next, start, allowDollar); 952 return _tokenizeIdentifier(next, start, allowDollar);
936 } 953 }
937 if ((0x41 <= next && next <= 0x5A) || 954 if (($A <= next && next <= $Z) ||
938 (0x30 <= next && next <= 0x39) || 955 ($0 <= next && next <= $9) ||
939 next == 0x5F || 956 next == $_ ||
940 next == 0x24) { 957 next == $$) {
941 return _tokenizeIdentifier(next, start, allowDollar); 958 return _tokenizeIdentifier(next, start, allowDollar);
942 } else if (next < 128) { 959 } else if (next < 128) {
943 _appendKeywordToken(state.keyword()); 960 _appendKeywordToken(state.keyword());
944 return next; 961 return next;
945 } else { 962 } else {
946 return _tokenizeIdentifier(next, start, allowDollar); 963 return _tokenizeIdentifier(next, start, allowDollar);
947 } 964 }
948 } 965 }
949 966
950 int _tokenizeLessThan(int next) { 967 int _tokenizeLessThan(int next) {
951 // < <= << <<= 968 // < <= << <<=
952 next = _reader.advance(); 969 next = _reader.advance();
953 if (0x3D == next) { 970 if ($equal == next) {
954 _appendTokenOfType(TokenType.LT_EQ); 971 _appendTokenOfType(TokenType.LT_EQ);
955 return _reader.advance(); 972 return _reader.advance();
956 } else if (0x3C == next) { 973 } else if ($lt == next) {
957 return _select(0x3D, TokenType.LT_LT_EQ, TokenType.LT_LT); 974 return _select($equal, TokenType.LT_LT_EQ, TokenType.LT_LT);
958 } else { 975 } else {
959 _appendTokenOfType(TokenType.LT); 976 _appendTokenOfType(TokenType.LT);
960 return next; 977 return next;
961 } 978 }
962 } 979 }
963 980
964 int _tokenizeMinus(int next) { 981 int _tokenizeMinus(int next) {
965 // - -- -= 982 // - -- -=
966 next = _reader.advance(); 983 next = _reader.advance();
967 if (next == 0x2D) { 984 if (next == $minus) {
968 _appendTokenOfType(TokenType.MINUS_MINUS); 985 _appendTokenOfType(TokenType.MINUS_MINUS);
969 return _reader.advance(); 986 return _reader.advance();
970 } else if (next == 0x3D) { 987 } else if (next == $equal) {
971 _appendTokenOfType(TokenType.MINUS_EQ); 988 _appendTokenOfType(TokenType.MINUS_EQ);
972 return _reader.advance(); 989 return _reader.advance();
973 } else { 990 } else {
974 _appendTokenOfType(TokenType.MINUS); 991 _appendTokenOfType(TokenType.MINUS);
975 return next; 992 return next;
976 } 993 }
977 } 994 }
978 995
979 int _tokenizeMultiLineComment(int next) { 996 int _tokenizeMultiLineComment(int next) {
980 int nesting = 1; 997 int nesting = 1;
981 next = _reader.advance(); 998 next = _reader.advance();
982 while (true) { 999 while (true) {
983 if (-1 == next) { 1000 if (-1 == next) {
984 _reportError(ScannerErrorCode.UNTERMINATED_MULTI_LINE_COMMENT); 1001 _reportError(ScannerErrorCode.UNTERMINATED_MULTI_LINE_COMMENT);
985 _appendCommentToken( 1002 _appendCommentToken(
986 TokenType.MULTI_LINE_COMMENT, _reader.getString(_tokenStart, 0)); 1003 TokenType.MULTI_LINE_COMMENT, _reader.getString(_tokenStart, 0));
987 return next; 1004 return next;
988 } else if (0x2A == next) { 1005 } else if ($asterisk == next) {
989 next = _reader.advance(); 1006 next = _reader.advance();
990 if (0x2F == next) { 1007 if ($slash == next) {
991 --nesting; 1008 --nesting;
992 if (0 == nesting) { 1009 if (0 == nesting) {
993 _appendCommentToken(TokenType.MULTI_LINE_COMMENT, 1010 _appendCommentToken(TokenType.MULTI_LINE_COMMENT,
994 _reader.getString(_tokenStart, 0)); 1011 _reader.getString(_tokenStart, 0));
995 return _reader.advance(); 1012 return _reader.advance();
996 } else { 1013 } else {
997 next = _reader.advance(); 1014 next = _reader.advance();
998 } 1015 }
999 } 1016 }
1000 } else if (0x2F == next) { 1017 } else if ($slash == next) {
1001 next = _reader.advance(); 1018 next = _reader.advance();
1002 if (0x2A == next) { 1019 if ($asterisk == next) {
1003 next = _reader.advance(); 1020 next = _reader.advance();
1004 ++nesting; 1021 ++nesting;
1005 } 1022 }
1006 } else if (next == 0xD) { 1023 } else if (next == $cr) {
1007 next = _reader.advance(); 1024 next = _reader.advance();
1008 if (next == 0xA) { 1025 if (next == $lf) {
1009 next = _reader.advance(); 1026 next = _reader.advance();
1010 } 1027 }
1011 recordStartOfLine(); 1028 recordStartOfLine();
1012 } else if (next == 0xA) { 1029 } else if (next == $lf) {
1013 next = _reader.advance(); 1030 next = _reader.advance();
1014 recordStartOfLine(); 1031 recordStartOfLine();
1015 } else { 1032 } else {
1016 next = _reader.advance(); 1033 next = _reader.advance();
1017 } 1034 }
1018 } 1035 }
1019 } 1036 }
1020 1037
1021 int _tokenizeMultiLineRawString(int quoteChar, int start) { 1038 int _tokenizeMultiLineRawString(int quoteChar, int start) {
1022 int next = _reader.advance(); 1039 int next = _reader.advance();
1023 outer: while (next != -1) { 1040 outer: while (next != -1) {
1024 while (next != quoteChar) { 1041 while (next != quoteChar) {
1025 if (next == -1) { 1042 if (next == -1) {
1026 break outer; 1043 break outer;
1027 } else if (next == 0xD) { 1044 } else if (next == $cr) {
1028 next = _reader.advance(); 1045 next = _reader.advance();
1029 if (next == 0xA) { 1046 if (next == $lf) {
1030 next = _reader.advance(); 1047 next = _reader.advance();
1031 } 1048 }
1032 recordStartOfLine(); 1049 recordStartOfLine();
1033 } else if (next == 0xA) { 1050 } else if (next == $lf) {
1034 next = _reader.advance(); 1051 next = _reader.advance();
1035 recordStartOfLine(); 1052 recordStartOfLine();
1036 } else { 1053 } else {
1037 next = _reader.advance(); 1054 next = _reader.advance();
1038 } 1055 }
1039 } 1056 }
1040 next = _reader.advance(); 1057 next = _reader.advance();
1041 if (next == quoteChar) { 1058 if (next == quoteChar) {
1042 next = _reader.advance(); 1059 next = _reader.advance();
1043 if (next == quoteChar) { 1060 if (next == quoteChar) {
1044 _appendStringToken(TokenType.STRING, _reader.getString(start, 0)); 1061 _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
1045 return _reader.advance(); 1062 return _reader.advance();
1046 } 1063 }
1047 } 1064 }
1048 } 1065 }
1049 _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL); 1066 _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL);
1050 _appendStringToken(TokenType.STRING, _reader.getString(start, 0)); 1067 _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
1051 return _reader.advance(); 1068 return _reader.advance();
1052 } 1069 }
1053 1070
1054 int _tokenizeMultiLineString(int quoteChar, int start, bool raw) { 1071 int _tokenizeMultiLineString(int quoteChar, int start, bool raw) {
1055 if (raw) { 1072 if (raw) {
1056 return _tokenizeMultiLineRawString(quoteChar, start); 1073 return _tokenizeMultiLineRawString(quoteChar, start);
1057 } 1074 }
1058 int next = _reader.advance(); 1075 int next = _reader.advance();
1059 while (next != -1) { 1076 while (next != -1) {
1060 if (next == 0x24) { 1077 if (next == $$) {
1061 _appendStringToken(TokenType.STRING, _reader.getString(start, -1)); 1078 _appendStringToken(TokenType.STRING, _reader.getString(start, -1));
1062 next = _tokenizeStringInterpolation(start); 1079 next = _tokenizeStringInterpolation(start);
1063 _beginToken(); 1080 _beginToken();
1064 start = _reader.offset; 1081 start = _reader.offset;
1065 continue; 1082 continue;
1066 } 1083 }
1067 if (next == quoteChar) { 1084 if (next == quoteChar) {
1068 next = _reader.advance(); 1085 next = _reader.advance();
1069 if (next == quoteChar) { 1086 if (next == quoteChar) {
1070 next = _reader.advance(); 1087 next = _reader.advance();
1071 if (next == quoteChar) { 1088 if (next == quoteChar) {
1072 _appendStringToken(TokenType.STRING, _reader.getString(start, 0)); 1089 _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
1073 return _reader.advance(); 1090 return _reader.advance();
1074 } 1091 }
1075 } 1092 }
1076 continue; 1093 continue;
1077 } 1094 }
1078 if (next == 0x5C) { 1095 if (next == $backslash) {
1079 next = _reader.advance(); 1096 next = _reader.advance();
1080 if (next == -1) { 1097 if (next == -1) {
1081 break; 1098 break;
1082 } 1099 }
1083 if (next == 0xD) { 1100 if (next == $cr) {
1084 next = _reader.advance(); 1101 next = _reader.advance();
1085 if (next == 0xA) { 1102 if (next == $lf) {
1086 next = _reader.advance(); 1103 next = _reader.advance();
1087 } 1104 }
1088 recordStartOfLine(); 1105 recordStartOfLine();
1089 } else if (next == 0xA) { 1106 } else if (next == $lf) {
1090 recordStartOfLine(); 1107 recordStartOfLine();
1091 next = _reader.advance(); 1108 next = _reader.advance();
1092 } else { 1109 } else {
1093 next = _reader.advance(); 1110 next = _reader.advance();
1094 } 1111 }
1095 } else if (next == 0xD) { 1112 } else if (next == $cr) {
1096 next = _reader.advance(); 1113 next = _reader.advance();
1097 if (next == 0xA) { 1114 if (next == $lf) {
1098 next = _reader.advance(); 1115 next = _reader.advance();
1099 } 1116 }
1100 recordStartOfLine(); 1117 recordStartOfLine();
1101 } else if (next == 0xA) { 1118 } else if (next == $lf) {
1102 recordStartOfLine(); 1119 recordStartOfLine();
1103 next = _reader.advance(); 1120 next = _reader.advance();
1104 } else { 1121 } else {
1105 next = _reader.advance(); 1122 next = _reader.advance();
1106 } 1123 }
1107 } 1124 }
1108 _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL); 1125 _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL);
1109 if (start == _reader.offset) { 1126 if (start == _reader.offset) {
1110 _appendStringTokenWithOffset(TokenType.STRING, "", 1); 1127 _appendStringTokenWithOffset(TokenType.STRING, "", 1);
1111 } else { 1128 } else {
1112 _appendStringToken(TokenType.STRING, _reader.getString(start, 0)); 1129 _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
1113 } 1130 }
1114 return _reader.advance(); 1131 return _reader.advance();
1115 } 1132 }
1116 1133
1117 int _tokenizeMultiply(int next) => 1134 int _tokenizeMultiply(int next) =>
1118 _select(0x3D, TokenType.STAR_EQ, TokenType.STAR); 1135 _select($equal, TokenType.STAR_EQ, TokenType.STAR);
1119 1136
1120 int _tokenizeNumber(int next) { 1137 int _tokenizeNumber(int next) {
1121 int start = _reader.offset; 1138 int start = _reader.offset;
1122 while (true) { 1139 while (true) {
1123 next = _reader.advance(); 1140 next = _reader.advance();
1124 if (0x30 <= next && next <= 0x39) { 1141 if ($0 <= next && next <= $9) {
1125 continue; 1142 continue;
1126 } else if (next == 0x2E) { 1143 } else if (next == $dot) {
1127 return _tokenizeFractionPart(_reader.advance(), start); 1144 return _tokenizeFractionPart(_reader.advance(), start);
1128 } else if (next == 0x65 || next == 0x45) { 1145 } else if (next == $e || next == $E) {
1129 return _tokenizeFractionPart(next, start); 1146 return _tokenizeFractionPart(next, start);
1130 } else { 1147 } else {
1131 _appendStringToken( 1148 _appendStringToken(
1132 TokenType.INT, _reader.getString(start, next < 0 ? 0 : -1)); 1149 TokenType.INT, _reader.getString(start, next < 0 ? 0 : -1));
1133 return next; 1150 return next;
1134 } 1151 }
1135 } 1152 }
1136 } 1153 }
1137 1154
1138 int _tokenizeOpenSquareBracket(int next) { 1155 int _tokenizeOpenSquareBracket(int next) {
1139 // [ [] []= 1156 // [ [] []=
1140 next = _reader.advance(); 1157 next = _reader.advance();
1141 if (next == 0x5D) { 1158 if (next == $close_bracket) {
1142 return _select(0x3D, TokenType.INDEX_EQ, TokenType.INDEX); 1159 return _select($equal, TokenType.INDEX_EQ, TokenType.INDEX);
1143 } else { 1160 } else {
1144 _appendBeginToken(TokenType.OPEN_SQUARE_BRACKET); 1161 _appendBeginToken(TokenType.OPEN_SQUARE_BRACKET);
1145 return next; 1162 return next;
1146 } 1163 }
1147 } 1164 }
1148 1165
1149 int _tokenizePercent(int next) => 1166 int _tokenizePercent(int next) =>
1150 _select(0x3D, TokenType.PERCENT_EQ, TokenType.PERCENT); 1167 _select($equal, TokenType.PERCENT_EQ, TokenType.PERCENT);
1151 1168
1152 int _tokenizePlus(int next) { 1169 int _tokenizePlus(int next) {
1153 // + ++ += 1170 // + ++ +=
1154 next = _reader.advance(); 1171 next = _reader.advance();
1155 if (0x2B == next) { 1172 if ($plus == next) {
1156 _appendTokenOfType(TokenType.PLUS_PLUS); 1173 _appendTokenOfType(TokenType.PLUS_PLUS);
1157 return _reader.advance(); 1174 return _reader.advance();
1158 } else if (0x3D == next) { 1175 } else if ($equal == next) {
1159 _appendTokenOfType(TokenType.PLUS_EQ); 1176 _appendTokenOfType(TokenType.PLUS_EQ);
1160 return _reader.advance(); 1177 return _reader.advance();
1161 } else { 1178 } else {
1162 _appendTokenOfType(TokenType.PLUS); 1179 _appendTokenOfType(TokenType.PLUS);
1163 return next; 1180 return next;
1164 } 1181 }
1165 } 1182 }
1166 1183
1167 int _tokenizeQuestion() { 1184 int _tokenizeQuestion() {
1168 // ? ?. ?? ??= 1185 // ? ?. ?? ??=
1169 int next = _reader.advance(); 1186 int next = _reader.advance();
1170 if (next == 0x2E) { 1187 if (next == $dot) {
1171 // '.' 1188 // '.'
1172 _appendTokenOfType(TokenType.QUESTION_PERIOD); 1189 _appendTokenOfType(TokenType.QUESTION_PERIOD);
1173 return _reader.advance(); 1190 return _reader.advance();
1174 } else if (next == 0x3F) { 1191 } else if (next == $question) {
1175 // '?' 1192 // '?'
1176 next = _reader.advance(); 1193 next = _reader.advance();
1177 if (next == 0x3D) { 1194 if (next == $equal) {
1178 // '=' 1195 // '='
1179 _appendTokenOfType(TokenType.QUESTION_QUESTION_EQ); 1196 _appendTokenOfType(TokenType.QUESTION_QUESTION_EQ);
1180 return _reader.advance(); 1197 return _reader.advance();
1181 } else { 1198 } else {
1182 _appendTokenOfType(TokenType.QUESTION_QUESTION); 1199 _appendTokenOfType(TokenType.QUESTION_QUESTION);
1183 return next; 1200 return next;
1184 } 1201 }
1185 } else { 1202 } else {
1186 _appendTokenOfType(TokenType.QUESTION); 1203 _appendTokenOfType(TokenType.QUESTION);
1187 return next; 1204 return next;
1188 } 1205 }
1189 } 1206 }
1190 1207
1191 int _tokenizeSingleLineComment(int next) { 1208 int _tokenizeSingleLineComment(int next) {
1192 while (true) { 1209 while (true) {
1193 next = _reader.advance(); 1210 next = _reader.advance();
1194 if (-1 == next) { 1211 if (-1 == next) {
1195 _appendCommentToken( 1212 _appendCommentToken(
1196 TokenType.SINGLE_LINE_COMMENT, _reader.getString(_tokenStart, 0)); 1213 TokenType.SINGLE_LINE_COMMENT, _reader.getString(_tokenStart, 0));
1197 return next; 1214 return next;
1198 } else if (0xA == next || 0xD == next) { 1215 } else if ($lf == next || $cr == next) {
1199 _appendCommentToken( 1216 _appendCommentToken(
1200 TokenType.SINGLE_LINE_COMMENT, _reader.getString(_tokenStart, -1)); 1217 TokenType.SINGLE_LINE_COMMENT, _reader.getString(_tokenStart, -1));
1201 return next; 1218 return next;
1202 } 1219 }
1203 } 1220 }
1204 } 1221 }
1205 1222
1206 int _tokenizeSingleLineRawString(int next, int quoteChar, int start) { 1223 int _tokenizeSingleLineRawString(int next, int quoteChar, int start) {
1207 next = _reader.advance(); 1224 next = _reader.advance();
1208 while (next != -1) { 1225 while (next != -1) {
1209 if (next == quoteChar) { 1226 if (next == quoteChar) {
1210 _appendStringToken(TokenType.STRING, _reader.getString(start, 0)); 1227 _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
1211 return _reader.advance(); 1228 return _reader.advance();
1212 } else if (next == 0xD || next == 0xA) { 1229 } else if (next == $cr || next == $lf) {
1213 _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL); 1230 _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL);
1214 _appendStringToken(TokenType.STRING, _reader.getString(start, -1)); 1231 _appendStringToken(TokenType.STRING, _reader.getString(start, -1));
1215 return _reader.advance(); 1232 return _reader.advance();
1216 } 1233 }
1217 next = _reader.advance(); 1234 next = _reader.advance();
1218 } 1235 }
1219 _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL); 1236 _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL);
1220 _appendStringToken(TokenType.STRING, _reader.getString(start, 0)); 1237 _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
1221 return _reader.advance(); 1238 return _reader.advance();
1222 } 1239 }
1223 1240
1224 int _tokenizeSingleLineString(int next, int quoteChar, int start) { 1241 int _tokenizeSingleLineString(int next, int quoteChar, int start) {
1225 while (next != quoteChar) { 1242 while (next != quoteChar) {
1226 if (next == 0x5C) { 1243 if (next == $backslash) {
1227 next = _reader.advance(); 1244 next = _reader.advance();
1228 } else if (next == 0x24) { 1245 } else if (next == $$) {
1229 _appendStringToken(TokenType.STRING, _reader.getString(start, -1)); 1246 _appendStringToken(TokenType.STRING, _reader.getString(start, -1));
1230 next = _tokenizeStringInterpolation(start); 1247 next = _tokenizeStringInterpolation(start);
1231 _beginToken(); 1248 _beginToken();
1232 start = _reader.offset; 1249 start = _reader.offset;
1233 continue; 1250 continue;
1234 } 1251 }
1235 if (next <= 0xD && (next == 0xA || next == 0xD || next == -1)) { 1252 if (next <= $cr && (next == $lf || next == $cr || next == -1)) {
1236 _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL); 1253 _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL);
1237 if (start == _reader.offset) { 1254 if (start == _reader.offset) {
1238 _appendStringTokenWithOffset(TokenType.STRING, "", 1); 1255 _appendStringTokenWithOffset(TokenType.STRING, "", 1);
1239 } else if (next == -1) { 1256 } else if (next == -1) {
1240 _appendStringToken(TokenType.STRING, _reader.getString(start, 0)); 1257 _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
1241 } else { 1258 } else {
1242 _appendStringToken(TokenType.STRING, _reader.getString(start, -1)); 1259 _appendStringToken(TokenType.STRING, _reader.getString(start, -1));
1243 } 1260 }
1244 return _reader.advance(); 1261 return _reader.advance();
1245 } 1262 }
1246 next = _reader.advance(); 1263 next = _reader.advance();
1247 } 1264 }
1248 _appendStringToken(TokenType.STRING, _reader.getString(start, 0)); 1265 _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
1249 return _reader.advance(); 1266 return _reader.advance();
1250 } 1267 }
1251 1268
1252 int _tokenizeSlashOrComment(int next) { 1269 int _tokenizeSlashOrComment(int next) {
1253 next = _reader.advance(); 1270 next = _reader.advance();
1254 if (0x2A == next) { 1271 if ($asterisk == next) {
1255 return _tokenizeMultiLineComment(next); 1272 return _tokenizeMultiLineComment(next);
1256 } else if (0x2F == next) { 1273 } else if ($slash == next) {
1257 return _tokenizeSingleLineComment(next); 1274 return _tokenizeSingleLineComment(next);
1258 } else if (0x3D == next) { 1275 } else if ($equal == next) {
1259 _appendTokenOfType(TokenType.SLASH_EQ); 1276 _appendTokenOfType(TokenType.SLASH_EQ);
1260 return _reader.advance(); 1277 return _reader.advance();
1261 } else { 1278 } else {
1262 _appendTokenOfType(TokenType.SLASH); 1279 _appendTokenOfType(TokenType.SLASH);
1263 return next; 1280 return next;
1264 } 1281 }
1265 } 1282 }
1266 1283
1267 int _tokenizeString(int next, int start, bool raw) { 1284 int _tokenizeString(int next, int start, bool raw) {
1268 int quoteChar = next; 1285 int quoteChar = next;
(...skipping 12 matching lines...) Expand all
1281 if (raw) { 1298 if (raw) {
1282 return _tokenizeSingleLineRawString(next, quoteChar, start); 1299 return _tokenizeSingleLineRawString(next, quoteChar, start);
1283 } else { 1300 } else {
1284 return _tokenizeSingleLineString(next, quoteChar, start); 1301 return _tokenizeSingleLineString(next, quoteChar, start);
1285 } 1302 }
1286 } 1303 }
1287 1304
1288 int _tokenizeStringInterpolation(int start) { 1305 int _tokenizeStringInterpolation(int start) {
1289 _beginToken(); 1306 _beginToken();
1290 int next = _reader.advance(); 1307 int next = _reader.advance();
1291 if (next == 0x7B) { 1308 if (next == $lbrace) {
1292 return _tokenizeInterpolatedExpression(next, start); 1309 return _tokenizeInterpolatedExpression(next, start);
1293 } else { 1310 } else {
1294 return _tokenizeInterpolatedIdentifier(next, start); 1311 return _tokenizeInterpolatedIdentifier(next, start);
1295 } 1312 }
1296 } 1313 }
1297 1314
1298 int _tokenizeTag(int next) { 1315 int _tokenizeTag(int next) {
1299 // # or #!.*[\n\r] 1316 // # or #!.*[\n\r]
1300 if (_reader.offset == 0) { 1317 if (_reader.offset == 0) {
1301 if (_reader.peek() == 0x21) { 1318 if (_reader.peek() == $exclamation) {
1302 do { 1319 do {
1303 next = _reader.advance(); 1320 next = _reader.advance();
1304 } while (next != 0xA && next != 0xD && next > 0); 1321 } while (next != $lf && next != $cr && next > 0);
1305 _appendStringToken( 1322 _appendStringToken(
1306 TokenType.SCRIPT_TAG, _reader.getString(_tokenStart, 0)); 1323 TokenType.SCRIPT_TAG, _reader.getString(_tokenStart, 0));
1307 return next; 1324 return next;
1308 } 1325 }
1309 } 1326 }
1310 _appendTokenOfType(TokenType.HASH); 1327 _appendTokenOfType(TokenType.HASH);
1311 return _reader.advance(); 1328 return _reader.advance();
1312 } 1329 }
1313 1330
1314 int _tokenizeTilde(int next) { 1331 int _tokenizeTilde(int next) {
1315 // ~ ~/ ~/= 1332 // ~ ~/ ~/=
1316 next = _reader.advance(); 1333 next = _reader.advance();
1317 if (next == 0x2F) { 1334 if (next == $slash) {
1318 return _select(0x3D, TokenType.TILDE_SLASH_EQ, TokenType.TILDE_SLASH); 1335 return _select($equal, TokenType.TILDE_SLASH_EQ, TokenType.TILDE_SLASH);
1319 } else { 1336 } else {
1320 _appendTokenOfType(TokenType.TILDE); 1337 _appendTokenOfType(TokenType.TILDE);
1321 return next; 1338 return next;
1322 } 1339 }
1323 } 1340 }
1324 1341
1325 /** 1342 /**
1326 * Checks if [value] is a single-line or multi-line comment. 1343 * Checks if [value] is a single-line or multi-line comment.
1327 */ 1344 */
1328 static bool _isDocumentationComment(String value) { 1345 static bool _isDocumentationComment(String value) {
1329 return StringUtilities.startsWith3(value, 0, 0x2F, 0x2F, 0x2F) || 1346 return StringUtilities.startsWith3(value, 0, $slash, $slash, $slash) ||
1330 StringUtilities.startsWith3(value, 0, 0x2F, 0x2A, 0x2A); 1347 StringUtilities.startsWith3(value, 0, $slash, $asterisk, $asterisk);
1331 } 1348 }
1332 } 1349 }
1333 1350
1334 /** 1351 /**
1335 * The error codes used for errors detected by the scanner. 1352 * The error codes used for errors detected by the scanner.
1336 */ 1353 */
1337 class ScannerErrorCode extends ErrorCode { 1354 class ScannerErrorCode extends ErrorCode {
1338 static const ScannerErrorCode ILLEGAL_CHARACTER = 1355 static const ScannerErrorCode ILLEGAL_CHARACTER =
1339 const ScannerErrorCode('ILLEGAL_CHARACTER', "Illegal character {0}"); 1356 const ScannerErrorCode('ILLEGAL_CHARACTER', "Illegal character {0}");
1340 1357
(...skipping 25 matching lines...) Expand all
1366 */ 1383 */
1367 const ScannerErrorCode(String name, String message, [String correction]) 1384 const ScannerErrorCode(String name, String message, [String correction])
1368 : super(name, message, correction); 1385 : super(name, message, correction);
1369 1386
1370 @override 1387 @override
1371 ErrorSeverity get errorSeverity => ErrorSeverity.ERROR; 1388 ErrorSeverity get errorSeverity => ErrorSeverity.ERROR;
1372 1389
1373 @override 1390 @override
1374 ErrorType get type => ErrorType.SYNTACTIC_ERROR; 1391 ErrorType get type => ErrorType.SYNTACTIC_ERROR;
1375 } 1392 }
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698