| OLD | NEW |
| 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 // This code was auto-generated, is not intended to be edited, and is subject to | 5 // This code was auto-generated, is not intended to be edited, and is subject to |
| 6 // significant change. Please see the README file for more information. | 6 // significant change. Please see the README file for more information. |
| 7 | 7 |
| 8 library engine.scanner; | 8 library engine.scanner; |
| 9 | 9 |
| 10 import 'dart:collection'; | 10 import 'dart:collection'; |
| (...skipping 10 matching lines...) Expand all Loading... |
| 21 */ | 21 */ |
| 22 class KeywordState { | 22 class KeywordState { |
| 23 /** | 23 /** |
| 24 * An empty transition table used by leaf states. | 24 * An empty transition table used by leaf states. |
| 25 */ | 25 */ |
| 26 static List<KeywordState> _EMPTY_TABLE = new List<KeywordState>(26); | 26 static List<KeywordState> _EMPTY_TABLE = new List<KeywordState>(26); |
| 27 | 27 |
| 28 /** | 28 /** |
| 29 * The initial state in the state machine. | 29 * The initial state in the state machine. |
| 30 */ | 30 */ |
| 31 static KeywordState KEYWORD_STATE = createKeywordStateTable(); | 31 static KeywordState KEYWORD_STATE = _createKeywordStateTable(); |
| 32 | 32 |
| 33 /** | 33 /** |
| 34 * Create the next state in the state machine where we have already recognized
the subset of | 34 * Create the next state in the state machine where we have already recognized
the subset of |
| 35 * strings in the given array of strings starting at the given offset and havi
ng the given length. | 35 * strings in the given array of strings starting at the given offset and havi
ng the given length. |
| 36 * All of these strings have a common prefix and the next character is at the
given start index. | 36 * All of these strings have a common prefix and the next character is at the
given start index. |
| 37 * | 37 * |
| 38 * @param start the index of the character in the strings used to transition t
o a new state | 38 * @param start the index of the character in the strings used to transition t
o a new state |
| 39 * @param strings an array containing all of the strings that will be recogniz
ed by the state | 39 * @param strings an array containing all of the strings that will be recogniz
ed by the state |
| 40 * machine | 40 * machine |
| 41 * @param offset the offset of the first string in the array that has the pref
ix that is assumed | 41 * @param offset the offset of the first string in the array that has the pref
ix that is assumed |
| 42 * to have been recognized by the time we reach the state being built | 42 * to have been recognized by the time we reach the state being built |
| 43 * @param length the number of strings in the array that pass through the stat
e being built | 43 * @param length the number of strings in the array that pass through the stat
e being built |
| 44 * @return the state that was created | 44 * @return the state that was created |
| 45 */ | 45 */ |
| 46 static KeywordState computeKeywordStateTable(int start, List<String> strings,
int offset, int length) { | 46 static KeywordState _computeKeywordStateTable(int start, List<String> strings,
int offset, int length) { |
| 47 List<KeywordState> result = new List<KeywordState>(26); | 47 List<KeywordState> result = new List<KeywordState>(26); |
| 48 assert(length != 0); | 48 assert(length != 0); |
| 49 int chunk = 0x0; | 49 int chunk = 0x0; |
| 50 int chunkStart = -1; | 50 int chunkStart = -1; |
| 51 bool isLeaf = false; | 51 bool isLeaf = false; |
| 52 for (int i = offset; i < offset + length; i++) { | 52 for (int i = offset; i < offset + length; i++) { |
| 53 if (strings[i].length == start) { | 53 if (strings[i].length == start) { |
| 54 isLeaf = true; | 54 isLeaf = true; |
| 55 } | 55 } |
| 56 if (strings[i].length > start) { | 56 if (strings[i].length > start) { |
| 57 int c = strings[i].codeUnitAt(start); | 57 int c = strings[i].codeUnitAt(start); |
| 58 if (chunk != c) { | 58 if (chunk != c) { |
| 59 if (chunkStart != -1) { | 59 if (chunkStart != -1) { |
| 60 result[chunk - 0x61] = computeKeywordStateTable(start + 1, strings,
chunkStart, i - chunkStart); | 60 result[chunk - 0x61] = _computeKeywordStateTable(start + 1, strings,
chunkStart, i - chunkStart); |
| 61 } | 61 } |
| 62 chunkStart = i; | 62 chunkStart = i; |
| 63 chunk = c; | 63 chunk = c; |
| 64 } | 64 } |
| 65 } | 65 } |
| 66 } | 66 } |
| 67 if (chunkStart != -1) { | 67 if (chunkStart != -1) { |
| 68 assert(result[chunk - 0x61] == null); | 68 assert(result[chunk - 0x61] == null); |
| 69 result[chunk - 0x61] = computeKeywordStateTable(start + 1, strings, chunkS
tart, offset + length - chunkStart); | 69 result[chunk - 0x61] = _computeKeywordStateTable(start + 1, strings, chunk
Start, offset + length - chunkStart); |
| 70 } else { | 70 } else { |
| 71 assert(length == 1); | 71 assert(length == 1); |
| 72 return new KeywordState(_EMPTY_TABLE, strings[offset]); | 72 return new KeywordState(_EMPTY_TABLE, strings[offset]); |
| 73 } | 73 } |
| 74 if (isLeaf) { | 74 if (isLeaf) { |
| 75 return new KeywordState(result, strings[offset]); | 75 return new KeywordState(result, strings[offset]); |
| 76 } else { | 76 } else { |
| 77 return new KeywordState(result, null); | 77 return new KeywordState(result, null); |
| 78 } | 78 } |
| 79 } | 79 } |
| 80 | 80 |
| 81 /** | 81 /** |
| 82 * Create the initial state in the state machine. | 82 * Create the initial state in the state machine. |
| 83 * | 83 * |
| 84 * @return the state that was created | 84 * @return the state that was created |
| 85 */ | 85 */ |
| 86 static KeywordState createKeywordStateTable() { | 86 static KeywordState _createKeywordStateTable() { |
| 87 List<Keyword> values = Keyword.values; | 87 List<Keyword> values = Keyword.values; |
| 88 List<String> strings = new List<String>(values.length); | 88 List<String> strings = new List<String>(values.length); |
| 89 for (int i = 0; i < values.length; i++) { | 89 for (int i = 0; i < values.length; i++) { |
| 90 strings[i] = values[i].syntax; | 90 strings[i] = values[i].syntax; |
| 91 } | 91 } |
| 92 strings.sort(); | 92 strings.sort(); |
| 93 return computeKeywordStateTable(0, strings, 0, strings.length); | 93 return _computeKeywordStateTable(0, strings, 0, strings.length); |
| 94 } | 94 } |
| 95 | 95 |
| 96 /** | 96 /** |
| 97 * A table mapping characters to the states to which those characters will tra
nsition. (The index | 97 * A table mapping characters to the states to which those characters will tra
nsition. (The index |
| 98 * into the array is the offset from the character `'a'` to the transitioning
character.) | 98 * into the array is the offset from the character `'a'` to the transitioning
character.) |
| 99 */ | 99 */ |
| 100 List<KeywordState> _table; | 100 List<KeywordState> _table; |
| 101 | 101 |
| 102 /** | 102 /** |
| 103 * The keyword that is recognized by this state, or `null` if this state is no
t a terminal | 103 * The keyword that is recognized by this state, or `null` if this state is no
t a terminal |
| 104 * state. | 104 * state. |
| 105 */ | 105 */ |
| 106 Keyword _keyword2; | 106 Keyword _keyword; |
| 107 | 107 |
| 108 /** | 108 /** |
| 109 * Initialize a newly created state to have the given transitions and to recog
nize the keyword | 109 * Initialize a newly created state to have the given transitions and to recog
nize the keyword |
| 110 * with the given syntax. | 110 * with the given syntax. |
| 111 * | 111 * |
| 112 * @param table a table mapping characters to the states to which those charac
ters will transition | 112 * @param table a table mapping characters to the states to which those charac
ters will transition |
| 113 * @param syntax the syntax of the keyword that is recognized by the state | 113 * @param syntax the syntax of the keyword that is recognized by the state |
| 114 */ | 114 */ |
| 115 KeywordState(List<KeywordState> table, String syntax) { | 115 KeywordState(List<KeywordState> table, String syntax) { |
| 116 this._table = table; | 116 this._table = table; |
| 117 this._keyword2 = (syntax == null) ? null : Keyword.keywords[syntax]; | 117 this._keyword = (syntax == null) ? null : Keyword.keywords[syntax]; |
| 118 } | 118 } |
| 119 | 119 |
| 120 /** | 120 /** |
| 121 * Return the keyword that was recognized by this state, or `null` if this sta
te does not | 121 * Return the keyword that was recognized by this state, or `null` if this sta
te does not |
| 122 * recognized a keyword. | 122 * recognized a keyword. |
| 123 * | 123 * |
| 124 * @return the keyword that was matched by reaching this state | 124 * @return the keyword that was matched by reaching this state |
| 125 */ | 125 */ |
| 126 Keyword keyword() => _keyword2; | 126 Keyword keyword() => _keyword; |
| 127 | 127 |
| 128 /** | 128 /** |
| 129 * Return the state that follows this state on a transition of the given chara
cter, or | 129 * Return the state that follows this state on a transition of the given chara
cter, or |
| 130 * `null` if there is no valid state reachable from this state with such a tra
nsition. | 130 * `null` if there is no valid state reachable from this state with such a tra
nsition. |
| 131 * | 131 * |
| 132 * @param c the character used to transition from this state to another state | 132 * @param c the character used to transition from this state to another state |
| 133 * @return the state that follows this state on a transition of the given char
acter | 133 * @return the state that follows this state on a transition of the given char
acter |
| 134 */ | 134 */ |
| 135 KeywordState next(int c) => _table[c - 0x61]; | 135 KeywordState next(int c) => _table[c - 0x61]; |
| 136 } | 136 } |
| (...skipping 277 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 414 | 414 |
| 415 /** | 415 /** |
| 416 * The lexeme for the keyword. | 416 * The lexeme for the keyword. |
| 417 */ | 417 */ |
| 418 String _syntax; | 418 String _syntax; |
| 419 | 419 |
| 420 /** | 420 /** |
| 421 * A flag indicating whether the keyword is a pseudo-keyword. Pseudo keywords
can be used as | 421 * A flag indicating whether the keyword is a pseudo-keyword. Pseudo keywords
can be used as |
| 422 * identifiers. | 422 * identifiers. |
| 423 */ | 423 */ |
| 424 bool _isPseudoKeyword2 = false; | 424 bool _isPseudoKeyword = false; |
| 425 | 425 |
| 426 /** | 426 /** |
| 427 * A table mapping the lexemes of keywords to the corresponding keyword. | 427 * A table mapping the lexemes of keywords to the corresponding keyword. |
| 428 */ | 428 */ |
| 429 static Map<String, Keyword> keywords = createKeywordMap(); | 429 static Map<String, Keyword> keywords = _createKeywordMap(); |
| 430 | 430 |
| 431 /** | 431 /** |
| 432 * Create a table mapping the lexemes of keywords to the corresponding keyword
. | 432 * Create a table mapping the lexemes of keywords to the corresponding keyword
. |
| 433 * | 433 * |
| 434 * @return the table that was created | 434 * @return the table that was created |
| 435 */ | 435 */ |
| 436 static Map<String, Keyword> createKeywordMap() { | 436 static Map<String, Keyword> _createKeywordMap() { |
| 437 LinkedHashMap<String, Keyword> result = new LinkedHashMap<String, Keyword>()
; | 437 LinkedHashMap<String, Keyword> result = new LinkedHashMap<String, Keyword>()
; |
| 438 for (Keyword keyword in values) { | 438 for (Keyword keyword in values) { |
| 439 result[keyword._syntax] = keyword; | 439 result[keyword._syntax] = keyword; |
| 440 } | 440 } |
| 441 return result; | 441 return result; |
| 442 } | 442 } |
| 443 | 443 |
| 444 /** | 444 /** |
| 445 * Initialize a newly created keyword to have the given syntax. The keyword is
not a | 445 * Initialize a newly created keyword to have the given syntax. The keyword is
not a |
| 446 * pseudo-keyword. | 446 * pseudo-keyword. |
| 447 * | 447 * |
| 448 * @param syntax the lexeme for the keyword | 448 * @param syntax the lexeme for the keyword |
| 449 */ | 449 */ |
| 450 Keyword.con1(String name, int ordinal, String syntax) : this.con2(name, ordina
l, syntax, false); | 450 Keyword.con1(String name, int ordinal, String syntax) : this.con2(name, ordina
l, syntax, false); |
| 451 | 451 |
| 452 /** | 452 /** |
| 453 * Initialize a newly created keyword to have the given syntax. The keyword is
a pseudo-keyword if | 453 * Initialize a newly created keyword to have the given syntax. The keyword is
a pseudo-keyword if |
| 454 * the given flag is `true`. | 454 * the given flag is `true`. |
| 455 * | 455 * |
| 456 * @param syntax the lexeme for the keyword | 456 * @param syntax the lexeme for the keyword |
| 457 * @param isPseudoKeyword `true` if this keyword is a pseudo-keyword | 457 * @param isPseudoKeyword `true` if this keyword is a pseudo-keyword |
| 458 */ | 458 */ |
| 459 Keyword.con2(String name, int ordinal, String syntax, bool isPseudoKeyword) :
super(name, ordinal) { | 459 Keyword.con2(String name, int ordinal, String syntax, bool isPseudoKeyword) :
super(name, ordinal) { |
| 460 this._syntax = syntax; | 460 this._syntax = syntax; |
| 461 this._isPseudoKeyword2 = isPseudoKeyword; | 461 this._isPseudoKeyword = isPseudoKeyword; |
| 462 } | 462 } |
| 463 | 463 |
| 464 /** | 464 /** |
| 465 * Return the lexeme for the keyword. | 465 * Return the lexeme for the keyword. |
| 466 * | 466 * |
| 467 * @return the lexeme for the keyword | 467 * @return the lexeme for the keyword |
| 468 */ | 468 */ |
| 469 String get syntax => _syntax; | 469 String get syntax => _syntax; |
| 470 | 470 |
| 471 /** | 471 /** |
| 472 * Return `true` if this keyword is a pseudo-keyword. Pseudo keywords can be u
sed as | 472 * Return `true` if this keyword is a pseudo-keyword. Pseudo keywords can be u
sed as |
| 473 * identifiers. | 473 * identifiers. |
| 474 * | 474 * |
| 475 * @return `true` if this keyword is a pseudo-keyword | 475 * @return `true` if this keyword is a pseudo-keyword |
| 476 */ | 476 */ |
| 477 bool get isPseudoKeyword => _isPseudoKeyword2; | 477 bool get isPseudoKeyword => _isPseudoKeyword; |
| 478 } | 478 } |
| 479 | 479 |
| 480 /** | 480 /** |
| 481 * Instances of the class `CharSequenceReader` implement a [CharacterReader] tha
t reads | 481 * Instances of the class `CharSequenceReader` implement a [CharacterReader] tha
t reads |
| 482 * characters from a character sequence. | 482 * characters from a character sequence. |
| 483 */ | 483 */ |
| 484 class CharSequenceReader implements CharacterReader { | 484 class CharSequenceReader implements CharacterReader { |
| 485 /** | 485 /** |
| 486 * The sequence from which characters will be read. | 486 * The sequence from which characters will be read. |
| 487 */ | 487 */ |
| (...skipping 84 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 572 /** | 572 /** |
| 573 * The token in the new token stream immediately to the right of the range of
tokens that were | 573 * The token in the new token stream immediately to the right of the range of
tokens that were |
| 574 * inserted, or the token immediately to the right of the modified region if t
here were no new | 574 * inserted, or the token immediately to the right of the modified region if t
here were no new |
| 575 * tokens. | 575 * tokens. |
| 576 */ | 576 */ |
| 577 Token _rightToken; | 577 Token _rightToken; |
| 578 | 578 |
| 579 /** | 579 /** |
| 580 * A flag indicating whether there were any tokens changed as a result of the
modification. | 580 * A flag indicating whether there were any tokens changed as a result of the
modification. |
| 581 */ | 581 */ |
| 582 bool _hasNonWhitespaceChange2 = false; | 582 bool _hasNonWhitespaceChange = false; |
| 583 | 583 |
| 584 /** | 584 /** |
| 585 * Initialize a newly created scanner. | 585 * Initialize a newly created scanner. |
| 586 * | 586 * |
| 587 * @param source the source being scanned | 587 * @param source the source being scanned |
| 588 * @param reader the character reader used to read the characters in the sourc
e | 588 * @param reader the character reader used to read the characters in the sourc
e |
| 589 * @param errorListener the error listener that will be informed of any errors
that are found | 589 * @param errorListener the error listener that will be informed of any errors
that are found |
| 590 */ | 590 */ |
| 591 IncrementalScanner(Source source, CharacterReader reader, AnalysisErrorListene
r errorListener) : super(source, reader, errorListener) { | 591 IncrementalScanner(Source source, CharacterReader reader, AnalysisErrorListene
r errorListener) : super(source, reader, errorListener) { |
| 592 this._reader = reader; | 592 this._reader = reader; |
| (...skipping 23 matching lines...) Expand all Loading... |
| 616 * @return a map from tokens that were copied to the copies of the tokens | 616 * @return a map from tokens that were copied to the copies of the tokens |
| 617 */ | 617 */ |
| 618 TokenMap get tokenMap => _tokenMap; | 618 TokenMap get tokenMap => _tokenMap; |
| 619 | 619 |
| 620 /** | 620 /** |
| 621 * Return `true` if there were any tokens either added or removed (or both) as
a result of | 621 * Return `true` if there were any tokens either added or removed (or both) as
a result of |
| 622 * the modification. | 622 * the modification. |
| 623 * | 623 * |
| 624 * @return `true` if there were any tokens changed as a result of the modifica
tion | 624 * @return `true` if there were any tokens changed as a result of the modifica
tion |
| 625 */ | 625 */ |
| 626 bool get hasNonWhitespaceChange => _hasNonWhitespaceChange2; | 626 bool get hasNonWhitespaceChange => _hasNonWhitespaceChange; |
| 627 | 627 |
| 628 /** | 628 /** |
| 629 * Given the stream of tokens scanned from the original source, the modified s
ource (the result of | 629 * Given the stream of tokens scanned from the original source, the modified s
ource (the result of |
| 630 * replacing one contiguous range of characters with another string of charact
ers), and a | 630 * replacing one contiguous range of characters with another string of charact
ers), and a |
| 631 * specification of the modification that was made, return a stream of tokens
scanned from the | 631 * specification of the modification that was made, return a stream of tokens
scanned from the |
| 632 * modified source. The original stream of tokens will not be modified. | 632 * modified source. The original stream of tokens will not be modified. |
| 633 * | 633 * |
| 634 * @param originalStream the stream of tokens scanned from the original source | 634 * @param originalStream the stream of tokens scanned from the original source |
| 635 * @param index the index of the first character in both the original and modi
fied source that was | 635 * @param index the index of the first character in both the original and modi
fied source that was |
| 636 * affected by the modification | 636 * affected by the modification |
| 637 * @param removedLength the number of characters removed from the original sou
rce | 637 * @param removedLength the number of characters removed from the original sou
rce |
| 638 * @param insertedLength the number of characters added to the modified source | 638 * @param insertedLength the number of characters added to the modified source |
| 639 */ | 639 */ |
| 640 Token rescan(Token originalStream, int index, int removedLength, int insertedL
ength) { | 640 Token rescan(Token originalStream, int index, int removedLength, int insertedL
ength) { |
| 641 // | 641 // |
| 642 // Copy all of the tokens in the originalStream whose end is less than the r
eplacement start. | 642 // Copy all of the tokens in the originalStream whose end is less than the r
eplacement start. |
| 643 // (If the replacement start is equal to the end of an existing token, then
it means that the | 643 // (If the replacement start is equal to the end of an existing token, then
it means that the |
| 644 // existing token might have been modified, so we need to rescan it.) | 644 // existing token might have been modified, so we need to rescan it.) |
| 645 // | 645 // |
| 646 while (originalStream.type != TokenType.EOF && originalStream.end < index) { | 646 while (originalStream.type != TokenType.EOF && originalStream.end < index) { |
| 647 originalStream = copyAndAdvance(originalStream, 0); | 647 originalStream = _copyAndAdvance(originalStream, 0); |
| 648 } | 648 } |
| 649 Token oldFirst = originalStream; | 649 Token oldFirst = originalStream; |
| 650 Token oldLeftToken = originalStream.previous; | 650 Token oldLeftToken = originalStream.previous; |
| 651 _leftToken = tail; | 651 _leftToken = tail; |
| 652 // | 652 // |
| 653 // Skip tokens in the original stream until we find a token whose offset is
greater than the end | 653 // Skip tokens in the original stream until we find a token whose offset is
greater than the end |
| 654 // of the removed region. (If the end of the removed region is equal to the
beginning of an | 654 // of the removed region. (If the end of the removed region is equal to the
beginning of an |
| 655 // existing token, then it means that the existing token might have been mod
ified, so we need to | 655 // existing token, then it means that the existing token might have been mod
ified, so we need to |
| 656 // rescan it.) | 656 // rescan it.) |
| 657 // | 657 // |
| (...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 690 // | 690 // |
| 691 _reader.offset = scanStart - 1; | 691 _reader.offset = scanStart - 1; |
| 692 int next = _reader.advance(); | 692 int next = _reader.advance(); |
| 693 while (next != -1 && _reader.offset <= scanEnd) { | 693 while (next != -1 && _reader.offset <= scanEnd) { |
| 694 next = bigSwitch(next); | 694 next = bigSwitch(next); |
| 695 } | 695 } |
| 696 // | 696 // |
| 697 // Copy the remaining tokens in the original stream, but apply the delta to
the token's offset. | 697 // Copy the remaining tokens in the original stream, but apply the delta to
the token's offset. |
| 698 // | 698 // |
| 699 if (identical(originalStream.type, TokenType.EOF)) { | 699 if (identical(originalStream.type, TokenType.EOF)) { |
| 700 copyAndAdvance(originalStream, delta); | 700 _copyAndAdvance(originalStream, delta); |
| 701 _rightToken = tail; | 701 _rightToken = tail; |
| 702 _rightToken.setNextWithoutSettingPrevious(_rightToken); | 702 _rightToken.setNextWithoutSettingPrevious(_rightToken); |
| 703 } else { | 703 } else { |
| 704 originalStream = copyAndAdvance(originalStream, delta); | 704 originalStream = _copyAndAdvance(originalStream, delta); |
| 705 _rightToken = tail; | 705 _rightToken = tail; |
| 706 while (originalStream.type != TokenType.EOF) { | 706 while (originalStream.type != TokenType.EOF) { |
| 707 originalStream = copyAndAdvance(originalStream, delta); | 707 originalStream = _copyAndAdvance(originalStream, delta); |
| 708 } | 708 } |
| 709 Token eof = copyAndAdvance(originalStream, delta); | 709 Token eof = _copyAndAdvance(originalStream, delta); |
| 710 eof.setNextWithoutSettingPrevious(eof); | 710 eof.setNextWithoutSettingPrevious(eof); |
| 711 } | 711 } |
| 712 // | 712 // |
| 713 // If the index is immediately after an existing token and the inserted char
acters did not | 713 // If the index is immediately after an existing token and the inserted char
acters did not |
| 714 // change that original token, then adjust the leftToken to be the next toke
n. For example, in | 714 // change that original token, then adjust the leftToken to be the next toke
n. For example, in |
| 715 // "a; c;" --> "a;b c;", the leftToken was ";", but this code advances it to
"b" since "b" is | 715 // "a; c;" --> "a;b c;", the leftToken was ";", but this code advances it to
"b" since "b" is |
| 716 // the first new token. | 716 // the first new token. |
| 717 // | 717 // |
| 718 Token newFirst = _leftToken.next; | 718 Token newFirst = _leftToken.next; |
| 719 while (newFirst != _rightToken && oldFirst != oldRightToken && newFirst.type
!= TokenType.EOF && equalTokens(oldFirst, newFirst)) { | 719 while (newFirst != _rightToken && oldFirst != oldRightToken && newFirst.type
!= TokenType.EOF && _equalTokens(oldFirst, newFirst)) { |
| 720 _tokenMap.put(oldFirst, newFirst); | 720 _tokenMap.put(oldFirst, newFirst); |
| 721 oldLeftToken = oldFirst; | 721 oldLeftToken = oldFirst; |
| 722 oldFirst = oldFirst.next; | 722 oldFirst = oldFirst.next; |
| 723 _leftToken = newFirst; | 723 _leftToken = newFirst; |
| 724 newFirst = newFirst.next; | 724 newFirst = newFirst.next; |
| 725 } | 725 } |
| 726 Token newLast = _rightToken.previous; | 726 Token newLast = _rightToken.previous; |
| 727 while (newLast != _leftToken && oldLast != oldLeftToken && newLast.type != T
okenType.EOF && equalTokens(oldLast, newLast)) { | 727 while (newLast != _leftToken && oldLast != oldLeftToken && newLast.type != T
okenType.EOF && _equalTokens(oldLast, newLast)) { |
| 728 _tokenMap.put(oldLast, newLast); | 728 _tokenMap.put(oldLast, newLast); |
| 729 oldRightToken = oldLast; | 729 oldRightToken = oldLast; |
| 730 oldLast = oldLast.previous; | 730 oldLast = oldLast.previous; |
| 731 _rightToken = newLast; | 731 _rightToken = newLast; |
| 732 newLast = newLast.previous; | 732 newLast = newLast.previous; |
| 733 } | 733 } |
| 734 _hasNonWhitespaceChange2 = _leftToken.next != _rightToken || oldLeftToken.ne
xt != oldRightToken; | 734 _hasNonWhitespaceChange = _leftToken.next != _rightToken || oldLeftToken.nex
t != oldRightToken; |
| 735 // | 735 // |
| 736 // TODO(brianwilkerson) Begin tokens are not getting associated with the cor
responding end | 736 // TODO(brianwilkerson) Begin tokens are not getting associated with the cor
responding end |
| 737 // tokens (because the end tokens have not been copied when we're copyin
g the begin tokens). | 737 // tokens (because the end tokens have not been copied when we're copyin
g the begin tokens). |
| 738 // This could have implications for parsing. | 738 // This could have implications for parsing. |
| 739 // TODO(brianwilkerson) Update the lineInfo. | 739 // TODO(brianwilkerson) Update the lineInfo. |
| 740 // | 740 // |
| 741 return firstToken; | 741 return firstToken; |
| 742 } | 742 } |
| 743 | 743 |
| 744 Token copyAndAdvance(Token originalToken, int delta) { | 744 Token _copyAndAdvance(Token originalToken, int delta) { |
| 745 Token copiedToken = originalToken.copy(); | 745 Token copiedToken = originalToken.copy(); |
| 746 _tokenMap.put(originalToken, copiedToken); | 746 _tokenMap.put(originalToken, copiedToken); |
| 747 copiedToken.applyDelta(delta); | 747 copiedToken.applyDelta(delta); |
| 748 appendToken(copiedToken); | 748 appendToken(copiedToken); |
| 749 Token originalComment = originalToken.precedingComments; | 749 Token originalComment = originalToken.precedingComments; |
| 750 Token copiedComment = originalToken.precedingComments; | 750 Token copiedComment = originalToken.precedingComments; |
| 751 while (originalComment != null) { | 751 while (originalComment != null) { |
| 752 _tokenMap.put(originalComment, copiedComment); | 752 _tokenMap.put(originalComment, copiedComment); |
| 753 originalComment = originalComment.next; | 753 originalComment = originalComment.next; |
| 754 copiedComment = copiedComment.next; | 754 copiedComment = copiedComment.next; |
| 755 } | 755 } |
| 756 return originalToken.next; | 756 return originalToken.next; |
| 757 } | 757 } |
| 758 | 758 |
| 759 /** | 759 /** |
| 760 * Return `true` if the two tokens are equal to each other. For the purposes o
f the | 760 * Return `true` if the two tokens are equal to each other. For the purposes o
f the |
| 761 * incremental scanner, two tokens are equal if they have the same type and le
xeme. | 761 * incremental scanner, two tokens are equal if they have the same type and le
xeme. |
| 762 * | 762 * |
| 763 * @param oldToken the token from the old stream that is being compared | 763 * @param oldToken the token from the old stream that is being compared |
| 764 * @param newToken the token from the new stream that is being compared | 764 * @param newToken the token from the new stream that is being compared |
| 765 * @return `true` if the two tokens are equal to each other | 765 * @return `true` if the two tokens are equal to each other |
| 766 */ | 766 */ |
| 767 bool equalTokens(Token oldToken, Token newToken) => identical(oldToken.type, n
ewToken.type) && oldToken.length == newToken.length && oldToken.lexeme == newTok
en.lexeme; | 767 bool _equalTokens(Token oldToken, Token newToken) => identical(oldToken.type,
newToken.type) && oldToken.length == newToken.length && oldToken.lexeme == newTo
ken.lexeme; |
| 768 } | 768 } |
| 769 | 769 |
| 770 /** | 770 /** |
| 771 * The class `Scanner` implements a scanner for Dart code. | 771 * The class `Scanner` implements a scanner for Dart code. |
| 772 * | 772 * |
| 773 * The lexical structure of Dart is ambiguous without knowledge of the context i
n which a token is | 773 * The lexical structure of Dart is ambiguous without knowledge of the context i
n which a token is |
| 774 * being scanned. For example, without context we cannot determine whether sourc
e of the form "<<" | 774 * being scanned. For example, without context we cannot determine whether sourc
e of the form "<<" |
| 775 * should be scanned as a single left-shift operator or as two left angle bracke
ts. This scanner | 775 * should be scanned as a single left-shift operator or as two left angle bracke
ts. This scanner |
| 776 * does not have any context, so it always resolves such conflicts by scanning t
he longest possible | 776 * does not have any context, so it always resolves such conflicts by scanning t
he longest possible |
| 777 * token. | 777 * token. |
| (...skipping 56 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 834 List<BeginToken> _groupingStack = new List<BeginToken>(); | 834 List<BeginToken> _groupingStack = new List<BeginToken>(); |
| 835 | 835 |
| 836 /** | 836 /** |
| 837 * The index of the last item in the [groupingStack], or `-1` if the stack is
empty. | 837 * The index of the last item in the [groupingStack], or `-1` if the stack is
empty. |
| 838 */ | 838 */ |
| 839 int _stackEnd = -1; | 839 int _stackEnd = -1; |
| 840 | 840 |
| 841 /** | 841 /** |
| 842 * A flag indicating whether any unmatched groups were found during the parse. | 842 * A flag indicating whether any unmatched groups were found during the parse. |
| 843 */ | 843 */ |
| 844 bool _hasUnmatchedGroups2 = false; | 844 bool _hasUnmatchedGroups = false; |
| 845 | 845 |
| 846 /** | 846 /** |
| 847 * Initialize a newly created scanner. | 847 * Initialize a newly created scanner. |
| 848 * | 848 * |
| 849 * @param source the source being scanned | 849 * @param source the source being scanned |
| 850 * @param reader the character reader used to read the characters in the sourc
e | 850 * @param reader the character reader used to read the characters in the sourc
e |
| 851 * @param errorListener the error listener that will be informed of any errors
that are found | 851 * @param errorListener the error listener that will be informed of any errors
that are found |
| 852 */ | 852 */ |
| 853 Scanner(this.source, CharacterReader reader, AnalysisErrorListener errorListen
er) { | 853 Scanner(this.source, CharacterReader reader, AnalysisErrorListener errorListen
er) { |
| 854 this._reader = reader; | 854 this._reader = reader; |
| (...skipping 10 matching lines...) Expand all Loading... |
| 865 * | 865 * |
| 866 * @return an array containing the offsets of the first character of each line
in the source code | 866 * @return an array containing the offsets of the first character of each line
in the source code |
| 867 */ | 867 */ |
| 868 List<int> get lineStarts => _lineStarts; | 868 List<int> get lineStarts => _lineStarts; |
| 869 | 869 |
| 870 /** | 870 /** |
| 871 * Return `true` if any unmatched groups were found during the parse. | 871 * Return `true` if any unmatched groups were found during the parse. |
| 872 * | 872 * |
| 873 * @return `true` if any unmatched groups were found during the parse | 873 * @return `true` if any unmatched groups were found during the parse |
| 874 */ | 874 */ |
| 875 bool get hasUnmatchedGroups => _hasUnmatchedGroups2; | 875 bool get hasUnmatchedGroups => _hasUnmatchedGroups; |
| 876 | 876 |
| 877 /** | 877 /** |
| 878 * Set whether documentation tokens should be scanned. | 878 * Set whether documentation tokens should be scanned. |
| 879 * | 879 * |
| 880 * @param preserveComments `true` if documentation tokens should be scanned | 880 * @param preserveComments `true` if documentation tokens should be scanned |
| 881 */ | 881 */ |
| 882 void set preserveComments(bool preserveComments) { | 882 void set preserveComments(bool preserveComments) { |
| 883 this._preserveComments = preserveComments; | 883 this._preserveComments = preserveComments; |
| 884 } | 884 } |
| 885 | 885 |
| (...skipping 26 matching lines...) Expand all Loading... |
| 912 */ | 912 */ |
| 913 Token tokenize() { | 913 Token tokenize() { |
| 914 InstrumentationBuilder instrumentation = Instrumentation.builder2("dart.engi
ne.AbstractScanner.tokenize"); | 914 InstrumentationBuilder instrumentation = Instrumentation.builder2("dart.engi
ne.AbstractScanner.tokenize"); |
| 915 int tokenCounter = 0; | 915 int tokenCounter = 0; |
| 916 try { | 916 try { |
| 917 int next = _reader.advance(); | 917 int next = _reader.advance(); |
| 918 while (next != -1) { | 918 while (next != -1) { |
| 919 tokenCounter++; | 919 tokenCounter++; |
| 920 next = bigSwitch(next); | 920 next = bigSwitch(next); |
| 921 } | 921 } |
| 922 appendEofToken(); | 922 _appendEofToken(); |
| 923 instrumentation.metric2("tokensCount", tokenCounter); | 923 instrumentation.metric2("tokensCount", tokenCounter); |
| 924 return firstToken; | 924 return firstToken; |
| 925 } finally { | 925 } finally { |
| 926 instrumentation.log2(2); | 926 instrumentation.log2(2); |
| 927 } | 927 } |
| 928 } | 928 } |
| 929 | 929 |
| 930 /** | 930 /** |
| 931 * Append the given token to the end of the token stream being scanned. This m
ethod is intended to | 931 * Append the given token to the end of the token stream being scanned. This m
ethod is intended to |
| 932 * be used by subclasses that copy existing tokens and should not normally be
used because it will | 932 * be used by subclasses that copy existing tokens and should not normally be
used because it will |
| 933 * fail to correctly associate any comments with the token being passed in. | 933 * fail to correctly associate any comments with the token being passed in. |
| 934 * | 934 * |
| 935 * @param token the token to be appended | 935 * @param token the token to be appended |
| 936 */ | 936 */ |
| 937 void appendToken(Token token) { | 937 void appendToken(Token token) { |
| 938 _tail = _tail.setNext(token); | 938 _tail = _tail.setNext(token); |
| 939 } | 939 } |
| 940 | 940 |
| 941 int bigSwitch(int next) { | 941 int bigSwitch(int next) { |
| 942 beginToken(); | 942 _beginToken(); |
| 943 if (next == 0xD) { | 943 if (next == 0xD) { |
| 944 next = _reader.advance(); | 944 next = _reader.advance(); |
| 945 if (next == 0xA) { | 945 if (next == 0xA) { |
| 946 next = _reader.advance(); | 946 next = _reader.advance(); |
| 947 } | 947 } |
| 948 recordStartOfLine(); | 948 recordStartOfLine(); |
| 949 return next; | 949 return next; |
| 950 } else if (next == 0xA) { | 950 } else if (next == 0xA) { |
| 951 next = _reader.advance(); | 951 next = _reader.advance(); |
| 952 recordStartOfLine(); | 952 recordStartOfLine(); |
| 953 return next; | 953 return next; |
| 954 } else if (next == 0x9 || next == 0x20) { | 954 } else if (next == 0x9 || next == 0x20) { |
| 955 return _reader.advance(); | 955 return _reader.advance(); |
| 956 } | 956 } |
| 957 if (next == 0x72) { | 957 if (next == 0x72) { |
| 958 int peek = _reader.peek(); | 958 int peek = _reader.peek(); |
| 959 if (peek == 0x22 || peek == 0x27) { | 959 if (peek == 0x22 || peek == 0x27) { |
| 960 int start = _reader.offset; | 960 int start = _reader.offset; |
| 961 return tokenizeString(_reader.advance(), start, true); | 961 return _tokenizeString(_reader.advance(), start, true); |
| 962 } | 962 } |
| 963 } | 963 } |
| 964 if (0x61 <= next && next <= 0x7A) { | 964 if (0x61 <= next && next <= 0x7A) { |
| 965 return tokenizeKeywordOrIdentifier(next, true); | 965 return _tokenizeKeywordOrIdentifier(next, true); |
| 966 } | 966 } |
| 967 if ((0x41 <= next && next <= 0x5A) || next == 0x5F || next == 0x24) { | 967 if ((0x41 <= next && next <= 0x5A) || next == 0x5F || next == 0x24) { |
| 968 return tokenizeIdentifier(next, _reader.offset, true); | 968 return _tokenizeIdentifier(next, _reader.offset, true); |
| 969 } | 969 } |
| 970 if (next == 0x3C) { | 970 if (next == 0x3C) { |
| 971 return tokenizeLessThan(next); | 971 return _tokenizeLessThan(next); |
| 972 } | 972 } |
| 973 if (next == 0x3E) { | 973 if (next == 0x3E) { |
| 974 return tokenizeGreaterThan(next); | 974 return _tokenizeGreaterThan(next); |
| 975 } | 975 } |
| 976 if (next == 0x3D) { | 976 if (next == 0x3D) { |
| 977 return tokenizeEquals(next); | 977 return _tokenizeEquals(next); |
| 978 } | 978 } |
| 979 if (next == 0x21) { | 979 if (next == 0x21) { |
| 980 return tokenizeExclamation(next); | 980 return _tokenizeExclamation(next); |
| 981 } | 981 } |
| 982 if (next == 0x2B) { | 982 if (next == 0x2B) { |
| 983 return tokenizePlus(next); | 983 return _tokenizePlus(next); |
| 984 } | 984 } |
| 985 if (next == 0x2D) { | 985 if (next == 0x2D) { |
| 986 return tokenizeMinus(next); | 986 return _tokenizeMinus(next); |
| 987 } | 987 } |
| 988 if (next == 0x2A) { | 988 if (next == 0x2A) { |
| 989 return tokenizeMultiply(next); | 989 return _tokenizeMultiply(next); |
| 990 } | 990 } |
| 991 if (next == 0x25) { | 991 if (next == 0x25) { |
| 992 return tokenizePercent(next); | 992 return _tokenizePercent(next); |
| 993 } | 993 } |
| 994 if (next == 0x26) { | 994 if (next == 0x26) { |
| 995 return tokenizeAmpersand(next); | 995 return _tokenizeAmpersand(next); |
| 996 } | 996 } |
| 997 if (next == 0x7C) { | 997 if (next == 0x7C) { |
| 998 return tokenizeBar(next); | 998 return _tokenizeBar(next); |
| 999 } | 999 } |
| 1000 if (next == 0x5E) { | 1000 if (next == 0x5E) { |
| 1001 return tokenizeCaret(next); | 1001 return _tokenizeCaret(next); |
| 1002 } | 1002 } |
| 1003 if (next == 0x5B) { | 1003 if (next == 0x5B) { |
| 1004 return tokenizeOpenSquareBracket(next); | 1004 return _tokenizeOpenSquareBracket(next); |
| 1005 } | 1005 } |
| 1006 if (next == 0x7E) { | 1006 if (next == 0x7E) { |
| 1007 return tokenizeTilde(next); | 1007 return _tokenizeTilde(next); |
| 1008 } | 1008 } |
| 1009 if (next == 0x5C) { | 1009 if (next == 0x5C) { |
| 1010 appendTokenOfType(TokenType.BACKSLASH); | 1010 _appendTokenOfType(TokenType.BACKSLASH); |
| 1011 return _reader.advance(); | 1011 return _reader.advance(); |
| 1012 } | 1012 } |
| 1013 if (next == 0x23) { | 1013 if (next == 0x23) { |
| 1014 return tokenizeTag(next); | 1014 return _tokenizeTag(next); |
| 1015 } | 1015 } |
| 1016 if (next == 0x28) { | 1016 if (next == 0x28) { |
| 1017 appendBeginToken(TokenType.OPEN_PAREN); | 1017 _appendBeginToken(TokenType.OPEN_PAREN); |
| 1018 return _reader.advance(); | 1018 return _reader.advance(); |
| 1019 } | 1019 } |
| 1020 if (next == 0x29) { | 1020 if (next == 0x29) { |
| 1021 appendEndToken(TokenType.CLOSE_PAREN, TokenType.OPEN_PAREN); | 1021 _appendEndToken(TokenType.CLOSE_PAREN, TokenType.OPEN_PAREN); |
| 1022 return _reader.advance(); | 1022 return _reader.advance(); |
| 1023 } | 1023 } |
| 1024 if (next == 0x2C) { | 1024 if (next == 0x2C) { |
| 1025 appendTokenOfType(TokenType.COMMA); | 1025 _appendTokenOfType(TokenType.COMMA); |
| 1026 return _reader.advance(); | 1026 return _reader.advance(); |
| 1027 } | 1027 } |
| 1028 if (next == 0x3A) { | 1028 if (next == 0x3A) { |
| 1029 appendTokenOfType(TokenType.COLON); | 1029 _appendTokenOfType(TokenType.COLON); |
| 1030 return _reader.advance(); | 1030 return _reader.advance(); |
| 1031 } | 1031 } |
| 1032 if (next == 0x3B) { | 1032 if (next == 0x3B) { |
| 1033 appendTokenOfType(TokenType.SEMICOLON); | 1033 _appendTokenOfType(TokenType.SEMICOLON); |
| 1034 return _reader.advance(); | 1034 return _reader.advance(); |
| 1035 } | 1035 } |
| 1036 if (next == 0x3F) { | 1036 if (next == 0x3F) { |
| 1037 appendTokenOfType(TokenType.QUESTION); | 1037 _appendTokenOfType(TokenType.QUESTION); |
| 1038 return _reader.advance(); | 1038 return _reader.advance(); |
| 1039 } | 1039 } |
| 1040 if (next == 0x5D) { | 1040 if (next == 0x5D) { |
| 1041 appendEndToken(TokenType.CLOSE_SQUARE_BRACKET, TokenType.OPEN_SQUARE_BRACK
ET); | 1041 _appendEndToken(TokenType.CLOSE_SQUARE_BRACKET, TokenType.OPEN_SQUARE_BRAC
KET); |
| 1042 return _reader.advance(); | 1042 return _reader.advance(); |
| 1043 } | 1043 } |
| 1044 if (next == 0x60) { | 1044 if (next == 0x60) { |
| 1045 appendTokenOfType(TokenType.BACKPING); | 1045 _appendTokenOfType(TokenType.BACKPING); |
| 1046 return _reader.advance(); | 1046 return _reader.advance(); |
| 1047 } | 1047 } |
| 1048 if (next == 0x7B) { | 1048 if (next == 0x7B) { |
| 1049 appendBeginToken(TokenType.OPEN_CURLY_BRACKET); | 1049 _appendBeginToken(TokenType.OPEN_CURLY_BRACKET); |
| 1050 return _reader.advance(); | 1050 return _reader.advance(); |
| 1051 } | 1051 } |
| 1052 if (next == 0x7D) { | 1052 if (next == 0x7D) { |
| 1053 appendEndToken(TokenType.CLOSE_CURLY_BRACKET, TokenType.OPEN_CURLY_BRACKET
); | 1053 _appendEndToken(TokenType.CLOSE_CURLY_BRACKET, TokenType.OPEN_CURLY_BRACKE
T); |
| 1054 return _reader.advance(); | 1054 return _reader.advance(); |
| 1055 } | 1055 } |
| 1056 if (next == 0x2F) { | 1056 if (next == 0x2F) { |
| 1057 return tokenizeSlashOrComment(next); | 1057 return _tokenizeSlashOrComment(next); |
| 1058 } | 1058 } |
| 1059 if (next == 0x40) { | 1059 if (next == 0x40) { |
| 1060 appendTokenOfType(TokenType.AT); | 1060 _appendTokenOfType(TokenType.AT); |
| 1061 return _reader.advance(); | 1061 return _reader.advance(); |
| 1062 } | 1062 } |
| 1063 if (next == 0x22 || next == 0x27) { | 1063 if (next == 0x22 || next == 0x27) { |
| 1064 return tokenizeString(next, _reader.offset, false); | 1064 return _tokenizeString(next, _reader.offset, false); |
| 1065 } | 1065 } |
| 1066 if (next == 0x2E) { | 1066 if (next == 0x2E) { |
| 1067 return tokenizeDotOrNumber(next); | 1067 return _tokenizeDotOrNumber(next); |
| 1068 } | 1068 } |
| 1069 if (next == 0x30) { | 1069 if (next == 0x30) { |
| 1070 return tokenizeHexOrNumber(next); | 1070 return _tokenizeHexOrNumber(next); |
| 1071 } | 1071 } |
| 1072 if (0x31 <= next && next <= 0x39) { | 1072 if (0x31 <= next && next <= 0x39) { |
| 1073 return tokenizeNumber(next); | 1073 return _tokenizeNumber(next); |
| 1074 } | 1074 } |
| 1075 if (next == -1) { | 1075 if (next == -1) { |
| 1076 return -1; | 1076 return -1; |
| 1077 } | 1077 } |
| 1078 reportError(ScannerErrorCode.ILLEGAL_CHARACTER, [next]); | 1078 _reportError(ScannerErrorCode.ILLEGAL_CHARACTER, [next]); |
| 1079 return _reader.advance(); | 1079 return _reader.advance(); |
| 1080 } | 1080 } |
| 1081 | 1081 |
| 1082 /** | 1082 /** |
| 1083 * Return the first token in the token stream that was scanned. | 1083 * Return the first token in the token stream that was scanned. |
| 1084 * | 1084 * |
| 1085 * @return the first token in the token stream that was scanned | 1085 * @return the first token in the token stream that was scanned |
| 1086 */ | 1086 */ |
| 1087 Token get firstToken => _tokens.next; | 1087 Token get firstToken => _tokens.next; |
| 1088 | 1088 |
| 1089 /** | 1089 /** |
| 1090 * Return the last token that was scanned. | 1090 * Return the last token that was scanned. |
| 1091 * | 1091 * |
| 1092 * @return the last token that was scanned | 1092 * @return the last token that was scanned |
| 1093 */ | 1093 */ |
| 1094 Token get tail => _tail; | 1094 Token get tail => _tail; |
| 1095 | 1095 |
| 1096 /** | 1096 /** |
| 1097 * Record the fact that we are at the beginning of a new line in the source. | 1097 * Record the fact that we are at the beginning of a new line in the source. |
| 1098 */ | 1098 */ |
| 1099 void recordStartOfLine() { | 1099 void recordStartOfLine() { |
| 1100 _lineStarts.add(_reader.offset); | 1100 _lineStarts.add(_reader.offset); |
| 1101 } | 1101 } |
| 1102 | 1102 |
| 1103 void appendBeginToken(TokenType type) { | 1103 void _appendBeginToken(TokenType type) { |
| 1104 BeginToken token; | 1104 BeginToken token; |
| 1105 if (_firstComment == null) { | 1105 if (_firstComment == null) { |
| 1106 token = new BeginToken(type, _tokenStart); | 1106 token = new BeginToken(type, _tokenStart); |
| 1107 } else { | 1107 } else { |
| 1108 token = new BeginTokenWithComment(type, _tokenStart, _firstComment); | 1108 token = new BeginTokenWithComment(type, _tokenStart, _firstComment); |
| 1109 _firstComment = null; | 1109 _firstComment = null; |
| 1110 _lastComment = null; | 1110 _lastComment = null; |
| 1111 } | 1111 } |
| 1112 _tail = _tail.setNext(token); | 1112 _tail = _tail.setNext(token); |
| 1113 _groupingStack.add(token); | 1113 _groupingStack.add(token); |
| 1114 _stackEnd++; | 1114 _stackEnd++; |
| 1115 } | 1115 } |
| 1116 | 1116 |
| 1117 void appendCommentToken(TokenType type, String value) { | 1117 void _appendCommentToken(TokenType type, String value) { |
| 1118 // Ignore comment tokens if client specified that it doesn't need them. | 1118 // Ignore comment tokens if client specified that it doesn't need them. |
| 1119 if (!_preserveComments) { | 1119 if (!_preserveComments) { |
| 1120 return; | 1120 return; |
| 1121 } | 1121 } |
| 1122 // OK, remember comment tokens. | 1122 // OK, remember comment tokens. |
| 1123 if (_firstComment == null) { | 1123 if (_firstComment == null) { |
| 1124 _firstComment = new StringToken(type, value, _tokenStart); | 1124 _firstComment = new StringToken(type, value, _tokenStart); |
| 1125 _lastComment = _firstComment; | 1125 _lastComment = _firstComment; |
| 1126 } else { | 1126 } else { |
| 1127 _lastComment = _lastComment.setNext(new StringToken(type, value, _tokenSta
rt)); | 1127 _lastComment = _lastComment.setNext(new StringToken(type, value, _tokenSta
rt)); |
| 1128 } | 1128 } |
| 1129 } | 1129 } |
| 1130 | 1130 |
| 1131 void appendEndToken(TokenType type, TokenType beginType) { | 1131 void _appendEndToken(TokenType type, TokenType beginType) { |
| 1132 Token token; | 1132 Token token; |
| 1133 if (_firstComment == null) { | 1133 if (_firstComment == null) { |
| 1134 token = new Token(type, _tokenStart); | 1134 token = new Token(type, _tokenStart); |
| 1135 } else { | 1135 } else { |
| 1136 token = new TokenWithComment(type, _tokenStart, _firstComment); | 1136 token = new TokenWithComment(type, _tokenStart, _firstComment); |
| 1137 _firstComment = null; | 1137 _firstComment = null; |
| 1138 _lastComment = null; | 1138 _lastComment = null; |
| 1139 } | 1139 } |
| 1140 _tail = _tail.setNext(token); | 1140 _tail = _tail.setNext(token); |
| 1141 if (_stackEnd >= 0) { | 1141 if (_stackEnd >= 0) { |
| 1142 BeginToken begin = _groupingStack[_stackEnd]; | 1142 BeginToken begin = _groupingStack[_stackEnd]; |
| 1143 if (identical(begin.type, beginType)) { | 1143 if (identical(begin.type, beginType)) { |
| 1144 begin.endToken = token; | 1144 begin.endToken = token; |
| 1145 _groupingStack.removeAt(_stackEnd--); | 1145 _groupingStack.removeAt(_stackEnd--); |
| 1146 } | 1146 } |
| 1147 } | 1147 } |
| 1148 } | 1148 } |
| 1149 | 1149 |
| 1150 void appendEofToken() { | 1150 void _appendEofToken() { |
| 1151 Token eofToken; | 1151 Token eofToken; |
| 1152 if (_firstComment == null) { | 1152 if (_firstComment == null) { |
| 1153 eofToken = new Token(TokenType.EOF, _reader.offset + 1); | 1153 eofToken = new Token(TokenType.EOF, _reader.offset + 1); |
| 1154 } else { | 1154 } else { |
| 1155 eofToken = new TokenWithComment(TokenType.EOF, _reader.offset + 1, _firstC
omment); | 1155 eofToken = new TokenWithComment(TokenType.EOF, _reader.offset + 1, _firstC
omment); |
| 1156 _firstComment = null; | 1156 _firstComment = null; |
| 1157 _lastComment = null; | 1157 _lastComment = null; |
| 1158 } | 1158 } |
| 1159 // The EOF token points to itself so that there is always infinite look-ahea
d. | 1159 // The EOF token points to itself so that there is always infinite look-ahea
d. |
| 1160 eofToken.setNext(eofToken); | 1160 eofToken.setNext(eofToken); |
| 1161 _tail = _tail.setNext(eofToken); | 1161 _tail = _tail.setNext(eofToken); |
| 1162 if (_stackEnd >= 0) { | 1162 if (_stackEnd >= 0) { |
| 1163 _hasUnmatchedGroups2 = true; | 1163 _hasUnmatchedGroups = true; |
| 1164 } | 1164 } |
| 1165 } | 1165 } |
| 1166 | 1166 |
| 1167 void appendKeywordToken(Keyword keyword) { | 1167 void _appendKeywordToken(Keyword keyword) { |
| 1168 if (_firstComment == null) { | 1168 if (_firstComment == null) { |
| 1169 _tail = _tail.setNext(new KeywordToken(keyword, _tokenStart)); | 1169 _tail = _tail.setNext(new KeywordToken(keyword, _tokenStart)); |
| 1170 } else { | 1170 } else { |
| 1171 _tail = _tail.setNext(new KeywordTokenWithComment(keyword, _tokenStart, _f
irstComment)); | 1171 _tail = _tail.setNext(new KeywordTokenWithComment(keyword, _tokenStart, _f
irstComment)); |
| 1172 _firstComment = null; | 1172 _firstComment = null; |
| 1173 _lastComment = null; | 1173 _lastComment = null; |
| 1174 } | 1174 } |
| 1175 } | 1175 } |
| 1176 | 1176 |
| 1177 void appendStringToken(TokenType type, String value) { | 1177 void _appendStringToken(TokenType type, String value) { |
| 1178 if (_firstComment == null) { | 1178 if (_firstComment == null) { |
| 1179 _tail = _tail.setNext(new StringToken(type, value, _tokenStart)); | 1179 _tail = _tail.setNext(new StringToken(type, value, _tokenStart)); |
| 1180 } else { | 1180 } else { |
| 1181 _tail = _tail.setNext(new StringTokenWithComment(type, value, _tokenStart,
_firstComment)); | 1181 _tail = _tail.setNext(new StringTokenWithComment(type, value, _tokenStart,
_firstComment)); |
| 1182 _firstComment = null; | 1182 _firstComment = null; |
| 1183 _lastComment = null; | 1183 _lastComment = null; |
| 1184 } | 1184 } |
| 1185 } | 1185 } |
| 1186 | 1186 |
| 1187 void appendStringTokenWithOffset(TokenType type, String value, int offset) { | 1187 void _appendStringTokenWithOffset(TokenType type, String value, int offset) { |
| 1188 if (_firstComment == null) { | 1188 if (_firstComment == null) { |
| 1189 _tail = _tail.setNext(new StringToken(type, value, _tokenStart + offset)); | 1189 _tail = _tail.setNext(new StringToken(type, value, _tokenStart + offset)); |
| 1190 } else { | 1190 } else { |
| 1191 _tail = _tail.setNext(new StringTokenWithComment(type, value, _tokenStart
+ offset, _firstComment)); | 1191 _tail = _tail.setNext(new StringTokenWithComment(type, value, _tokenStart
+ offset, _firstComment)); |
| 1192 _firstComment = null; | 1192 _firstComment = null; |
| 1193 _lastComment = null; | 1193 _lastComment = null; |
| 1194 } | 1194 } |
| 1195 } | 1195 } |
| 1196 | 1196 |
| 1197 void appendTokenOfType(TokenType type) { | 1197 void _appendTokenOfType(TokenType type) { |
| 1198 if (_firstComment == null) { | 1198 if (_firstComment == null) { |
| 1199 _tail = _tail.setNext(new Token(type, _tokenStart)); | 1199 _tail = _tail.setNext(new Token(type, _tokenStart)); |
| 1200 } else { | 1200 } else { |
| 1201 _tail = _tail.setNext(new TokenWithComment(type, _tokenStart, _firstCommen
t)); | 1201 _tail = _tail.setNext(new TokenWithComment(type, _tokenStart, _firstCommen
t)); |
| 1202 _firstComment = null; | 1202 _firstComment = null; |
| 1203 _lastComment = null; | 1203 _lastComment = null; |
| 1204 } | 1204 } |
| 1205 } | 1205 } |
| 1206 | 1206 |
| 1207 void appendTokenOfTypeWithOffset(TokenType type, int offset) { | 1207 void _appendTokenOfTypeWithOffset(TokenType type, int offset) { |
| 1208 if (_firstComment == null) { | 1208 if (_firstComment == null) { |
| 1209 _tail = _tail.setNext(new Token(type, offset)); | 1209 _tail = _tail.setNext(new Token(type, offset)); |
| 1210 } else { | 1210 } else { |
| 1211 _tail = _tail.setNext(new TokenWithComment(type, offset, _firstComment)); | 1211 _tail = _tail.setNext(new TokenWithComment(type, offset, _firstComment)); |
| 1212 _firstComment = null; | 1212 _firstComment = null; |
| 1213 _lastComment = null; | 1213 _lastComment = null; |
| 1214 } | 1214 } |
| 1215 } | 1215 } |
| 1216 | 1216 |
| 1217 void beginToken() { | 1217 void _beginToken() { |
| 1218 _tokenStart = _reader.offset; | 1218 _tokenStart = _reader.offset; |
| 1219 } | 1219 } |
| 1220 | 1220 |
| 1221 /** | 1221 /** |
| 1222 * Return the beginning token corresponding to a closing brace that was found
while scanning | 1222 * Return the beginning token corresponding to a closing brace that was found
while scanning |
| 1223 * inside a string interpolation expression. Tokens that cannot be matched wit
h the closing brace | 1223 * inside a string interpolation expression. Tokens that cannot be matched wit
h the closing brace |
| 1224 * will be dropped from the stack. | 1224 * will be dropped from the stack. |
| 1225 * | 1225 * |
| 1226 * @return the token to be paired with the closing brace | 1226 * @return the token to be paired with the closing brace |
| 1227 */ | 1227 */ |
| 1228 BeginToken findTokenMatchingClosingBraceInInterpolationExpression() { | 1228 BeginToken _findTokenMatchingClosingBraceInInterpolationExpression() { |
| 1229 while (_stackEnd >= 0) { | 1229 while (_stackEnd >= 0) { |
| 1230 BeginToken begin = _groupingStack[_stackEnd]; | 1230 BeginToken begin = _groupingStack[_stackEnd]; |
| 1231 if (identical(begin.type, TokenType.OPEN_CURLY_BRACKET) || identical(begin
.type, TokenType.STRING_INTERPOLATION_EXPRESSION)) { | 1231 if (identical(begin.type, TokenType.OPEN_CURLY_BRACKET) || identical(begin
.type, TokenType.STRING_INTERPOLATION_EXPRESSION)) { |
| 1232 return begin; | 1232 return begin; |
| 1233 } | 1233 } |
| 1234 _hasUnmatchedGroups2 = true; | 1234 _hasUnmatchedGroups = true; |
| 1235 _groupingStack.removeAt(_stackEnd--); | 1235 _groupingStack.removeAt(_stackEnd--); |
| 1236 } | 1236 } |
| 1237 // | 1237 // |
| 1238 // We should never get to this point because we wouldn't be inside a string
interpolation | 1238 // We should never get to this point because we wouldn't be inside a string
interpolation |
| 1239 // expression unless we had previously found the start of the expression. | 1239 // expression unless we had previously found the start of the expression. |
| 1240 // | 1240 // |
| 1241 return null; | 1241 return null; |
| 1242 } | 1242 } |
| 1243 | 1243 |
| 1244 /** | 1244 /** |
| 1245 * Report an error at the current offset. | 1245 * Report an error at the current offset. |
| 1246 * | 1246 * |
| 1247 * @param errorCode the error code indicating the nature of the error | 1247 * @param errorCode the error code indicating the nature of the error |
| 1248 * @param arguments any arguments needed to complete the error message | 1248 * @param arguments any arguments needed to complete the error message |
| 1249 */ | 1249 */ |
| 1250 void reportError(ScannerErrorCode errorCode, List<Object> arguments) { | 1250 void _reportError(ScannerErrorCode errorCode, List<Object> arguments) { |
| 1251 _errorListener.onError(new AnalysisError.con2(source, _reader.offset, 1, err
orCode, arguments)); | 1251 _errorListener.onError(new AnalysisError.con2(source, _reader.offset, 1, err
orCode, arguments)); |
| 1252 } | 1252 } |
| 1253 | 1253 |
| 1254 int select(int choice, TokenType yesType, TokenType noType) { | 1254 int _select(int choice, TokenType yesType, TokenType noType) { |
| 1255 int next = _reader.advance(); | 1255 int next = _reader.advance(); |
| 1256 if (next == choice) { | 1256 if (next == choice) { |
| 1257 appendTokenOfType(yesType); | 1257 _appendTokenOfType(yesType); |
| 1258 return _reader.advance(); | 1258 return _reader.advance(); |
| 1259 } else { | 1259 } else { |
| 1260 appendTokenOfType(noType); | 1260 _appendTokenOfType(noType); |
| 1261 return next; | 1261 return next; |
| 1262 } | 1262 } |
| 1263 } | 1263 } |
| 1264 | 1264 |
| 1265 int selectWithOffset(int choice, TokenType yesType, TokenType noType, int offs
et) { | 1265 int _selectWithOffset(int choice, TokenType yesType, TokenType noType, int off
set) { |
| 1266 int next = _reader.advance(); | 1266 int next = _reader.advance(); |
| 1267 if (next == choice) { | 1267 if (next == choice) { |
| 1268 appendTokenOfTypeWithOffset(yesType, offset); | 1268 _appendTokenOfTypeWithOffset(yesType, offset); |
| 1269 return _reader.advance(); | 1269 return _reader.advance(); |
| 1270 } else { | 1270 } else { |
| 1271 appendTokenOfTypeWithOffset(noType, offset); | 1271 _appendTokenOfTypeWithOffset(noType, offset); |
| 1272 return next; | 1272 return next; |
| 1273 } | 1273 } |
| 1274 } | 1274 } |
| 1275 | 1275 |
| 1276 int tokenizeAmpersand(int next) { | 1276 int _tokenizeAmpersand(int next) { |
| 1277 // && &= & | 1277 // && &= & |
| 1278 next = _reader.advance(); | 1278 next = _reader.advance(); |
| 1279 if (next == 0x26) { | 1279 if (next == 0x26) { |
| 1280 appendTokenOfType(TokenType.AMPERSAND_AMPERSAND); | 1280 _appendTokenOfType(TokenType.AMPERSAND_AMPERSAND); |
| 1281 return _reader.advance(); | 1281 return _reader.advance(); |
| 1282 } else if (next == 0x3D) { | 1282 } else if (next == 0x3D) { |
| 1283 appendTokenOfType(TokenType.AMPERSAND_EQ); | 1283 _appendTokenOfType(TokenType.AMPERSAND_EQ); |
| 1284 return _reader.advance(); | 1284 return _reader.advance(); |
| 1285 } else { | 1285 } else { |
| 1286 appendTokenOfType(TokenType.AMPERSAND); | 1286 _appendTokenOfType(TokenType.AMPERSAND); |
| 1287 return next; | 1287 return next; |
| 1288 } | 1288 } |
| 1289 } | 1289 } |
| 1290 | 1290 |
| 1291 int tokenizeBar(int next) { | 1291 int _tokenizeBar(int next) { |
| 1292 // | || |= | 1292 // | || |= |
| 1293 next = _reader.advance(); | 1293 next = _reader.advance(); |
| 1294 if (next == 0x7C) { | 1294 if (next == 0x7C) { |
| 1295 appendTokenOfType(TokenType.BAR_BAR); | 1295 _appendTokenOfType(TokenType.BAR_BAR); |
| 1296 return _reader.advance(); | 1296 return _reader.advance(); |
| 1297 } else if (next == 0x3D) { | 1297 } else if (next == 0x3D) { |
| 1298 appendTokenOfType(TokenType.BAR_EQ); | 1298 _appendTokenOfType(TokenType.BAR_EQ); |
| 1299 return _reader.advance(); | 1299 return _reader.advance(); |
| 1300 } else { | 1300 } else { |
| 1301 appendTokenOfType(TokenType.BAR); | 1301 _appendTokenOfType(TokenType.BAR); |
| 1302 return next; | 1302 return next; |
| 1303 } | 1303 } |
| 1304 } | 1304 } |
| 1305 | 1305 |
| 1306 int tokenizeCaret(int next) => select(0x3D, TokenType.CARET_EQ, TokenType.CARE
T); | 1306 int _tokenizeCaret(int next) => _select(0x3D, TokenType.CARET_EQ, TokenType.CA
RET); |
| 1307 | 1307 |
| 1308 int tokenizeDotOrNumber(int next) { | 1308 int _tokenizeDotOrNumber(int next) { |
| 1309 int start = _reader.offset; | 1309 int start = _reader.offset; |
| 1310 next = _reader.advance(); | 1310 next = _reader.advance(); |
| 1311 if (0x30 <= next && next <= 0x39) { | 1311 if (0x30 <= next && next <= 0x39) { |
| 1312 return tokenizeFractionPart(next, start); | 1312 return _tokenizeFractionPart(next, start); |
| 1313 } else if (0x2E == next) { | 1313 } else if (0x2E == next) { |
| 1314 return select(0x2E, TokenType.PERIOD_PERIOD_PERIOD, TokenType.PERIOD_PERIO
D); | 1314 return _select(0x2E, TokenType.PERIOD_PERIOD_PERIOD, TokenType.PERIOD_PERI
OD); |
| 1315 } else { | 1315 } else { |
| 1316 appendTokenOfType(TokenType.PERIOD); | 1316 _appendTokenOfType(TokenType.PERIOD); |
| 1317 return next; | 1317 return next; |
| 1318 } | 1318 } |
| 1319 } | 1319 } |
| 1320 | 1320 |
| 1321 int tokenizeEquals(int next) { | 1321 int _tokenizeEquals(int next) { |
| 1322 // = == => | 1322 // = == => |
| 1323 next = _reader.advance(); | 1323 next = _reader.advance(); |
| 1324 if (next == 0x3D) { | 1324 if (next == 0x3D) { |
| 1325 appendTokenOfType(TokenType.EQ_EQ); | 1325 _appendTokenOfType(TokenType.EQ_EQ); |
| 1326 return _reader.advance(); | 1326 return _reader.advance(); |
| 1327 } else if (next == 0x3E) { | 1327 } else if (next == 0x3E) { |
| 1328 appendTokenOfType(TokenType.FUNCTION); | 1328 _appendTokenOfType(TokenType.FUNCTION); |
| 1329 return _reader.advance(); | 1329 return _reader.advance(); |
| 1330 } | 1330 } |
| 1331 appendTokenOfType(TokenType.EQ); | 1331 _appendTokenOfType(TokenType.EQ); |
| 1332 return next; | 1332 return next; |
| 1333 } | 1333 } |
| 1334 | 1334 |
| 1335 int tokenizeExclamation(int next) { | 1335 int _tokenizeExclamation(int next) { |
| 1336 // ! != | 1336 // ! != |
| 1337 next = _reader.advance(); | 1337 next = _reader.advance(); |
| 1338 if (next == 0x3D) { | 1338 if (next == 0x3D) { |
| 1339 appendTokenOfType(TokenType.BANG_EQ); | 1339 _appendTokenOfType(TokenType.BANG_EQ); |
| 1340 return _reader.advance(); | 1340 return _reader.advance(); |
| 1341 } | 1341 } |
| 1342 appendTokenOfType(TokenType.BANG); | 1342 _appendTokenOfType(TokenType.BANG); |
| 1343 return next; | 1343 return next; |
| 1344 } | 1344 } |
| 1345 | 1345 |
| 1346 int tokenizeExponent(int next) { | 1346 int _tokenizeExponent(int next) { |
| 1347 if (next == 0x2B || next == 0x2D) { | 1347 if (next == 0x2B || next == 0x2D) { |
| 1348 next = _reader.advance(); | 1348 next = _reader.advance(); |
| 1349 } | 1349 } |
| 1350 bool hasDigits = false; | 1350 bool hasDigits = false; |
| 1351 while (true) { | 1351 while (true) { |
| 1352 if (0x30 <= next && next <= 0x39) { | 1352 if (0x30 <= next && next <= 0x39) { |
| 1353 hasDigits = true; | 1353 hasDigits = true; |
| 1354 } else { | 1354 } else { |
| 1355 if (!hasDigits) { | 1355 if (!hasDigits) { |
| 1356 reportError(ScannerErrorCode.MISSING_DIGIT, []); | 1356 _reportError(ScannerErrorCode.MISSING_DIGIT, []); |
| 1357 } | 1357 } |
| 1358 return next; | 1358 return next; |
| 1359 } | 1359 } |
| 1360 next = _reader.advance(); | 1360 next = _reader.advance(); |
| 1361 } | 1361 } |
| 1362 } | 1362 } |
| 1363 | 1363 |
| 1364 int tokenizeFractionPart(int next, int start) { | 1364 int _tokenizeFractionPart(int next, int start) { |
| 1365 bool done = false; | 1365 bool done = false; |
| 1366 bool hasDigit = false; | 1366 bool hasDigit = false; |
| 1367 LOOP: while (!done) { | 1367 LOOP: while (!done) { |
| 1368 if (0x30 <= next && next <= 0x39) { | 1368 if (0x30 <= next && next <= 0x39) { |
| 1369 hasDigit = true; | 1369 hasDigit = true; |
| 1370 } else if (0x65 == next || 0x45 == next) { | 1370 } else if (0x65 == next || 0x45 == next) { |
| 1371 hasDigit = true; | 1371 hasDigit = true; |
| 1372 next = tokenizeExponent(_reader.advance()); | 1372 next = _tokenizeExponent(_reader.advance()); |
| 1373 done = true; | 1373 done = true; |
| 1374 continue LOOP; | 1374 continue LOOP; |
| 1375 } else { | 1375 } else { |
| 1376 done = true; | 1376 done = true; |
| 1377 continue LOOP; | 1377 continue LOOP; |
| 1378 } | 1378 } |
| 1379 next = _reader.advance(); | 1379 next = _reader.advance(); |
| 1380 } | 1380 } |
| 1381 if (!hasDigit) { | 1381 if (!hasDigit) { |
| 1382 appendStringToken(TokenType.INT, _reader.getString(start, -2)); | 1382 _appendStringToken(TokenType.INT, _reader.getString(start, -2)); |
| 1383 if (0x2E == next) { | 1383 if (0x2E == next) { |
| 1384 return selectWithOffset(0x2E, TokenType.PERIOD_PERIOD_PERIOD, TokenType.
PERIOD_PERIOD, _reader.offset - 1); | 1384 return _selectWithOffset(0x2E, TokenType.PERIOD_PERIOD_PERIOD, TokenType
.PERIOD_PERIOD, _reader.offset - 1); |
| 1385 } | 1385 } |
| 1386 appendTokenOfTypeWithOffset(TokenType.PERIOD, _reader.offset - 1); | 1386 _appendTokenOfTypeWithOffset(TokenType.PERIOD, _reader.offset - 1); |
| 1387 return bigSwitch(next); | 1387 return bigSwitch(next); |
| 1388 } | 1388 } |
| 1389 appendStringToken(TokenType.DOUBLE, _reader.getString(start, next < 0 ? 0 :
-1)); | 1389 _appendStringToken(TokenType.DOUBLE, _reader.getString(start, next < 0 ? 0 :
-1)); |
| 1390 return next; | 1390 return next; |
| 1391 } | 1391 } |
| 1392 | 1392 |
| 1393 int tokenizeGreaterThan(int next) { | 1393 int _tokenizeGreaterThan(int next) { |
| 1394 // > >= >> >>= | 1394 // > >= >> >>= |
| 1395 next = _reader.advance(); | 1395 next = _reader.advance(); |
| 1396 if (0x3D == next) { | 1396 if (0x3D == next) { |
| 1397 appendTokenOfType(TokenType.GT_EQ); | 1397 _appendTokenOfType(TokenType.GT_EQ); |
| 1398 return _reader.advance(); | 1398 return _reader.advance(); |
| 1399 } else if (0x3E == next) { | 1399 } else if (0x3E == next) { |
| 1400 next = _reader.advance(); | 1400 next = _reader.advance(); |
| 1401 if (0x3D == next) { | 1401 if (0x3D == next) { |
| 1402 appendTokenOfType(TokenType.GT_GT_EQ); | 1402 _appendTokenOfType(TokenType.GT_GT_EQ); |
| 1403 return _reader.advance(); | 1403 return _reader.advance(); |
| 1404 } else { | 1404 } else { |
| 1405 appendTokenOfType(TokenType.GT_GT); | 1405 _appendTokenOfType(TokenType.GT_GT); |
| 1406 return next; | 1406 return next; |
| 1407 } | 1407 } |
| 1408 } else { | 1408 } else { |
| 1409 appendTokenOfType(TokenType.GT); | 1409 _appendTokenOfType(TokenType.GT); |
| 1410 return next; | 1410 return next; |
| 1411 } | 1411 } |
| 1412 } | 1412 } |
| 1413 | 1413 |
| 1414 int tokenizeHex(int next) { | 1414 int _tokenizeHex(int next) { |
| 1415 int start = _reader.offset - 1; | 1415 int start = _reader.offset - 1; |
| 1416 bool hasDigits = false; | 1416 bool hasDigits = false; |
| 1417 while (true) { | 1417 while (true) { |
| 1418 next = _reader.advance(); | 1418 next = _reader.advance(); |
| 1419 if ((0x30 <= next && next <= 0x39) || (0x41 <= next && next <= 0x46) || (0
x61 <= next && next <= 0x66)) { | 1419 if ((0x30 <= next && next <= 0x39) || (0x41 <= next && next <= 0x46) || (0
x61 <= next && next <= 0x66)) { |
| 1420 hasDigits = true; | 1420 hasDigits = true; |
| 1421 } else { | 1421 } else { |
| 1422 if (!hasDigits) { | 1422 if (!hasDigits) { |
| 1423 reportError(ScannerErrorCode.MISSING_HEX_DIGIT, []); | 1423 _reportError(ScannerErrorCode.MISSING_HEX_DIGIT, []); |
| 1424 } | 1424 } |
| 1425 appendStringToken(TokenType.HEXADECIMAL, _reader.getString(start, next <
0 ? 0 : -1)); | 1425 _appendStringToken(TokenType.HEXADECIMAL, _reader.getString(start, next
< 0 ? 0 : -1)); |
| 1426 return next; | 1426 return next; |
| 1427 } | 1427 } |
| 1428 } | 1428 } |
| 1429 } | 1429 } |
| 1430 | 1430 |
| 1431 int tokenizeHexOrNumber(int next) { | 1431 int _tokenizeHexOrNumber(int next) { |
| 1432 int x = _reader.peek(); | 1432 int x = _reader.peek(); |
| 1433 if (x == 0x78 || x == 0x58) { | 1433 if (x == 0x78 || x == 0x58) { |
| 1434 _reader.advance(); | 1434 _reader.advance(); |
| 1435 return tokenizeHex(x); | 1435 return _tokenizeHex(x); |
| 1436 } | 1436 } |
| 1437 return tokenizeNumber(next); | 1437 return _tokenizeNumber(next); |
| 1438 } | 1438 } |
| 1439 | 1439 |
| 1440 int tokenizeIdentifier(int next, int start, bool allowDollar) { | 1440 int _tokenizeIdentifier(int next, int start, bool allowDollar) { |
| 1441 while ((0x61 <= next && next <= 0x7A) || (0x41 <= next && next <= 0x5A) || (
0x30 <= next && next <= 0x39) || next == 0x5F || (next == 0x24 && allowDollar))
{ | 1441 while ((0x61 <= next && next <= 0x7A) || (0x41 <= next && next <= 0x5A) || (
0x30 <= next && next <= 0x39) || next == 0x5F || (next == 0x24 && allowDollar))
{ |
| 1442 next = _reader.advance(); | 1442 next = _reader.advance(); |
| 1443 } | 1443 } |
| 1444 appendStringToken(TokenType.IDENTIFIER, _reader.getString(start, next < 0 ?
0 : -1)); | 1444 _appendStringToken(TokenType.IDENTIFIER, _reader.getString(start, next < 0 ?
0 : -1)); |
| 1445 return next; | 1445 return next; |
| 1446 } | 1446 } |
| 1447 | 1447 |
| 1448 int tokenizeInterpolatedExpression(int next, int start) { | 1448 int _tokenizeInterpolatedExpression(int next, int start) { |
| 1449 appendBeginToken(TokenType.STRING_INTERPOLATION_EXPRESSION); | 1449 _appendBeginToken(TokenType.STRING_INTERPOLATION_EXPRESSION); |
| 1450 next = _reader.advance(); | 1450 next = _reader.advance(); |
| 1451 while (next != -1) { | 1451 while (next != -1) { |
| 1452 if (next == 0x7D) { | 1452 if (next == 0x7D) { |
| 1453 BeginToken begin = findTokenMatchingClosingBraceInInterpolationExpressio
n(); | 1453 BeginToken begin = _findTokenMatchingClosingBraceInInterpolationExpressi
on(); |
| 1454 if (begin == null) { | 1454 if (begin == null) { |
| 1455 beginToken(); | 1455 _beginToken(); |
| 1456 appendTokenOfType(TokenType.CLOSE_CURLY_BRACKET); | 1456 _appendTokenOfType(TokenType.CLOSE_CURLY_BRACKET); |
| 1457 next = _reader.advance(); | 1457 next = _reader.advance(); |
| 1458 beginToken(); | 1458 _beginToken(); |
| 1459 return next; | 1459 return next; |
| 1460 } else if (identical(begin.type, TokenType.OPEN_CURLY_BRACKET)) { | 1460 } else if (identical(begin.type, TokenType.OPEN_CURLY_BRACKET)) { |
| 1461 beginToken(); | 1461 _beginToken(); |
| 1462 appendEndToken(TokenType.CLOSE_CURLY_BRACKET, TokenType.OPEN_CURLY_BRA
CKET); | 1462 _appendEndToken(TokenType.CLOSE_CURLY_BRACKET, TokenType.OPEN_CURLY_BR
ACKET); |
| 1463 next = _reader.advance(); | 1463 next = _reader.advance(); |
| 1464 beginToken(); | 1464 _beginToken(); |
| 1465 } else if (identical(begin.type, TokenType.STRING_INTERPOLATION_EXPRESSI
ON)) { | 1465 } else if (identical(begin.type, TokenType.STRING_INTERPOLATION_EXPRESSI
ON)) { |
| 1466 beginToken(); | 1466 _beginToken(); |
| 1467 appendEndToken(TokenType.CLOSE_CURLY_BRACKET, TokenType.STRING_INTERPO
LATION_EXPRESSION); | 1467 _appendEndToken(TokenType.CLOSE_CURLY_BRACKET, TokenType.STRING_INTERP
OLATION_EXPRESSION); |
| 1468 next = _reader.advance(); | 1468 next = _reader.advance(); |
| 1469 beginToken(); | 1469 _beginToken(); |
| 1470 return next; | 1470 return next; |
| 1471 } | 1471 } |
| 1472 } else { | 1472 } else { |
| 1473 next = bigSwitch(next); | 1473 next = bigSwitch(next); |
| 1474 } | 1474 } |
| 1475 } | 1475 } |
| 1476 if (next == -1) { | 1476 if (next == -1) { |
| 1477 return next; | 1477 return next; |
| 1478 } | 1478 } |
| 1479 next = _reader.advance(); | 1479 next = _reader.advance(); |
| 1480 beginToken(); | 1480 _beginToken(); |
| 1481 return next; | 1481 return next; |
| 1482 } | 1482 } |
| 1483 | 1483 |
| 1484 int tokenizeInterpolatedIdentifier(int next, int start) { | 1484 int _tokenizeInterpolatedIdentifier(int next, int start) { |
| 1485 appendStringTokenWithOffset(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$",
0); | 1485 _appendStringTokenWithOffset(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$"
, 0); |
| 1486 if ((0x41 <= next && next <= 0x5A) || (0x61 <= next && next <= 0x7A) || next
== 0x5F) { | 1486 if ((0x41 <= next && next <= 0x5A) || (0x61 <= next && next <= 0x7A) || next
== 0x5F) { |
| 1487 beginToken(); | 1487 _beginToken(); |
| 1488 next = tokenizeKeywordOrIdentifier(next, false); | 1488 next = _tokenizeKeywordOrIdentifier(next, false); |
| 1489 } | 1489 } |
| 1490 beginToken(); | 1490 _beginToken(); |
| 1491 return next; | 1491 return next; |
| 1492 } | 1492 } |
| 1493 | 1493 |
| 1494 int tokenizeKeywordOrIdentifier(int next, bool allowDollar) { | 1494 int _tokenizeKeywordOrIdentifier(int next, bool allowDollar) { |
| 1495 KeywordState state = KeywordState.KEYWORD_STATE; | 1495 KeywordState state = KeywordState.KEYWORD_STATE; |
| 1496 int start = _reader.offset; | 1496 int start = _reader.offset; |
| 1497 while (state != null && 0x61 <= next && next <= 0x7A) { | 1497 while (state != null && 0x61 <= next && next <= 0x7A) { |
| 1498 state = state.next(next); | 1498 state = state.next(next); |
| 1499 next = _reader.advance(); | 1499 next = _reader.advance(); |
| 1500 } | 1500 } |
| 1501 if (state == null || state.keyword() == null) { | 1501 if (state == null || state.keyword() == null) { |
| 1502 return tokenizeIdentifier(next, start, allowDollar); | 1502 return _tokenizeIdentifier(next, start, allowDollar); |
| 1503 } | 1503 } |
| 1504 if ((0x41 <= next && next <= 0x5A) || (0x30 <= next && next <= 0x39) || next
== 0x5F || next == 0x24) { | 1504 if ((0x41 <= next && next <= 0x5A) || (0x30 <= next && next <= 0x39) || next
== 0x5F || next == 0x24) { |
| 1505 return tokenizeIdentifier(next, start, allowDollar); | 1505 return _tokenizeIdentifier(next, start, allowDollar); |
| 1506 } else if (next < 128) { | 1506 } else if (next < 128) { |
| 1507 appendKeywordToken(state.keyword()); | 1507 _appendKeywordToken(state.keyword()); |
| 1508 return next; | 1508 return next; |
| 1509 } else { | 1509 } else { |
| 1510 return tokenizeIdentifier(next, start, allowDollar); | 1510 return _tokenizeIdentifier(next, start, allowDollar); |
| 1511 } | 1511 } |
| 1512 } | 1512 } |
| 1513 | 1513 |
| 1514 int tokenizeLessThan(int next) { | 1514 int _tokenizeLessThan(int next) { |
| 1515 // < <= << <<= | 1515 // < <= << <<= |
| 1516 next = _reader.advance(); | 1516 next = _reader.advance(); |
| 1517 if (0x3D == next) { | 1517 if (0x3D == next) { |
| 1518 appendTokenOfType(TokenType.LT_EQ); | 1518 _appendTokenOfType(TokenType.LT_EQ); |
| 1519 return _reader.advance(); | 1519 return _reader.advance(); |
| 1520 } else if (0x3C == next) { | 1520 } else if (0x3C == next) { |
| 1521 return select(0x3D, TokenType.LT_LT_EQ, TokenType.LT_LT); | 1521 return _select(0x3D, TokenType.LT_LT_EQ, TokenType.LT_LT); |
| 1522 } else { | 1522 } else { |
| 1523 appendTokenOfType(TokenType.LT); | 1523 _appendTokenOfType(TokenType.LT); |
| 1524 return next; | 1524 return next; |
| 1525 } | 1525 } |
| 1526 } | 1526 } |
| 1527 | 1527 |
| 1528 int tokenizeMinus(int next) { | 1528 int _tokenizeMinus(int next) { |
| 1529 // - -- -= | 1529 // - -- -= |
| 1530 next = _reader.advance(); | 1530 next = _reader.advance(); |
| 1531 if (next == 0x2D) { | 1531 if (next == 0x2D) { |
| 1532 appendTokenOfType(TokenType.MINUS_MINUS); | 1532 _appendTokenOfType(TokenType.MINUS_MINUS); |
| 1533 return _reader.advance(); | 1533 return _reader.advance(); |
| 1534 } else if (next == 0x3D) { | 1534 } else if (next == 0x3D) { |
| 1535 appendTokenOfType(TokenType.MINUS_EQ); | 1535 _appendTokenOfType(TokenType.MINUS_EQ); |
| 1536 return _reader.advance(); | 1536 return _reader.advance(); |
| 1537 } else { | 1537 } else { |
| 1538 appendTokenOfType(TokenType.MINUS); | 1538 _appendTokenOfType(TokenType.MINUS); |
| 1539 return next; | 1539 return next; |
| 1540 } | 1540 } |
| 1541 } | 1541 } |
| 1542 | 1542 |
| 1543 int tokenizeMultiLineComment(int next) { | 1543 int _tokenizeMultiLineComment(int next) { |
| 1544 int nesting = 1; | 1544 int nesting = 1; |
| 1545 next = _reader.advance(); | 1545 next = _reader.advance(); |
| 1546 while (true) { | 1546 while (true) { |
| 1547 if (-1 == next) { | 1547 if (-1 == next) { |
| 1548 reportError(ScannerErrorCode.UNTERMINATED_MULTI_LINE_COMMENT, []); | 1548 _reportError(ScannerErrorCode.UNTERMINATED_MULTI_LINE_COMMENT, []); |
| 1549 appendCommentToken(TokenType.MULTI_LINE_COMMENT, _reader.getString(_toke
nStart, 0)); | 1549 _appendCommentToken(TokenType.MULTI_LINE_COMMENT, _reader.getString(_tok
enStart, 0)); |
| 1550 return next; | 1550 return next; |
| 1551 } else if (0x2A == next) { | 1551 } else if (0x2A == next) { |
| 1552 next = _reader.advance(); | 1552 next = _reader.advance(); |
| 1553 if (0x2F == next) { | 1553 if (0x2F == next) { |
| 1554 --nesting; | 1554 --nesting; |
| 1555 if (0 == nesting) { | 1555 if (0 == nesting) { |
| 1556 appendCommentToken(TokenType.MULTI_LINE_COMMENT, _reader.getString(_
tokenStart, 0)); | 1556 _appendCommentToken(TokenType.MULTI_LINE_COMMENT, _reader.getString(
_tokenStart, 0)); |
| 1557 return _reader.advance(); | 1557 return _reader.advance(); |
| 1558 } else { | 1558 } else { |
| 1559 next = _reader.advance(); | 1559 next = _reader.advance(); |
| 1560 } | 1560 } |
| 1561 } | 1561 } |
| 1562 } else if (0x2F == next) { | 1562 } else if (0x2F == next) { |
| 1563 next = _reader.advance(); | 1563 next = _reader.advance(); |
| 1564 if (0x2A == next) { | 1564 if (0x2A == next) { |
| 1565 next = _reader.advance(); | 1565 next = _reader.advance(); |
| 1566 ++nesting; | 1566 ++nesting; |
| 1567 } | 1567 } |
| 1568 } else if (next == 0xD) { | 1568 } else if (next == 0xD) { |
| 1569 next = _reader.advance(); | 1569 next = _reader.advance(); |
| 1570 if (next == 0xA) { | 1570 if (next == 0xA) { |
| 1571 next = _reader.advance(); | 1571 next = _reader.advance(); |
| 1572 } | 1572 } |
| 1573 recordStartOfLine(); | 1573 recordStartOfLine(); |
| 1574 } else if (next == 0xA) { | 1574 } else if (next == 0xA) { |
| 1575 recordStartOfLine(); | 1575 recordStartOfLine(); |
| 1576 next = _reader.advance(); | 1576 next = _reader.advance(); |
| 1577 } else { | 1577 } else { |
| 1578 next = _reader.advance(); | 1578 next = _reader.advance(); |
| 1579 } | 1579 } |
| 1580 } | 1580 } |
| 1581 } | 1581 } |
| 1582 | 1582 |
| 1583 int tokenizeMultiLineRawString(int quoteChar, int start) { | 1583 int _tokenizeMultiLineRawString(int quoteChar, int start) { |
| 1584 int next = _reader.advance(); | 1584 int next = _reader.advance(); |
| 1585 outer: while (next != -1) { | 1585 outer: while (next != -1) { |
| 1586 while (next != quoteChar) { | 1586 while (next != quoteChar) { |
| 1587 next = _reader.advance(); | 1587 next = _reader.advance(); |
| 1588 if (next == -1) { | 1588 if (next == -1) { |
| 1589 break outer; | 1589 break outer; |
| 1590 } else if (next == 0xD) { | 1590 } else if (next == 0xD) { |
| 1591 next = _reader.advance(); | 1591 next = _reader.advance(); |
| 1592 if (next == 0xA) { | 1592 if (next == 0xA) { |
| 1593 next = _reader.advance(); | 1593 next = _reader.advance(); |
| 1594 } | 1594 } |
| 1595 recordStartOfLine(); | 1595 recordStartOfLine(); |
| 1596 } else if (next == 0xA) { | 1596 } else if (next == 0xA) { |
| 1597 recordStartOfLine(); | 1597 recordStartOfLine(); |
| 1598 next = _reader.advance(); | 1598 next = _reader.advance(); |
| 1599 } | 1599 } |
| 1600 } | 1600 } |
| 1601 next = _reader.advance(); | 1601 next = _reader.advance(); |
| 1602 if (next == quoteChar) { | 1602 if (next == quoteChar) { |
| 1603 next = _reader.advance(); | 1603 next = _reader.advance(); |
| 1604 if (next == quoteChar) { | 1604 if (next == quoteChar) { |
| 1605 appendStringToken(TokenType.STRING, _reader.getString(start, 0)); | 1605 _appendStringToken(TokenType.STRING, _reader.getString(start, 0)); |
| 1606 return _reader.advance(); | 1606 return _reader.advance(); |
| 1607 } | 1607 } |
| 1608 } | 1608 } |
| 1609 } | 1609 } |
| 1610 reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, []); | 1610 _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, []); |
| 1611 appendStringToken(TokenType.STRING, _reader.getString(start, 0)); | 1611 _appendStringToken(TokenType.STRING, _reader.getString(start, 0)); |
| 1612 return _reader.advance(); | 1612 return _reader.advance(); |
| 1613 } | 1613 } |
| 1614 | 1614 |
| 1615 int tokenizeMultiLineString(int quoteChar, int start, bool raw) { | 1615 int _tokenizeMultiLineString(int quoteChar, int start, bool raw) { |
| 1616 if (raw) { | 1616 if (raw) { |
| 1617 return tokenizeMultiLineRawString(quoteChar, start); | 1617 return _tokenizeMultiLineRawString(quoteChar, start); |
| 1618 } | 1618 } |
| 1619 int next = _reader.advance(); | 1619 int next = _reader.advance(); |
| 1620 while (next != -1) { | 1620 while (next != -1) { |
| 1621 if (next == 0x24) { | 1621 if (next == 0x24) { |
| 1622 appendStringToken(TokenType.STRING, _reader.getString(start, -1)); | 1622 _appendStringToken(TokenType.STRING, _reader.getString(start, -1)); |
| 1623 beginToken(); | 1623 _beginToken(); |
| 1624 next = tokenizeStringInterpolation(start); | 1624 next = _tokenizeStringInterpolation(start); |
| 1625 start = _reader.offset; | 1625 start = _reader.offset; |
| 1626 continue; | 1626 continue; |
| 1627 } | 1627 } |
| 1628 if (next == quoteChar) { | 1628 if (next == quoteChar) { |
| 1629 next = _reader.advance(); | 1629 next = _reader.advance(); |
| 1630 if (next == quoteChar) { | 1630 if (next == quoteChar) { |
| 1631 next = _reader.advance(); | 1631 next = _reader.advance(); |
| 1632 if (next == quoteChar) { | 1632 if (next == quoteChar) { |
| 1633 appendStringToken(TokenType.STRING, _reader.getString(start, 0)); | 1633 _appendStringToken(TokenType.STRING, _reader.getString(start, 0)); |
| 1634 return _reader.advance(); | 1634 return _reader.advance(); |
| 1635 } | 1635 } |
| 1636 } | 1636 } |
| 1637 continue; | 1637 continue; |
| 1638 } | 1638 } |
| 1639 if (next == 0x5C) { | 1639 if (next == 0x5C) { |
| 1640 next = _reader.advance(); | 1640 next = _reader.advance(); |
| 1641 if (next == -1) { | 1641 if (next == -1) { |
| 1642 break; | 1642 break; |
| 1643 } | 1643 } |
| (...skipping 15 matching lines...) Expand all Loading... |
| 1659 next = _reader.advance(); | 1659 next = _reader.advance(); |
| 1660 } | 1660 } |
| 1661 recordStartOfLine(); | 1661 recordStartOfLine(); |
| 1662 } else if (next == 0xA) { | 1662 } else if (next == 0xA) { |
| 1663 recordStartOfLine(); | 1663 recordStartOfLine(); |
| 1664 next = _reader.advance(); | 1664 next = _reader.advance(); |
| 1665 } else { | 1665 } else { |
| 1666 next = _reader.advance(); | 1666 next = _reader.advance(); |
| 1667 } | 1667 } |
| 1668 } | 1668 } |
| 1669 reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, []); | 1669 _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, []); |
| 1670 appendStringToken(TokenType.STRING, _reader.getString(start, 0)); | 1670 _appendStringToken(TokenType.STRING, _reader.getString(start, 0)); |
| 1671 return _reader.advance(); | 1671 return _reader.advance(); |
| 1672 } | 1672 } |
| 1673 | 1673 |
| 1674 int tokenizeMultiply(int next) => select(0x3D, TokenType.STAR_EQ, TokenType.ST
AR); | 1674 int _tokenizeMultiply(int next) => _select(0x3D, TokenType.STAR_EQ, TokenType.
STAR); |
| 1675 | 1675 |
| 1676 int tokenizeNumber(int next) { | 1676 int _tokenizeNumber(int next) { |
| 1677 int start = _reader.offset; | 1677 int start = _reader.offset; |
| 1678 while (true) { | 1678 while (true) { |
| 1679 next = _reader.advance(); | 1679 next = _reader.advance(); |
| 1680 if (0x30 <= next && next <= 0x39) { | 1680 if (0x30 <= next && next <= 0x39) { |
| 1681 continue; | 1681 continue; |
| 1682 } else if (next == 0x2E) { | 1682 } else if (next == 0x2E) { |
| 1683 return tokenizeFractionPart(_reader.advance(), start); | 1683 return _tokenizeFractionPart(_reader.advance(), start); |
| 1684 } else if (next == 0x65 || next == 0x45) { | 1684 } else if (next == 0x65 || next == 0x45) { |
| 1685 return tokenizeFractionPart(next, start); | 1685 return _tokenizeFractionPart(next, start); |
| 1686 } else { | 1686 } else { |
| 1687 appendStringToken(TokenType.INT, _reader.getString(start, next < 0 ? 0 :
-1)); | 1687 _appendStringToken(TokenType.INT, _reader.getString(start, next < 0 ? 0
: -1)); |
| 1688 return next; | 1688 return next; |
| 1689 } | 1689 } |
| 1690 } | 1690 } |
| 1691 } | 1691 } |
| 1692 | 1692 |
| 1693 int tokenizeOpenSquareBracket(int next) { | 1693 int _tokenizeOpenSquareBracket(int next) { |
| 1694 // [ [] []= | 1694 // [ [] []= |
| 1695 next = _reader.advance(); | 1695 next = _reader.advance(); |
| 1696 if (next == 0x5D) { | 1696 if (next == 0x5D) { |
| 1697 return select(0x3D, TokenType.INDEX_EQ, TokenType.INDEX); | 1697 return _select(0x3D, TokenType.INDEX_EQ, TokenType.INDEX); |
| 1698 } else { | 1698 } else { |
| 1699 appendBeginToken(TokenType.OPEN_SQUARE_BRACKET); | 1699 _appendBeginToken(TokenType.OPEN_SQUARE_BRACKET); |
| 1700 return next; | 1700 return next; |
| 1701 } | 1701 } |
| 1702 } | 1702 } |
| 1703 | 1703 |
| 1704 int tokenizePercent(int next) => select(0x3D, TokenType.PERCENT_EQ, TokenType.
PERCENT); | 1704 int _tokenizePercent(int next) => _select(0x3D, TokenType.PERCENT_EQ, TokenTyp
e.PERCENT); |
| 1705 | 1705 |
| 1706 int tokenizePlus(int next) { | 1706 int _tokenizePlus(int next) { |
| 1707 // + ++ += | 1707 // + ++ += |
| 1708 next = _reader.advance(); | 1708 next = _reader.advance(); |
| 1709 if (0x2B == next) { | 1709 if (0x2B == next) { |
| 1710 appendTokenOfType(TokenType.PLUS_PLUS); | 1710 _appendTokenOfType(TokenType.PLUS_PLUS); |
| 1711 return _reader.advance(); | 1711 return _reader.advance(); |
| 1712 } else if (0x3D == next) { | 1712 } else if (0x3D == next) { |
| 1713 appendTokenOfType(TokenType.PLUS_EQ); | 1713 _appendTokenOfType(TokenType.PLUS_EQ); |
| 1714 return _reader.advance(); | 1714 return _reader.advance(); |
| 1715 } else { | 1715 } else { |
| 1716 appendTokenOfType(TokenType.PLUS); | 1716 _appendTokenOfType(TokenType.PLUS); |
| 1717 return next; | 1717 return next; |
| 1718 } | 1718 } |
| 1719 } | 1719 } |
| 1720 | 1720 |
| 1721 int tokenizeSingleLineComment(int next) { | 1721 int _tokenizeSingleLineComment(int next) { |
| 1722 while (true) { | 1722 while (true) { |
| 1723 next = _reader.advance(); | 1723 next = _reader.advance(); |
| 1724 if (-1 == next) { | 1724 if (-1 == next) { |
| 1725 appendCommentToken(TokenType.SINGLE_LINE_COMMENT, _reader.getString(_tok
enStart, 0)); | 1725 _appendCommentToken(TokenType.SINGLE_LINE_COMMENT, _reader.getString(_to
kenStart, 0)); |
| 1726 return next; | 1726 return next; |
| 1727 } else if (0xA == next || 0xD == next) { | 1727 } else if (0xA == next || 0xD == next) { |
| 1728 appendCommentToken(TokenType.SINGLE_LINE_COMMENT, _reader.getString(_tok
enStart, -1)); | 1728 _appendCommentToken(TokenType.SINGLE_LINE_COMMENT, _reader.getString(_to
kenStart, -1)); |
| 1729 return next; | 1729 return next; |
| 1730 } | 1730 } |
| 1731 } | 1731 } |
| 1732 } | 1732 } |
| 1733 | 1733 |
| 1734 int tokenizeSingleLineRawString(int next, int quoteChar, int start) { | 1734 int _tokenizeSingleLineRawString(int next, int quoteChar, int start) { |
| 1735 next = _reader.advance(); | 1735 next = _reader.advance(); |
| 1736 while (next != -1) { | 1736 while (next != -1) { |
| 1737 if (next == quoteChar) { | 1737 if (next == quoteChar) { |
| 1738 appendStringToken(TokenType.STRING, _reader.getString(start, 0)); | 1738 _appendStringToken(TokenType.STRING, _reader.getString(start, 0)); |
| 1739 return _reader.advance(); | 1739 return _reader.advance(); |
| 1740 } else if (next == 0xD || next == 0xA) { | 1740 } else if (next == 0xD || next == 0xA) { |
| 1741 reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, []); | 1741 _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, []); |
| 1742 appendStringToken(TokenType.STRING, _reader.getString(start, 0)); | 1742 _appendStringToken(TokenType.STRING, _reader.getString(start, 0)); |
| 1743 return _reader.advance(); | 1743 return _reader.advance(); |
| 1744 } | 1744 } |
| 1745 next = _reader.advance(); | 1745 next = _reader.advance(); |
| 1746 } | 1746 } |
| 1747 reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, []); | 1747 _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, []); |
| 1748 appendStringToken(TokenType.STRING, _reader.getString(start, 0)); | 1748 _appendStringToken(TokenType.STRING, _reader.getString(start, 0)); |
| 1749 return _reader.advance(); | 1749 return _reader.advance(); |
| 1750 } | 1750 } |
| 1751 | 1751 |
| 1752 int tokenizeSingleLineString(int next, int quoteChar, int start) { | 1752 int _tokenizeSingleLineString(int next, int quoteChar, int start) { |
| 1753 while (next != quoteChar) { | 1753 while (next != quoteChar) { |
| 1754 if (next == 0x5C) { | 1754 if (next == 0x5C) { |
| 1755 next = _reader.advance(); | 1755 next = _reader.advance(); |
| 1756 } else if (next == 0x24) { | 1756 } else if (next == 0x24) { |
| 1757 appendStringToken(TokenType.STRING, _reader.getString(start, -1)); | 1757 _appendStringToken(TokenType.STRING, _reader.getString(start, -1)); |
| 1758 beginToken(); | 1758 _beginToken(); |
| 1759 next = tokenizeStringInterpolation(start); | 1759 next = _tokenizeStringInterpolation(start); |
| 1760 start = _reader.offset; | 1760 start = _reader.offset; |
| 1761 continue; | 1761 continue; |
| 1762 } | 1762 } |
| 1763 if (next <= 0xD && (next == 0xA || next == 0xD || next == -1)) { | 1763 if (next <= 0xD && (next == 0xA || next == 0xD || next == -1)) { |
| 1764 reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, []); | 1764 _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, []); |
| 1765 appendStringToken(TokenType.STRING, _reader.getString(start, 0)); | 1765 _appendStringToken(TokenType.STRING, _reader.getString(start, 0)); |
| 1766 return _reader.advance(); | 1766 return _reader.advance(); |
| 1767 } | 1767 } |
| 1768 next = _reader.advance(); | 1768 next = _reader.advance(); |
| 1769 } | 1769 } |
| 1770 appendStringToken(TokenType.STRING, _reader.getString(start, 0)); | 1770 _appendStringToken(TokenType.STRING, _reader.getString(start, 0)); |
| 1771 return _reader.advance(); | 1771 return _reader.advance(); |
| 1772 } | 1772 } |
| 1773 | 1773 |
| 1774 int tokenizeSlashOrComment(int next) { | 1774 int _tokenizeSlashOrComment(int next) { |
| 1775 next = _reader.advance(); | 1775 next = _reader.advance(); |
| 1776 if (0x2A == next) { | 1776 if (0x2A == next) { |
| 1777 return tokenizeMultiLineComment(next); | 1777 return _tokenizeMultiLineComment(next); |
| 1778 } else if (0x2F == next) { | 1778 } else if (0x2F == next) { |
| 1779 return tokenizeSingleLineComment(next); | 1779 return _tokenizeSingleLineComment(next); |
| 1780 } else if (0x3D == next) { | 1780 } else if (0x3D == next) { |
| 1781 appendTokenOfType(TokenType.SLASH_EQ); | 1781 _appendTokenOfType(TokenType.SLASH_EQ); |
| 1782 return _reader.advance(); | 1782 return _reader.advance(); |
| 1783 } else { | 1783 } else { |
| 1784 appendTokenOfType(TokenType.SLASH); | 1784 _appendTokenOfType(TokenType.SLASH); |
| 1785 return next; | 1785 return next; |
| 1786 } | 1786 } |
| 1787 } | 1787 } |
| 1788 | 1788 |
| 1789 int tokenizeString(int next, int start, bool raw) { | 1789 int _tokenizeString(int next, int start, bool raw) { |
| 1790 int quoteChar = next; | 1790 int quoteChar = next; |
| 1791 next = _reader.advance(); | 1791 next = _reader.advance(); |
| 1792 if (quoteChar == next) { | 1792 if (quoteChar == next) { |
| 1793 next = _reader.advance(); | 1793 next = _reader.advance(); |
| 1794 if (quoteChar == next) { | 1794 if (quoteChar == next) { |
| 1795 // Multiline string. | 1795 // Multiline string. |
| 1796 return tokenizeMultiLineString(quoteChar, start, raw); | 1796 return _tokenizeMultiLineString(quoteChar, start, raw); |
| 1797 } else { | 1797 } else { |
| 1798 // Empty string. | 1798 // Empty string. |
| 1799 appendStringToken(TokenType.STRING, _reader.getString(start, -1)); | 1799 _appendStringToken(TokenType.STRING, _reader.getString(start, -1)); |
| 1800 return next; | 1800 return next; |
| 1801 } | 1801 } |
| 1802 } | 1802 } |
| 1803 if (raw) { | 1803 if (raw) { |
| 1804 return tokenizeSingleLineRawString(next, quoteChar, start); | 1804 return _tokenizeSingleLineRawString(next, quoteChar, start); |
| 1805 } else { | 1805 } else { |
| 1806 return tokenizeSingleLineString(next, quoteChar, start); | 1806 return _tokenizeSingleLineString(next, quoteChar, start); |
| 1807 } | 1807 } |
| 1808 } | 1808 } |
| 1809 | 1809 |
| 1810 int tokenizeStringInterpolation(int start) { | 1810 int _tokenizeStringInterpolation(int start) { |
| 1811 beginToken(); | 1811 _beginToken(); |
| 1812 int next = _reader.advance(); | 1812 int next = _reader.advance(); |
| 1813 if (next == 0x7B) { | 1813 if (next == 0x7B) { |
| 1814 return tokenizeInterpolatedExpression(next, start); | 1814 return _tokenizeInterpolatedExpression(next, start); |
| 1815 } else { | 1815 } else { |
| 1816 return tokenizeInterpolatedIdentifier(next, start); | 1816 return _tokenizeInterpolatedIdentifier(next, start); |
| 1817 } | 1817 } |
| 1818 } | 1818 } |
| 1819 | 1819 |
| 1820 int tokenizeTag(int next) { | 1820 int _tokenizeTag(int next) { |
| 1821 // # or #!.*[\n\r] | 1821 // # or #!.*[\n\r] |
| 1822 if (_reader.offset == 0) { | 1822 if (_reader.offset == 0) { |
| 1823 if (_reader.peek() == 0x21) { | 1823 if (_reader.peek() == 0x21) { |
| 1824 do { | 1824 do { |
| 1825 next = _reader.advance(); | 1825 next = _reader.advance(); |
| 1826 } while (next != 0xA && next != 0xD && next > 0); | 1826 } while (next != 0xA && next != 0xD && next > 0); |
| 1827 appendStringToken(TokenType.SCRIPT_TAG, _reader.getString(_tokenStart, 0
)); | 1827 _appendStringToken(TokenType.SCRIPT_TAG, _reader.getString(_tokenStart,
0)); |
| 1828 return next; | 1828 return next; |
| 1829 } | 1829 } |
| 1830 } | 1830 } |
| 1831 appendTokenOfType(TokenType.HASH); | 1831 _appendTokenOfType(TokenType.HASH); |
| 1832 return _reader.advance(); | 1832 return _reader.advance(); |
| 1833 } | 1833 } |
| 1834 | 1834 |
| 1835 int tokenizeTilde(int next) { | 1835 int _tokenizeTilde(int next) { |
| 1836 // ~ ~/ ~/= | 1836 // ~ ~/ ~/= |
| 1837 next = _reader.advance(); | 1837 next = _reader.advance(); |
| 1838 if (next == 0x2F) { | 1838 if (next == 0x2F) { |
| 1839 return select(0x3D, TokenType.TILDE_SLASH_EQ, TokenType.TILDE_SLASH); | 1839 return _select(0x3D, TokenType.TILDE_SLASH_EQ, TokenType.TILDE_SLASH); |
| 1840 } else { | 1840 } else { |
| 1841 appendTokenOfType(TokenType.TILDE); | 1841 _appendTokenOfType(TokenType.TILDE); |
| 1842 return next; | 1842 return next; |
| 1843 } | 1843 } |
| 1844 } | 1844 } |
| 1845 } | 1845 } |
| 1846 | 1846 |
| 1847 /** | 1847 /** |
| 1848 * Instances of the class `StringToken` represent a token whose value is indepen
dent of it's | 1848 * Instances of the class `StringToken` represent a token whose value is indepen
dent of it's |
| 1849 * type. | 1849 * type. |
| 1850 */ | 1850 */ |
| 1851 class StringToken extends Token { | 1851 class StringToken extends Token { |
| 1852 /** | 1852 /** |
| 1853 * The lexeme represented by this token. | 1853 * The lexeme represented by this token. |
| 1854 */ | 1854 */ |
| 1855 String _value2; | 1855 String _value; |
| 1856 | 1856 |
| 1857 /** | 1857 /** |
| 1858 * Initialize a newly created token to represent a token of the given type wit
h the given value. | 1858 * Initialize a newly created token to represent a token of the given type wit
h the given value. |
| 1859 * | 1859 * |
| 1860 * @param type the type of the token | 1860 * @param type the type of the token |
| 1861 * @param value the lexeme represented by this token | 1861 * @param value the lexeme represented by this token |
| 1862 * @param offset the offset from the beginning of the file to the first charac
ter in the token | 1862 * @param offset the offset from the beginning of the file to the first charac
ter in the token |
| 1863 */ | 1863 */ |
| 1864 StringToken(TokenType type, String value, int offset) : super(type, offset) { | 1864 StringToken(TokenType type, String value, int offset) : super(type, offset) { |
| 1865 this._value2 = StringUtilities.intern(value); | 1865 this._value = StringUtilities.intern(value); |
| 1866 } | 1866 } |
| 1867 | 1867 |
| 1868 Token copy() => new StringToken(type, _value2, offset); | 1868 Token copy() => new StringToken(type, _value, offset); |
| 1869 | 1869 |
| 1870 String get lexeme => _value2; | 1870 String get lexeme => _value; |
| 1871 | 1871 |
| 1872 String value() => _value2; | 1872 String value() => _value; |
| 1873 } | 1873 } |
| 1874 | 1874 |
| 1875 /** | 1875 /** |
| 1876 * Instances of the class `TokenWithComment` represent a normal token that is pr
eceded by | 1876 * Instances of the class `TokenWithComment` represent a normal token that is pr
eceded by |
| 1877 * comments. | 1877 * comments. |
| 1878 */ | 1878 */ |
| 1879 class TokenWithComment extends Token { | 1879 class TokenWithComment extends Token { |
| 1880 /** | 1880 /** |
| 1881 * The first comment in the list of comments that precede this token. | 1881 * The first comment in the list of comments that precede this token. |
| 1882 */ | 1882 */ |
| (...skipping 114 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1997 bool get isSynthetic => length == 0; | 1997 bool get isSynthetic => length == 0; |
| 1998 | 1998 |
| 1999 /** | 1999 /** |
| 2000 * Return `true` if this token represents an operator that can be defined by u
sers. | 2000 * Return `true` if this token represents an operator that can be defined by u
sers. |
| 2001 * | 2001 * |
| 2002 * @return `true` if this token represents an operator that can be defined by
users | 2002 * @return `true` if this token represents an operator that can be defined by
users |
| 2003 */ | 2003 */ |
| 2004 bool get isUserDefinableOperator => type.isUserDefinableOperator; | 2004 bool get isUserDefinableOperator => type.isUserDefinableOperator; |
| 2005 | 2005 |
| 2006 /** | 2006 /** |
| 2007 * Return `true` if this token has any one of the given types. |
| 2008 * |
| 2009 * @param types the types of token that are being tested for |
| 2010 * @return `true` if this token has any of the given types |
| 2011 */ |
| 2012 bool matchesAny(List<TokenType> types) { |
| 2013 for (TokenType type in types) { |
| 2014 if (identical(this.type, type)) { |
| 2015 return true; |
| 2016 } |
| 2017 } |
| 2018 return false; |
| 2019 } |
| 2020 |
| 2021 /** |
| 2007 * Set the next token in the token stream to the given token. This has the sid
e-effect of setting | 2022 * Set the next token in the token stream to the given token. This has the sid
e-effect of setting |
| 2008 * this token to be the previous token for the given token. | 2023 * this token to be the previous token for the given token. |
| 2009 * | 2024 * |
| 2010 * @param token the next token in the token stream | 2025 * @param token the next token in the token stream |
| 2011 * @return the token that was passed in | 2026 * @return the token that was passed in |
| 2012 */ | 2027 */ |
| 2013 Token setNext(Token token) { | 2028 Token setNext(Token token) { |
| 2014 _next = token; | 2029 _next = token; |
| 2015 token.previous = this; | 2030 token.previous = this; |
| 2016 return token; | 2031 return token; |
| (...skipping 678 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2695 * @return `true` if this token type represents an operator that can be define
d by users | 2710 * @return `true` if this token type represents an operator that can be define
d by users |
| 2696 */ | 2711 */ |
| 2697 bool get isUserDefinableOperator => identical(_lexeme, "==") || identical(_lex
eme, "~") || identical(_lexeme, "[]") || identical(_lexeme, "[]=") || identical(
_lexeme, "*") || identical(_lexeme, "/") || identical(_lexeme, "%") || identical
(_lexeme, "~/") || identical(_lexeme, "+") || identical(_lexeme, "-") || identic
al(_lexeme, "<<") || identical(_lexeme, ">>") || identical(_lexeme, ">=") || ide
ntical(_lexeme, ">") || identical(_lexeme, "<=") || identical(_lexeme, "<") || i
dentical(_lexeme, "&") || identical(_lexeme, "^") || identical(_lexeme, "|"); | 2712 bool get isUserDefinableOperator => identical(_lexeme, "==") || identical(_lex
eme, "~") || identical(_lexeme, "[]") || identical(_lexeme, "[]=") || identical(
_lexeme, "*") || identical(_lexeme, "/") || identical(_lexeme, "%") || identical
(_lexeme, "~/") || identical(_lexeme, "+") || identical(_lexeme, "-") || identic
al(_lexeme, "<<") || identical(_lexeme, ">>") || identical(_lexeme, ">=") || ide
ntical(_lexeme, ">") || identical(_lexeme, "<=") || identical(_lexeme, "<") || i
dentical(_lexeme, "&") || identical(_lexeme, "^") || identical(_lexeme, "|"); |
| 2698 } | 2713 } |
| 2699 | 2714 |
| 2700 class TokenType_EOF extends TokenType { | 2715 class TokenType_EOF extends TokenType { |
| 2701 TokenType_EOF(String name, int ordinal, TokenClass arg0, String arg1) : super.
con2(name, ordinal, arg0, arg1); | 2716 TokenType_EOF(String name, int ordinal, TokenClass arg0, String arg1) : super.
con2(name, ordinal, arg0, arg1); |
| 2702 | 2717 |
| 2703 String toString() => "-eof-"; | 2718 String toString() => "-eof-"; |
| 2704 } | 2719 } |
| OLD | NEW |