| OLD | NEW |
| 1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 import 'package:charcode/ascii.dart'; | 5 import 'package:charcode/ascii.dart'; |
| 6 import 'package:front_end/src/scanner/errors.dart'; | 6 import 'package:front_end/src/scanner/errors.dart'; |
| 7 import 'package:front_end/src/scanner/reader.dart'; | 7 import 'package:front_end/src/scanner/reader.dart'; |
| 8 import 'package:front_end/src/scanner/string_utilities.dart'; | 8 import 'package:front_end/src/scanner/string_utilities.dart'; |
| 9 import 'package:front_end/src/scanner/token.dart'; | 9 import 'package:front_end/src/scanner/token.dart'; |
| 10 | 10 |
| 11 /** | 11 /** |
| 12 * A state in a state machine used to scan keywords. | 12 * A state in a state machine used to scan keywords. |
| 13 */ | 13 */ |
| 14 class KeywordState { | 14 class KeywordState { |
| 15 /** | 15 /** |
| 16 * An empty transition table used by leaf states. | 16 * An empty transition table used by leaf states. |
| 17 */ | 17 */ |
| 18 static List<KeywordState> _EMPTY_TABLE = new List<KeywordState>(26); | 18 static List<KeywordState> _EMPTY_TABLE = new List<KeywordState>($z - $A + 1); |
| 19 | 19 |
| 20 /** | 20 /** |
| 21 * The initial state in the state machine. | 21 * The initial state in the state machine. |
| 22 */ | 22 */ |
| 23 static final KeywordState KEYWORD_STATE = _createKeywordStateTable(); | 23 static final KeywordState KEYWORD_STATE = _createKeywordStateTable(); |
| 24 | 24 |
| 25 /** | 25 /** |
| 26 * A table mapping characters to the states to which those characters will | 26 * A table mapping characters to the states to which those characters will |
| 27 * transition. (The index into the array is the offset from the character | 27 * transition. (The index into the array is the offset from the character |
| 28 * `'a'` to the transitioning character.) | 28 * `'a'` to the transitioning character.) |
| (...skipping 18 matching lines...) Expand all Loading... |
| 47 * Return the keyword that was recognized by this state, or `null` if this | 47 * Return the keyword that was recognized by this state, or `null` if this |
| 48 * state does not recognized a keyword. | 48 * state does not recognized a keyword. |
| 49 */ | 49 */ |
| 50 Keyword keyword() => _keyword; | 50 Keyword keyword() => _keyword; |
| 51 | 51 |
| 52 /** | 52 /** |
| 53 * Return the state that follows this state on a transition of the given | 53 * Return the state that follows this state on a transition of the given |
| 54 * [character], or `null` if there is no valid state reachable from this state | 54 * [character], or `null` if there is no valid state reachable from this state |
| 55 * with such a transition. | 55 * with such a transition. |
| 56 */ | 56 */ |
| 57 KeywordState next(int character) => _table[character - $a]; | 57 KeywordState next(int character) => _table[character - $A]; |
| 58 | 58 |
| 59 /** | 59 /** |
| 60 * Create the next state in the state machine where we have already recognized | 60 * Create the next state in the state machine where we have already recognized |
| 61 * the subset of strings in the given array of [strings] starting at the given | 61 * the subset of strings in the given array of [strings] starting at the given |
| 62 * [offset] and having the given [length]. All of these strings have a common | 62 * [offset] and having the given [length]. All of these strings have a common |
| 63 * prefix and the next character is at the given [start] index. | 63 * prefix and the next character is at the given [start] index. |
| 64 */ | 64 */ |
| 65 static KeywordState _computeKeywordStateTable( | 65 static KeywordState _computeKeywordStateTable( |
| 66 int start, List<String> strings, int offset, int length) { | 66 int start, List<String> strings, int offset, int length) { |
| 67 List<KeywordState> result = new List<KeywordState>(26); | 67 List<KeywordState> result = new List<KeywordState>($z - $A + 1); |
| 68 assert(length != 0); | 68 assert(length != 0); |
| 69 int chunk = $nul; | 69 int chunk = $nul; |
| 70 int chunkStart = -1; | 70 int chunkStart = -1; |
| 71 bool isLeaf = false; | 71 bool isLeaf = false; |
| 72 for (int i = offset; i < offset + length; i++) { | 72 for (int i = offset; i < offset + length; i++) { |
| 73 if (strings[i].length == start) { | 73 if (strings[i].length == start) { |
| 74 isLeaf = true; | 74 isLeaf = true; |
| 75 } | 75 } |
| 76 if (strings[i].length > start) { | 76 if (strings[i].length > start) { |
| 77 int c = strings[i].codeUnitAt(start); | 77 int c = strings[i].codeUnitAt(start); |
| 78 if (chunk != c) { | 78 if (chunk != c) { |
| 79 if (chunkStart != -1) { | 79 if (chunkStart != -1) { |
| 80 result[chunk - $a] = _computeKeywordStateTable( | 80 result[chunk - $A] = _computeKeywordStateTable( |
| 81 start + 1, strings, chunkStart, i - chunkStart); | 81 start + 1, strings, chunkStart, i - chunkStart); |
| 82 } | 82 } |
| 83 chunkStart = i; | 83 chunkStart = i; |
| 84 chunk = c; | 84 chunk = c; |
| 85 } | 85 } |
| 86 } | 86 } |
| 87 } | 87 } |
| 88 if (chunkStart != -1) { | 88 if (chunkStart != -1) { |
| 89 assert(result[chunk - $a] == null); | 89 assert(result[chunk - $A] == null); |
| 90 result[chunk - $a] = _computeKeywordStateTable( | 90 result[chunk - $A] = _computeKeywordStateTable( |
| 91 start + 1, strings, chunkStart, offset + length - chunkStart); | 91 start + 1, strings, chunkStart, offset + length - chunkStart); |
| 92 } else { | 92 } else { |
| 93 assert(length == 1); | 93 assert(length == 1); |
| 94 return new KeywordState(_EMPTY_TABLE, strings[offset]); | 94 return new KeywordState(_EMPTY_TABLE, strings[offset]); |
| 95 } | 95 } |
| 96 if (isLeaf) { | 96 if (isLeaf) { |
| 97 return new KeywordState(result, strings[offset]); | 97 return new KeywordState(result, strings[offset]); |
| 98 } else { | 98 } else { |
| 99 return new KeywordState(result, null); | 99 return new KeywordState(result, null); |
| 100 } | 100 } |
| (...skipping 175 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 276 } | 276 } |
| 277 if (next == $r) { | 277 if (next == $r) { |
| 278 // 'r' | 278 // 'r' |
| 279 int peek = _reader.peek(); | 279 int peek = _reader.peek(); |
| 280 if (peek == $double_quote || peek == $single_quote) { | 280 if (peek == $double_quote || peek == $single_quote) { |
| 281 // '"' || "'" | 281 // '"' || "'" |
| 282 int start = _reader.offset; | 282 int start = _reader.offset; |
| 283 return _tokenizeString(_reader.advance(), start, true); | 283 return _tokenizeString(_reader.advance(), start, true); |
| 284 } | 284 } |
| 285 } | 285 } |
| 286 if ($a <= next && next <= $z) { | 286 if (($A <= next && next <= $Z) || ($a <= next && next <= $z)) { |
| 287 // 'a'-'z' | 287 // 'A'-'Z' || 'a'-'z' |
| 288 return _tokenizeKeywordOrIdentifier(next, true); | 288 return _tokenizeKeywordOrIdentifier(next, true); |
| 289 } | 289 } |
| 290 if (($A <= next && next <= $Z) || next == $_ || next == $$) { | 290 if (next == $_ || next == $$) { |
| 291 // 'A'-'Z' || '_' || '$' | 291 // '_' || '$' |
| 292 return _tokenizeIdentifier(next, _reader.offset, true); | 292 return _tokenizeIdentifier(next, _reader.offset, true); |
| 293 } | 293 } |
| 294 if (next == $lt) { | 294 if (next == $lt) { |
| 295 // '<' | 295 // '<' |
| 296 return _tokenizeLessThan(next); | 296 return _tokenizeLessThan(next); |
| 297 } | 297 } |
| 298 if (next == $gt) { | 298 if (next == $gt) { |
| 299 // '>' | 299 // '>' |
| 300 return _tokenizeGreaterThan(next); | 300 return _tokenizeGreaterThan(next); |
| 301 } | 301 } |
| (...skipping 632 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 934 _beginToken(); | 934 _beginToken(); |
| 935 next = _tokenizeKeywordOrIdentifier(next, false); | 935 next = _tokenizeKeywordOrIdentifier(next, false); |
| 936 } | 936 } |
| 937 _beginToken(); | 937 _beginToken(); |
| 938 return next; | 938 return next; |
| 939 } | 939 } |
| 940 | 940 |
| 941 int _tokenizeKeywordOrIdentifier(int next, bool allowDollar) { | 941 int _tokenizeKeywordOrIdentifier(int next, bool allowDollar) { |
| 942 KeywordState state = KeywordState.KEYWORD_STATE; | 942 KeywordState state = KeywordState.KEYWORD_STATE; |
| 943 int start = _reader.offset; | 943 int start = _reader.offset; |
| 944 while (state != null && $a <= next && next <= $z) { | 944 while (state != null && |
| 945 (($A <= next && next <= $Z) || $a <= next && next <= $z)) { |
| 945 state = state.next(next); | 946 state = state.next(next); |
| 946 next = _reader.advance(); | 947 next = _reader.advance(); |
| 947 } | 948 } |
| 948 if (state == null || state.keyword() == null) { | 949 if (state == null || state.keyword() == null) { |
| 949 return _tokenizeIdentifier(next, start, allowDollar); | 950 return _tokenizeIdentifier(next, start, allowDollar); |
| 950 } | 951 } |
| 951 if (($A <= next && next <= $Z) || | 952 if (($0 <= next && next <= $9) || next == $_ || next == $$) { |
| 952 ($0 <= next && next <= $9) || | |
| 953 next == $_ || | |
| 954 next == $$) { | |
| 955 return _tokenizeIdentifier(next, start, allowDollar); | 953 return _tokenizeIdentifier(next, start, allowDollar); |
| 956 } else if (next < 128) { | 954 } else if (next < 128) { |
| 957 _appendKeywordToken(state.keyword()); | 955 _appendKeywordToken(state.keyword()); |
| 958 return next; | 956 return next; |
| 959 } else { | 957 } else { |
| 960 return _tokenizeIdentifier(next, start, allowDollar); | 958 return _tokenizeIdentifier(next, start, allowDollar); |
| 961 } | 959 } |
| 962 } | 960 } |
| 963 | 961 |
| 964 int _tokenizeLessThan(int next) { | 962 int _tokenizeLessThan(int next) { |
| (...skipping 373 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1338 } | 1336 } |
| 1339 | 1337 |
| 1340 /** | 1338 /** |
| 1341 * Checks if [value] is a single-line or multi-line comment. | 1339 * Checks if [value] is a single-line or multi-line comment. |
| 1342 */ | 1340 */ |
| 1343 static bool _isDocumentationComment(String value) { | 1341 static bool _isDocumentationComment(String value) { |
| 1344 return StringUtilities.startsWith3(value, 0, $slash, $slash, $slash) || | 1342 return StringUtilities.startsWith3(value, 0, $slash, $slash, $slash) || |
| 1345 StringUtilities.startsWith3(value, 0, $slash, $asterisk, $asterisk); | 1343 StringUtilities.startsWith3(value, 0, $slash, $asterisk, $asterisk); |
| 1346 } | 1344 } |
| 1347 } | 1345 } |
| OLD | NEW |