OLD | NEW |
1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 import 'package:charcode/ascii.dart'; | 5 import 'package:charcode/ascii.dart'; |
6 import 'package:front_end/src/scanner/errors.dart'; | 6 import 'package:front_end/src/scanner/errors.dart'; |
7 import 'package:front_end/src/scanner/reader.dart'; | 7 import 'package:front_end/src/scanner/reader.dart'; |
8 import 'package:front_end/src/scanner/string_utilities.dart'; | 8 import 'package:front_end/src/scanner/string_utilities.dart'; |
9 import 'package:front_end/src/scanner/token.dart'; | 9 import 'package:front_end/src/scanner/token.dart'; |
10 | 10 |
11 /** | 11 /** |
12 * A state in a state machine used to scan keywords. | 12 * A state in a state machine used to scan keywords. |
13 */ | 13 */ |
14 class KeywordState { | 14 class KeywordState { |
15 /** | 15 /** |
16 * An empty transition table used by leaf states. | 16 * An empty transition table used by leaf states. |
17 */ | 17 */ |
18 static List<KeywordState> _EMPTY_TABLE = new List<KeywordState>(26); | 18 static List<KeywordState> _EMPTY_TABLE = new List<KeywordState>($z - $A + 1); |
19 | 19 |
20 /** | 20 /** |
21 * The initial state in the state machine. | 21 * The initial state in the state machine. |
22 */ | 22 */ |
23 static final KeywordState KEYWORD_STATE = _createKeywordStateTable(); | 23 static final KeywordState KEYWORD_STATE = _createKeywordStateTable(); |
24 | 24 |
25 /** | 25 /** |
26 * A table mapping characters to the states to which those characters will | 26 * A table mapping characters to the states to which those characters will |
27 * transition. (The index into the array is the offset from the character | 27 * transition. (The index into the array is the offset from the character |
28 * `'a'` to the transitioning character.) | 28 * `'a'` to the transitioning character.) |
(...skipping 18 matching lines...) Expand all Loading... |
47 * Return the keyword that was recognized by this state, or `null` if this | 47 * Return the keyword that was recognized by this state, or `null` if this |
48 * state does not recognized a keyword. | 48 * state does not recognized a keyword. |
49 */ | 49 */ |
50 Keyword keyword() => _keyword; | 50 Keyword keyword() => _keyword; |
51 | 51 |
52 /** | 52 /** |
53 * Return the state that follows this state on a transition of the given | 53 * Return the state that follows this state on a transition of the given |
54 * [character], or `null` if there is no valid state reachable from this state | 54 * [character], or `null` if there is no valid state reachable from this state |
55 * with such a transition. | 55 * with such a transition. |
56 */ | 56 */ |
57 KeywordState next(int character) => _table[character - $a]; | 57 KeywordState next(int character) => _table[character - $A]; |
58 | 58 |
59 /** | 59 /** |
60 * Create the next state in the state machine where we have already recognized | 60 * Create the next state in the state machine where we have already recognized |
61 * the subset of strings in the given array of [strings] starting at the given | 61 * the subset of strings in the given array of [strings] starting at the given |
62 * [offset] and having the given [length]. All of these strings have a common | 62 * [offset] and having the given [length]. All of these strings have a common |
63 * prefix and the next character is at the given [start] index. | 63 * prefix and the next character is at the given [start] index. |
64 */ | 64 */ |
65 static KeywordState _computeKeywordStateTable( | 65 static KeywordState _computeKeywordStateTable( |
66 int start, List<String> strings, int offset, int length) { | 66 int start, List<String> strings, int offset, int length) { |
67 List<KeywordState> result = new List<KeywordState>(26); | 67 List<KeywordState> result = new List<KeywordState>($z - $A + 1); |
68 assert(length != 0); | 68 assert(length != 0); |
69 int chunk = $nul; | 69 int chunk = $nul; |
70 int chunkStart = -1; | 70 int chunkStart = -1; |
71 bool isLeaf = false; | 71 bool isLeaf = false; |
72 for (int i = offset; i < offset + length; i++) { | 72 for (int i = offset; i < offset + length; i++) { |
73 if (strings[i].length == start) { | 73 if (strings[i].length == start) { |
74 isLeaf = true; | 74 isLeaf = true; |
75 } | 75 } |
76 if (strings[i].length > start) { | 76 if (strings[i].length > start) { |
77 int c = strings[i].codeUnitAt(start); | 77 int c = strings[i].codeUnitAt(start); |
78 if (chunk != c) { | 78 if (chunk != c) { |
79 if (chunkStart != -1) { | 79 if (chunkStart != -1) { |
80 result[chunk - $a] = _computeKeywordStateTable( | 80 result[chunk - $A] = _computeKeywordStateTable( |
81 start + 1, strings, chunkStart, i - chunkStart); | 81 start + 1, strings, chunkStart, i - chunkStart); |
82 } | 82 } |
83 chunkStart = i; | 83 chunkStart = i; |
84 chunk = c; | 84 chunk = c; |
85 } | 85 } |
86 } | 86 } |
87 } | 87 } |
88 if (chunkStart != -1) { | 88 if (chunkStart != -1) { |
89 assert(result[chunk - $a] == null); | 89 assert(result[chunk - $A] == null); |
90 result[chunk - $a] = _computeKeywordStateTable( | 90 result[chunk - $A] = _computeKeywordStateTable( |
91 start + 1, strings, chunkStart, offset + length - chunkStart); | 91 start + 1, strings, chunkStart, offset + length - chunkStart); |
92 } else { | 92 } else { |
93 assert(length == 1); | 93 assert(length == 1); |
94 return new KeywordState(_EMPTY_TABLE, strings[offset]); | 94 return new KeywordState(_EMPTY_TABLE, strings[offset]); |
95 } | 95 } |
96 if (isLeaf) { | 96 if (isLeaf) { |
97 return new KeywordState(result, strings[offset]); | 97 return new KeywordState(result, strings[offset]); |
98 } else { | 98 } else { |
99 return new KeywordState(result, null); | 99 return new KeywordState(result, null); |
100 } | 100 } |
(...skipping 175 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
276 } | 276 } |
277 if (next == $r) { | 277 if (next == $r) { |
278 // 'r' | 278 // 'r' |
279 int peek = _reader.peek(); | 279 int peek = _reader.peek(); |
280 if (peek == $double_quote || peek == $single_quote) { | 280 if (peek == $double_quote || peek == $single_quote) { |
281 // '"' || "'" | 281 // '"' || "'" |
282 int start = _reader.offset; | 282 int start = _reader.offset; |
283 return _tokenizeString(_reader.advance(), start, true); | 283 return _tokenizeString(_reader.advance(), start, true); |
284 } | 284 } |
285 } | 285 } |
286 if ($a <= next && next <= $z) { | 286 if (($A <= next && next <= $Z) || ($a <= next && next <= $z)) { |
287 // 'a'-'z' | 287 // 'A'-'Z' || 'a'-'z' |
288 return _tokenizeKeywordOrIdentifier(next, true); | 288 return _tokenizeKeywordOrIdentifier(next, true); |
289 } | 289 } |
290 if (($A <= next && next <= $Z) || next == $_ || next == $$) { | 290 if (next == $_ || next == $$) { |
291 // 'A'-'Z' || '_' || '$' | 291 // '_' || '$' |
292 return _tokenizeIdentifier(next, _reader.offset, true); | 292 return _tokenizeIdentifier(next, _reader.offset, true); |
293 } | 293 } |
294 if (next == $lt) { | 294 if (next == $lt) { |
295 // '<' | 295 // '<' |
296 return _tokenizeLessThan(next); | 296 return _tokenizeLessThan(next); |
297 } | 297 } |
298 if (next == $gt) { | 298 if (next == $gt) { |
299 // '>' | 299 // '>' |
300 return _tokenizeGreaterThan(next); | 300 return _tokenizeGreaterThan(next); |
301 } | 301 } |
(...skipping 632 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
934 _beginToken(); | 934 _beginToken(); |
935 next = _tokenizeKeywordOrIdentifier(next, false); | 935 next = _tokenizeKeywordOrIdentifier(next, false); |
936 } | 936 } |
937 _beginToken(); | 937 _beginToken(); |
938 return next; | 938 return next; |
939 } | 939 } |
940 | 940 |
941 int _tokenizeKeywordOrIdentifier(int next, bool allowDollar) { | 941 int _tokenizeKeywordOrIdentifier(int next, bool allowDollar) { |
942 KeywordState state = KeywordState.KEYWORD_STATE; | 942 KeywordState state = KeywordState.KEYWORD_STATE; |
943 int start = _reader.offset; | 943 int start = _reader.offset; |
944 while (state != null && $a <= next && next <= $z) { | 944 while (state != null && |
| 945 (($A <= next && next <= $Z) || $a <= next && next <= $z)) { |
945 state = state.next(next); | 946 state = state.next(next); |
946 next = _reader.advance(); | 947 next = _reader.advance(); |
947 } | 948 } |
948 if (state == null || state.keyword() == null) { | 949 if (state == null || state.keyword() == null) { |
949 return _tokenizeIdentifier(next, start, allowDollar); | 950 return _tokenizeIdentifier(next, start, allowDollar); |
950 } | 951 } |
951 if (($A <= next && next <= $Z) || | 952 if (($0 <= next && next <= $9) || next == $_ || next == $$) { |
952 ($0 <= next && next <= $9) || | |
953 next == $_ || | |
954 next == $$) { | |
955 return _tokenizeIdentifier(next, start, allowDollar); | 953 return _tokenizeIdentifier(next, start, allowDollar); |
956 } else if (next < 128) { | 954 } else if (next < 128) { |
957 _appendKeywordToken(state.keyword()); | 955 _appendKeywordToken(state.keyword()); |
958 return next; | 956 return next; |
959 } else { | 957 } else { |
960 return _tokenizeIdentifier(next, start, allowDollar); | 958 return _tokenizeIdentifier(next, start, allowDollar); |
961 } | 959 } |
962 } | 960 } |
963 | 961 |
964 int _tokenizeLessThan(int next) { | 962 int _tokenizeLessThan(int next) { |
(...skipping 373 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1338 } | 1336 } |
1339 | 1337 |
1340 /** | 1338 /** |
1341 * Checks if [value] is a single-line or multi-line comment. | 1339 * Checks if [value] is a single-line or multi-line comment. |
1342 */ | 1340 */ |
1343 static bool _isDocumentationComment(String value) { | 1341 static bool _isDocumentationComment(String value) { |
1344 return StringUtilities.startsWith3(value, 0, $slash, $slash, $slash) || | 1342 return StringUtilities.startsWith3(value, 0, $slash, $slash, $slash) || |
1345 StringUtilities.startsWith3(value, 0, $slash, $asterisk, $asterisk); | 1343 StringUtilities.startsWith3(value, 0, $slash, $asterisk, $asterisk); |
1346 } | 1344 } |
1347 } | 1345 } |
OLD | NEW |