Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(199)

Side by Side Diff: pkg/front_end/lib/src/scanner/scanner.dart

Issue 2803563003: enhance analyzer to parse uppercase and built-in/pseudo keywords (Closed)
Patch Set: rebase Created 3 years, 8 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « pkg/analyzer/lib/src/generated/parser.dart ('k') | pkg/front_end/lib/src/scanner/token.dart » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 import 'package:charcode/ascii.dart'; 5 import 'package:charcode/ascii.dart';
6 import 'package:front_end/src/scanner/errors.dart'; 6 import 'package:front_end/src/scanner/errors.dart';
7 import 'package:front_end/src/scanner/reader.dart'; 7 import 'package:front_end/src/scanner/reader.dart';
8 import 'package:front_end/src/scanner/string_utilities.dart'; 8 import 'package:front_end/src/scanner/string_utilities.dart';
9 import 'package:front_end/src/scanner/token.dart'; 9 import 'package:front_end/src/scanner/token.dart';
10 10
11 /** 11 /**
12 * A state in a state machine used to scan keywords. 12 * A state in a state machine used to scan keywords.
13 */ 13 */
14 class KeywordState { 14 class KeywordState {
15 /** 15 /**
16 * An empty transition table used by leaf states. 16 * An empty transition table used by leaf states.
17 */ 17 */
18 static List<KeywordState> _EMPTY_TABLE = new List<KeywordState>(26); 18 static List<KeywordState> _EMPTY_TABLE = new List<KeywordState>($z - $A + 1);
19 19
20 /** 20 /**
21 * The initial state in the state machine. 21 * The initial state in the state machine.
22 */ 22 */
23 static final KeywordState KEYWORD_STATE = _createKeywordStateTable(); 23 static final KeywordState KEYWORD_STATE = _createKeywordStateTable();
24 24
25 /** 25 /**
26 * A table mapping characters to the states to which those characters will 26 * A table mapping characters to the states to which those characters will
27 * transition. (The index into the array is the offset from the character 27 * transition. (The index into the array is the offset from the character
28 * `'a'` to the transitioning character.) 28 * `'a'` to the transitioning character.)
(...skipping 18 matching lines...) Expand all
47 * Return the keyword that was recognized by this state, or `null` if this 47 * Return the keyword that was recognized by this state, or `null` if this
48 * state does not recognized a keyword. 48 * state does not recognized a keyword.
49 */ 49 */
50 Keyword keyword() => _keyword; 50 Keyword keyword() => _keyword;
51 51
52 /** 52 /**
53 * Return the state that follows this state on a transition of the given 53 * Return the state that follows this state on a transition of the given
54 * [character], or `null` if there is no valid state reachable from this state 54 * [character], or `null` if there is no valid state reachable from this state
55 * with such a transition. 55 * with such a transition.
56 */ 56 */
57 KeywordState next(int character) => _table[character - $a]; 57 KeywordState next(int character) => _table[character - $A];
58 58
59 /** 59 /**
60 * Create the next state in the state machine where we have already recognized 60 * Create the next state in the state machine where we have already recognized
61 * the subset of strings in the given array of [strings] starting at the given 61 * the subset of strings in the given array of [strings] starting at the given
62 * [offset] and having the given [length]. All of these strings have a common 62 * [offset] and having the given [length]. All of these strings have a common
63 * prefix and the next character is at the given [start] index. 63 * prefix and the next character is at the given [start] index.
64 */ 64 */
65 static KeywordState _computeKeywordStateTable( 65 static KeywordState _computeKeywordStateTable(
66 int start, List<String> strings, int offset, int length) { 66 int start, List<String> strings, int offset, int length) {
67 List<KeywordState> result = new List<KeywordState>(26); 67 List<KeywordState> result = new List<KeywordState>($z - $A + 1);
68 assert(length != 0); 68 assert(length != 0);
69 int chunk = $nul; 69 int chunk = $nul;
70 int chunkStart = -1; 70 int chunkStart = -1;
71 bool isLeaf = false; 71 bool isLeaf = false;
72 for (int i = offset; i < offset + length; i++) { 72 for (int i = offset; i < offset + length; i++) {
73 if (strings[i].length == start) { 73 if (strings[i].length == start) {
74 isLeaf = true; 74 isLeaf = true;
75 } 75 }
76 if (strings[i].length > start) { 76 if (strings[i].length > start) {
77 int c = strings[i].codeUnitAt(start); 77 int c = strings[i].codeUnitAt(start);
78 if (chunk != c) { 78 if (chunk != c) {
79 if (chunkStart != -1) { 79 if (chunkStart != -1) {
80 result[chunk - $a] = _computeKeywordStateTable( 80 result[chunk - $A] = _computeKeywordStateTable(
81 start + 1, strings, chunkStart, i - chunkStart); 81 start + 1, strings, chunkStart, i - chunkStart);
82 } 82 }
83 chunkStart = i; 83 chunkStart = i;
84 chunk = c; 84 chunk = c;
85 } 85 }
86 } 86 }
87 } 87 }
88 if (chunkStart != -1) { 88 if (chunkStart != -1) {
89 assert(result[chunk - $a] == null); 89 assert(result[chunk - $A] == null);
90 result[chunk - $a] = _computeKeywordStateTable( 90 result[chunk - $A] = _computeKeywordStateTable(
91 start + 1, strings, chunkStart, offset + length - chunkStart); 91 start + 1, strings, chunkStart, offset + length - chunkStart);
92 } else { 92 } else {
93 assert(length == 1); 93 assert(length == 1);
94 return new KeywordState(_EMPTY_TABLE, strings[offset]); 94 return new KeywordState(_EMPTY_TABLE, strings[offset]);
95 } 95 }
96 if (isLeaf) { 96 if (isLeaf) {
97 return new KeywordState(result, strings[offset]); 97 return new KeywordState(result, strings[offset]);
98 } else { 98 } else {
99 return new KeywordState(result, null); 99 return new KeywordState(result, null);
100 } 100 }
(...skipping 175 matching lines...) Expand 10 before | Expand all | Expand 10 after
276 } 276 }
277 if (next == $r) { 277 if (next == $r) {
278 // 'r' 278 // 'r'
279 int peek = _reader.peek(); 279 int peek = _reader.peek();
280 if (peek == $double_quote || peek == $single_quote) { 280 if (peek == $double_quote || peek == $single_quote) {
281 // '"' || "'" 281 // '"' || "'"
282 int start = _reader.offset; 282 int start = _reader.offset;
283 return _tokenizeString(_reader.advance(), start, true); 283 return _tokenizeString(_reader.advance(), start, true);
284 } 284 }
285 } 285 }
286 if ($a <= next && next <= $z) { 286 if (($A <= next && next <= $Z) || ($a <= next && next <= $z)) {
287 // 'a'-'z' 287 // 'A'-'Z' || 'a'-'z'
288 return _tokenizeKeywordOrIdentifier(next, true); 288 return _tokenizeKeywordOrIdentifier(next, true);
289 } 289 }
290 if (($A <= next && next <= $Z) || next == $_ || next == $$) { 290 if (next == $_ || next == $$) {
291 // 'A'-'Z' || '_' || '$' 291 // '_' || '$'
292 return _tokenizeIdentifier(next, _reader.offset, true); 292 return _tokenizeIdentifier(next, _reader.offset, true);
293 } 293 }
294 if (next == $lt) { 294 if (next == $lt) {
295 // '<' 295 // '<'
296 return _tokenizeLessThan(next); 296 return _tokenizeLessThan(next);
297 } 297 }
298 if (next == $gt) { 298 if (next == $gt) {
299 // '>' 299 // '>'
300 return _tokenizeGreaterThan(next); 300 return _tokenizeGreaterThan(next);
301 } 301 }
(...skipping 632 matching lines...) Expand 10 before | Expand all | Expand 10 after
934 _beginToken(); 934 _beginToken();
935 next = _tokenizeKeywordOrIdentifier(next, false); 935 next = _tokenizeKeywordOrIdentifier(next, false);
936 } 936 }
937 _beginToken(); 937 _beginToken();
938 return next; 938 return next;
939 } 939 }
940 940
941 int _tokenizeKeywordOrIdentifier(int next, bool allowDollar) { 941 int _tokenizeKeywordOrIdentifier(int next, bool allowDollar) {
942 KeywordState state = KeywordState.KEYWORD_STATE; 942 KeywordState state = KeywordState.KEYWORD_STATE;
943 int start = _reader.offset; 943 int start = _reader.offset;
944 while (state != null && $a <= next && next <= $z) { 944 while (state != null &&
945 (($A <= next && next <= $Z) || $a <= next && next <= $z)) {
945 state = state.next(next); 946 state = state.next(next);
946 next = _reader.advance(); 947 next = _reader.advance();
947 } 948 }
948 if (state == null || state.keyword() == null) { 949 if (state == null || state.keyword() == null) {
949 return _tokenizeIdentifier(next, start, allowDollar); 950 return _tokenizeIdentifier(next, start, allowDollar);
950 } 951 }
951 if (($A <= next && next <= $Z) || 952 if (($0 <= next && next <= $9) || next == $_ || next == $$) {
952 ($0 <= next && next <= $9) ||
953 next == $_ ||
954 next == $$) {
955 return _tokenizeIdentifier(next, start, allowDollar); 953 return _tokenizeIdentifier(next, start, allowDollar);
956 } else if (next < 128) { 954 } else if (next < 128) {
957 _appendKeywordToken(state.keyword()); 955 _appendKeywordToken(state.keyword());
958 return next; 956 return next;
959 } else { 957 } else {
960 return _tokenizeIdentifier(next, start, allowDollar); 958 return _tokenizeIdentifier(next, start, allowDollar);
961 } 959 }
962 } 960 }
963 961
964 int _tokenizeLessThan(int next) { 962 int _tokenizeLessThan(int next) {
(...skipping 373 matching lines...) Expand 10 before | Expand all | Expand 10 after
1338 } 1336 }
1339 1337
1340 /** 1338 /**
1341 * Checks if [value] is a single-line or multi-line comment. 1339 * Checks if [value] is a single-line or multi-line comment.
1342 */ 1340 */
1343 static bool _isDocumentationComment(String value) { 1341 static bool _isDocumentationComment(String value) {
1344 return StringUtilities.startsWith3(value, 0, $slash, $slash, $slash) || 1342 return StringUtilities.startsWith3(value, 0, $slash, $slash, $slash) ||
1345 StringUtilities.startsWith3(value, 0, $slash, $asterisk, $asterisk); 1343 StringUtilities.startsWith3(value, 0, $slash, $asterisk, $asterisk);
1346 } 1344 }
1347 } 1345 }
OLDNEW
« no previous file with comments | « pkg/analyzer/lib/src/generated/parser.dart ('k') | pkg/front_end/lib/src/scanner/token.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698