Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(104)

Side by Side Diff: pkg/front_end/lib/src/fasta/scanner/array_based_scanner.dart

Issue 2837473002: merge fasta.PrecedenceInfo into analyzer.TokenType (Closed)
Patch Set: rebase Created 3 years, 8 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 library fasta.scanner.array_based_scanner; 5 library fasta.scanner.array_based_scanner;
6 6
7 import 'error_token.dart' show ErrorToken, UnmatchedToken; 7 import 'error_token.dart' show ErrorToken, UnmatchedToken;
8 8
9 import '../../scanner/token.dart' show Keyword; 9 import '../../scanner/token.dart' show Keyword, TokenType;
10
11 import 'precedence.dart' show PrecedenceInfo;
12 10
13 import 'token.dart' 11 import 'token.dart'
14 show 12 show
15 BeginGroupToken, 13 BeginGroupToken,
16 KeywordToken, 14 KeywordToken,
17 StringToken, 15 StringToken,
18 SymbolToken, 16 SymbolToken,
19 SyntheticSymbolToken, 17 SyntheticSymbolToken,
20 Token; 18 Token;
21 19
(...skipping 23 matching lines...) Expand all
45 */ 43 */
46 Link<BeginGroupToken> groupingStack = const Link<BeginGroupToken>(); 44 Link<BeginGroupToken> groupingStack = const Link<BeginGroupToken>();
47 45
48 /** 46 /**
49 * Appends a fixed token whose kind and content is determined by [info]. 47 * Appends a fixed token whose kind and content is determined by [info].
50 * Appends an *operator* token from [info]. 48 * Appends an *operator* token from [info].
51 * 49 *
52 * An operator token represent operators like ':', '.', ';', '&&', '==', '--', 50 * An operator token represent operators like ':', '.', ';', '&&', '==', '--',
53 * '=>', etc. 51 * '=>', etc.
54 */ 52 */
55 void appendPrecedenceToken(PrecedenceInfo info) { 53 void appendPrecedenceToken(TokenType info) {
56 appendToken(new SymbolToken(info, tokenStart)); 54 appendToken(new SymbolToken(info, tokenStart));
57 } 55 }
58 56
59 /** 57 /**
60 * Appends a fixed token based on whether the current char is [choice] or not. 58 * Appends a fixed token based on whether the current char is [choice] or not.
61 * If the current char is [choice] a fixed token whose kind and content 59 * If the current char is [choice] a fixed token whose kind and content
62 * is determined by [yes] is appended, otherwise a fixed token whose kind 60 * is determined by [yes] is appended, otherwise a fixed token whose kind
63 * and content is determined by [no] is appended. 61 * and content is determined by [no] is appended.
64 */ 62 */
65 int select(int choice, PrecedenceInfo yes, PrecedenceInfo no) { 63 int select(int choice, TokenType yes, TokenType no) {
66 int next = advance(); 64 int next = advance();
67 if (identical(next, choice)) { 65 if (identical(next, choice)) {
68 appendPrecedenceToken(yes); 66 appendPrecedenceToken(yes);
69 return advance(); 67 return advance();
70 } else { 68 } else {
71 appendPrecedenceToken(no); 69 appendPrecedenceToken(no);
72 return next; 70 return next;
73 } 71 }
74 } 72 }
75 73
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after
115 * [lineStarts] map. 113 * [lineStarts] map.
116 */ 114 */
117 void lineFeedInMultiline() { 115 void lineFeedInMultiline() {
118 lineStarts.add(stringOffset + 1); 116 lineStarts.add(stringOffset + 1);
119 } 117 }
120 118
121 /** 119 /**
122 * Appends a token that begins a new group, represented by [info]. 120 * Appends a token that begins a new group, represented by [info].
123 * Group begin tokens are '{', '(', '[' and '${'. 121 * Group begin tokens are '{', '(', '[' and '${'.
124 */ 122 */
125 void appendBeginGroup(PrecedenceInfo info) { 123 void appendBeginGroup(TokenType info) {
126 Token token = new BeginGroupToken(info, tokenStart); 124 Token token = new BeginGroupToken(info, tokenStart);
127 appendToken(token); 125 appendToken(token);
128 126
129 // { [ ${ cannot appear inside a type parameters / arguments. 127 // { [ ${ cannot appear inside a type parameters / arguments.
130 if (!identical(info.kind, LT_TOKEN) && 128 if (!identical(info.kind, LT_TOKEN) &&
131 !identical(info.kind, OPEN_PAREN_TOKEN)) { 129 !identical(info.kind, OPEN_PAREN_TOKEN)) {
132 discardOpenLt(); 130 discardOpenLt();
133 } 131 }
134 groupingStack = groupingStack.prepend(token); 132 groupingStack = groupingStack.prepend(token);
135 } 133 }
136 134
137 /** 135 /**
138 * Appends a token that begins an end group, represented by [info]. 136 * Appends a token that begins an end group, represented by [info].
139 * It handles the group end tokens '}', ')' and ']'. The tokens '>' and 137 * It handles the group end tokens '}', ')' and ']'. The tokens '>' and
140 * '>>' are handled separately bo [appendGt] and [appendGtGt]. 138 * '>>' are handled separately bo [appendGt] and [appendGtGt].
141 */ 139 */
142 int appendEndGroup(PrecedenceInfo info, int openKind) { 140 int appendEndGroup(TokenType info, int openKind) {
143 assert(!identical(openKind, LT_TOKEN)); // openKind is < for > and >> 141 assert(!identical(openKind, LT_TOKEN)); // openKind is < for > and >>
144 discardBeginGroupUntil(openKind); 142 discardBeginGroupUntil(openKind);
145 appendPrecedenceToken(info); 143 appendPrecedenceToken(info);
146 Token close = tail; 144 Token close = tail;
147 if (groupingStack.isEmpty) { 145 if (groupingStack.isEmpty) {
148 return advance(); 146 return advance();
149 } 147 }
150 BeginGroupToken begin = groupingStack.head; 148 BeginGroupToken begin = groupingStack.head;
151 if (!identical(begin.kind, openKind)) { 149 if (!identical(begin.kind, openKind)) {
152 assert(begin.kind == STRING_INTERPOLATION_TOKEN && 150 assert(begin.kind == STRING_INTERPOLATION_TOKEN &&
(...skipping 26 matching lines...) Expand all
179 unmatchedBeginGroup(begin); 177 unmatchedBeginGroup(begin);
180 groupingStack = groupingStack.tail; 178 groupingStack = groupingStack.tail;
181 } 179 }
182 } 180 }
183 181
184 /** 182 /**
185 * Appends a token for '>'. 183 * Appends a token for '>'.
186 * This method does not issue unmatched errors, because > is also the 184 * This method does not issue unmatched errors, because > is also the
187 * greater-than operator. It does not necessarily have to close a group. 185 * greater-than operator. It does not necessarily have to close a group.
188 */ 186 */
189 void appendGt(PrecedenceInfo info) { 187 void appendGt(TokenType info) {
190 appendPrecedenceToken(info); 188 appendPrecedenceToken(info);
191 if (groupingStack.isEmpty) return; 189 if (groupingStack.isEmpty) return;
192 if (identical(groupingStack.head.kind, LT_TOKEN)) { 190 if (identical(groupingStack.head.kind, LT_TOKEN)) {
193 groupingStack.head.endGroup = tail; 191 groupingStack.head.endGroup = tail;
194 groupingStack = groupingStack.tail; 192 groupingStack = groupingStack.tail;
195 } 193 }
196 } 194 }
197 195
198 /** 196 /**
199 * Appends a token for '>>'. 197 * Appends a token for '>>'.
200 * This method does not issue unmatched errors, because >> is also the 198 * This method does not issue unmatched errors, because >> is also the
201 * shift operator. It does not necessarily have to close a group. 199 * shift operator. It does not necessarily have to close a group.
202 */ 200 */
203 void appendGtGt(PrecedenceInfo info) { 201 void appendGtGt(TokenType info) {
204 appendPrecedenceToken(info); 202 appendPrecedenceToken(info);
205 if (groupingStack.isEmpty) return; 203 if (groupingStack.isEmpty) return;
206 if (identical(groupingStack.head.kind, LT_TOKEN)) { 204 if (identical(groupingStack.head.kind, LT_TOKEN)) {
207 // Don't assign endGroup: in "T<U<V>>", the '>>' token closes the outer 205 // Don't assign endGroup: in "T<U<V>>", the '>>' token closes the outer
208 // '<', the inner '<' is left without endGroup. 206 // '<', the inner '<' is left without endGroup.
209 groupingStack = groupingStack.tail; 207 groupingStack = groupingStack.tail;
210 } 208 }
211 if (groupingStack.isEmpty) return; 209 if (groupingStack.isEmpty) return;
212 if (identical(groupingStack.head.kind, LT_TOKEN)) { 210 if (identical(groupingStack.head.kind, LT_TOKEN)) {
213 groupingStack.head.endGroup = tail; 211 groupingStack.head.endGroup = tail;
214 groupingStack = groupingStack.tail; 212 groupingStack = groupingStack.tail;
215 } 213 }
216 } 214 }
217 215
218 void appendErrorToken(ErrorToken token) { 216 void appendErrorToken(ErrorToken token) {
219 hasErrors = true; 217 hasErrors = true;
220 appendToken(token); 218 appendToken(token);
221 } 219 }
222 220
223 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, 221 void appendSubstringToken(TokenType info, int start, bool asciiOnly,
224 [int extraOffset = 0]) { 222 [int extraOffset = 0]) {
225 appendToken(createSubstringToken(info, start, asciiOnly, extraOffset)); 223 appendToken(createSubstringToken(info, start, asciiOnly, extraOffset));
226 } 224 }
227 225
228 /** 226 /**
229 * Returns a new substring from the scan offset [start] to the current 227 * Returns a new substring from the scan offset [start] to the current
230 * [scanOffset] plus the [extraOffset]. For example, if the current 228 * [scanOffset] plus the [extraOffset]. For example, if the current
231 * scanOffset is 10, then [appendSubstringToken(5, -1)] will append the 229 * scanOffset is 10, then [appendSubstringToken(5, -1)] will append the
232 * substring string [5,9). 230 * substring string [5,9).
233 * 231 *
234 * Note that [extraOffset] can only be used if the covered character(s) are 232 * Note that [extraOffset] can only be used if the covered character(s) are
235 * known to be ASCII. 233 * known to be ASCII.
236 */ 234 */
237 StringToken createSubstringToken( 235 StringToken createSubstringToken(TokenType info, int start, bool asciiOnly,
238 PrecedenceInfo info, int start, bool asciiOnly,
239 [int extraOffset = 0]); 236 [int extraOffset = 0]);
240 237
241 /** 238 /**
242 * This method is called to discard '<' from the "grouping" stack. 239 * This method is called to discard '<' from the "grouping" stack.
243 * 240 *
244 * [PartialParser.skipExpression] relies on the fact that we do not 241 * [PartialParser.skipExpression] relies on the fact that we do not
245 * create groups for stuff like: 242 * create groups for stuff like:
246 * [:a = b < c, d = e > f:]. 243 * [:a = b < c, d = e > f:].
247 * 244 *
248 * In other words, this method is called when the scanner recognizes 245 * In other words, this method is called when the scanner recognizes
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after
294 // v | 291 // v |
295 // SymbolToken(})<----------------------------------+ 292 // SymbolToken(})<----------------------------------+
296 // | 293 // |
297 // next 294 // next
298 // v 295 // v
299 // SymbolToken(;) 296 // SymbolToken(;)
300 // | 297 // |
301 // next 298 // next
302 // v 299 // v
303 // EOF 300 // EOF
304 PrecedenceInfo info = closeBraceInfoFor(begin); 301 TokenType info = closeBraceInfoFor(begin);
305 appendToken(new SyntheticSymbolToken(info, tokenStart)); 302 appendToken(new SyntheticSymbolToken(info, tokenStart));
306 begin.endGroup = tail; 303 begin.endGroup = tail;
307 appendErrorToken(new UnmatchedToken(begin)); 304 appendErrorToken(new UnmatchedToken(begin));
308 } 305 }
309 } 306 }
OLDNEW
« no previous file with comments | « pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart ('k') | pkg/front_end/lib/src/fasta/scanner/error_token.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698