Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(76)

Side by Side Diff: pkg/front_end/lib/src/fasta/analyzer/token_utils.dart

Issue 2693793004: Match "<", ">", and ">>" when translating analyzer tokens to Fasta tokens. (Closed)
Patch Set: Created 3 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | pkg/front_end/test/scanner_fasta_test.dart » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 library fasta.analyzer.token_utils; 5 library fasta.analyzer.token_utils;
6 6
7 import 'package:front_end/src/fasta/parser/error_kind.dart' show 7 import 'package:front_end/src/fasta/parser/error_kind.dart' show
8 ErrorKind; 8 ErrorKind;
9 9
10 import 'package:front_end/src/fasta/scanner/error_token.dart' show 10 import 'package:front_end/src/fasta/scanner/error_token.dart' show
(...skipping 117 matching lines...) Expand 10 before | Expand all | Expand 10 after
128 /// TODO(paulberry): Analyzer tokens do not record error conditions, so a round 128 /// TODO(paulberry): Analyzer tokens do not record error conditions, so a round
129 /// trip through this function and [toAnalyzerTokenStream] will lose error 129 /// trip through this function and [toAnalyzerTokenStream] will lose error
130 /// information. 130 /// information.
131 Token fromAnalyzerTokenStream(analyzer.Token analyzerToken) { 131 Token fromAnalyzerTokenStream(analyzer.Token analyzerToken) {
132 Token tokenHead = new SymbolToken(EOF_INFO, -1); 132 Token tokenHead = new SymbolToken(EOF_INFO, -1);
133 Token tokenTail = tokenHead; 133 Token tokenTail = tokenHead;
134 // Note: beginTokenStack and endTokenStack are seeded with a sentinel value 134 // Note: beginTokenStack and endTokenStack are seeded with a sentinel value
135 // so that we don't have to check if they're empty. 135 // so that we don't have to check if they're empty.
136 var beginTokenStack = <BeginGroupToken>[null]; 136 var beginTokenStack = <BeginGroupToken>[null];
137 var endTokenStack = <analyzer.Token>[null]; 137 var endTokenStack = <analyzer.Token>[null];
138 var angleBracketStack = <BeginGroupToken>[];
138 void matchGroups(analyzer.Token analyzerToken, Token translatedToken) { 139 void matchGroups(analyzer.Token analyzerToken, Token translatedToken) {
139 // If this token closes a group, set the corresponding opener token to point 140 // If this token closes a group, set the corresponding opener token to point
140 // to it. 141 // to it.
141 if (identical(endTokenStack.last, analyzerToken)) { 142 if (identical(endTokenStack.last, analyzerToken)) {
143 angleBracketStack.clear();
142 beginTokenStack.last.endGroup = translatedToken; 144 beginTokenStack.last.endGroup = translatedToken;
143 beginTokenStack.removeLast(); 145 beginTokenStack.removeLast();
144 endTokenStack.removeLast(); 146 endTokenStack.removeLast();
147 } else if (translatedToken.info.kind == LT_TOKEN) {
148 BeginGroupToken beginGroupToken = translatedToken;
149 angleBracketStack.add(beginGroupToken);
150 } else if (translatedToken.info.kind == GT_TOKEN &&
151 angleBracketStack.isNotEmpty) {
152 angleBracketStack.removeLast().endGroup = translatedToken;
153 } else if (translatedToken.info.kind == GT_GT_TOKEN &&
154 angleBracketStack.isNotEmpty) {
155 angleBracketStack.removeLast();
156 if (angleBracketStack.isNotEmpty) {
157 angleBracketStack.removeLast().endGroup = translatedToken;
158 }
145 } 159 }
146 // If this token opens a group, and there is a matching closer, put it on 160 // If this token opens a group, and there is a matching closer, put it on
147 // the stack. 161 // the stack.
148 // TODO(paulberry): generate synthetic closer tokens and "UnmatchedToken" 162 // TODO(paulberry): generate synthetic closer tokens and "UnmatchedToken"
149 // tokens as appropriate. 163 // tokens as appropriate.
150 // TODO(paulberry): match up "<" and ">"/">>" (analyzer doesn't match
151 // these).
152 if (translatedToken is BeginGroupToken && 164 if (translatedToken is BeginGroupToken &&
153 analyzerToken is analyzer.BeginToken && 165 analyzerToken is analyzer.BeginToken &&
154 analyzerToken.endToken != null) { 166 analyzerToken.endToken != null) {
167 angleBracketStack.clear();
155 beginTokenStack.add(translatedToken); 168 beginTokenStack.add(translatedToken);
156 endTokenStack.add(analyzerToken.endToken); 169 endTokenStack.add(analyzerToken.endToken);
157 } 170 }
158 } 171 }
159 172
160 analyzer.Token translateAndAppend(analyzer.Token analyzerToken) { 173 analyzer.Token translateAndAppend(analyzer.Token analyzerToken) {
161 var token = fromAnalyzerToken(analyzerToken); 174 var token = fromAnalyzerToken(analyzerToken);
162 tokenTail.next = token; 175 tokenTail.next = token;
163 tokenTail = token; 176 tokenTail = token;
164 matchGroups(analyzerToken, token); 177 matchGroups(analyzerToken, token);
(...skipping 104 matching lines...) Expand 10 before | Expand all | Expand 10 after
269 return symbol(GT_GT_INFO); 282 return symbol(GT_GT_INFO);
270 case TokenType.GT_GT_EQ: 283 case TokenType.GT_GT_EQ:
271 return symbol(GT_GT_EQ_INFO); 284 return symbol(GT_GT_EQ_INFO);
272 case TokenType.HASH: 285 case TokenType.HASH:
273 return symbol(HASH_INFO); 286 return symbol(HASH_INFO);
274 case TokenType.INDEX: 287 case TokenType.INDEX:
275 return symbol(INDEX_INFO); 288 return symbol(INDEX_INFO);
276 case TokenType.INDEX_EQ: 289 case TokenType.INDEX_EQ:
277 return symbol(INDEX_EQ_INFO); 290 return symbol(INDEX_EQ_INFO);
278 case TokenType.LT: 291 case TokenType.LT:
279 return symbol(LT_INFO); 292 return beginGroup(LT_INFO);
280 case TokenType.LT_EQ: 293 case TokenType.LT_EQ:
281 return symbol(LT_EQ_INFO); 294 return symbol(LT_EQ_INFO);
282 case TokenType.LT_LT: 295 case TokenType.LT_LT:
283 return symbol(LT_LT_INFO); 296 return symbol(LT_LT_INFO);
284 case TokenType.LT_LT_EQ: 297 case TokenType.LT_LT_EQ:
285 return symbol(LT_LT_EQ_INFO); 298 return symbol(LT_LT_EQ_INFO);
286 case TokenType.MINUS: 299 case TokenType.MINUS:
287 return symbol(MINUS_INFO); 300 return symbol(MINUS_INFO);
288 case TokenType.MINUS_EQ: 301 case TokenType.MINUS_EQ:
289 return symbol(MINUS_EQ_INFO); 302 return symbol(MINUS_EQ_INFO);
(...skipping 346 matching lines...) Expand 10 before | Expand all | Expand 10 after
636 case BACKSLASH_TOKEN: return TokenType.BACKSLASH; 649 case BACKSLASH_TOKEN: return TokenType.BACKSLASH;
637 case PERIOD_PERIOD_PERIOD_TOKEN: return TokenType.PERIOD_PERIOD_PERIOD; 650 case PERIOD_PERIOD_PERIOD_TOKEN: return TokenType.PERIOD_PERIOD_PERIOD;
638 // case GENERIC_METHOD_TYPE_LIST_TOKEN: 651 // case GENERIC_METHOD_TYPE_LIST_TOKEN:
639 // return TokenType.GENERIC_METHOD_TYPE_LIST; 652 // return TokenType.GENERIC_METHOD_TYPE_LIST;
640 // case GENERIC_METHOD_TYPE_ASSIGN_TOKEN: 653 // case GENERIC_METHOD_TYPE_ASSIGN_TOKEN:
641 // return TokenType.GENERIC_METHOD_TYPE_ASSIGN; 654 // return TokenType.GENERIC_METHOD_TYPE_ASSIGN;
642 default: 655 default:
643 return internalError("Unhandled token ${token.info}"); 656 return internalError("Unhandled token ${token.info}");
644 } 657 }
645 } 658 }
OLDNEW
« no previous file with comments | « no previous file | pkg/front_end/test/scanner_fasta_test.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698