Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(223)

Side by Side Diff: pkg/front_end/lib/src/fasta/analyzer/token_utils.dart

Issue 2738313002: rename fasta.Token.value --> lexeme (Closed)
Patch Set: merge Created 3 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 library fasta.analyzer.token_utils; 5 library fasta.analyzer.token_utils;
6 6
7 import '../parser/error_kind.dart' show ErrorKind; 7 import '../parser/error_kind.dart' show ErrorKind;
8 8
9 import '../scanner/error_token.dart' show ErrorToken; 9 import '../scanner/error_token.dart' show ErrorToken;
10 10
(...skipping 198 matching lines...) Expand 10 before | Expand all | Expand 10 after
209 default: 209 default:
210 throw new UnimplementedError('$errorCode'); 210 throw new UnimplementedError('$errorCode');
211 } 211 }
212 } 212 }
213 } 213 }
214 214
215 /// Converts a single Fasta comment token to an analyzer comment token. 215 /// Converts a single Fasta comment token to an analyzer comment token.
216 analyzer.CommentToken toAnalyzerCommentToken(Token token) { 216 analyzer.CommentToken toAnalyzerCommentToken(Token token) {
217 // TODO(paulberry,ahe): It would be nice if the scanner gave us an 217 // TODO(paulberry,ahe): It would be nice if the scanner gave us an
218 // easier way to distinguish between the two types of comment. 218 // easier way to distinguish between the two types of comment.
219 var type = token.value.startsWith('/*') 219 var type = token.lexeme.startsWith('/*')
220 ? TokenType.MULTI_LINE_COMMENT 220 ? TokenType.MULTI_LINE_COMMENT
221 : TokenType.SINGLE_LINE_COMMENT; 221 : TokenType.SINGLE_LINE_COMMENT;
222 return new analyzer.CommentToken(type, token.value, token.charOffset); 222 return new analyzer.CommentToken(type, token.lexeme, token.charOffset);
223 } 223 }
224 224
225 /// Converts a stream of Analyzer tokens (starting with [token] and continuing 225 /// Converts a stream of Analyzer tokens (starting with [token] and continuing
226 /// to EOF) to a stream of Fasta tokens. 226 /// to EOF) to a stream of Fasta tokens.
227 /// 227 ///
228 /// TODO(paulberry): Analyzer tokens do not record error conditions, so a round 228 /// TODO(paulberry): Analyzer tokens do not record error conditions, so a round
229 /// trip through this function and [toAnalyzerTokenStream] will lose error 229 /// trip through this function and [toAnalyzerTokenStream] will lose error
230 /// information. 230 /// information.
231 Token fromAnalyzerTokenStream(analyzer.Token analyzerToken) { 231 Token fromAnalyzerTokenStream(analyzer.Token analyzerToken) {
232 Token tokenHead = new SymbolToken(EOF_INFO, -1); 232 Token tokenHead = new SymbolToken(EOF_INFO, -1);
(...skipping 266 matching lines...) Expand 10 before | Expand all | Expand 10 after
499 if (token.info.kind != BAD_INPUT_TOKEN) return false; 499 if (token.info.kind != BAD_INPUT_TOKEN) return false;
500 // Otherwise keep looking. 500 // Otherwise keep looking.
501 } 501 }
502 } 502 }
503 503
504 analyzer.Token toAnalyzerToken(Token token, 504 analyzer.Token toAnalyzerToken(Token token,
505 [analyzer.CommentToken commentToken]) { 505 [analyzer.CommentToken commentToken]) {
506 if (token == null) return null; 506 if (token == null) return null;
507 analyzer.Token makeStringToken(TokenType tokenType) { 507 analyzer.Token makeStringToken(TokenType tokenType) {
508 if (commentToken == null) { 508 if (commentToken == null) {
509 return new analyzer.StringToken(tokenType, token.value, token.charOffset); 509 return new analyzer.StringToken(tokenType, token.lexeme, token.charOffset) ;
510 } else { 510 } else {
511 return new analyzer.StringTokenWithComment( 511 return new analyzer.StringTokenWithComment(
512 tokenType, token.value, token.charOffset, commentToken); 512 tokenType, token.lexeme, token.charOffset, commentToken);
513 } 513 }
514 } 514 }
515 515
516 analyzer.Token makeBeginToken(TokenType tokenType) { 516 analyzer.Token makeBeginToken(TokenType tokenType) {
517 if (commentToken == null) { 517 if (commentToken == null) {
518 return new analyzer.BeginToken(tokenType, token.charOffset); 518 return new analyzer.BeginToken(tokenType, token.charOffset);
519 } else { 519 } else {
520 return new analyzer.BeginTokenWithComment( 520 return new analyzer.BeginTokenWithComment(
521 tokenType, token.charOffset, commentToken); 521 tokenType, token.charOffset, commentToken);
522 } 522 }
(...skipping 259 matching lines...) Expand 10 before | Expand all | Expand 10 after
782 case PERIOD_PERIOD_PERIOD_TOKEN: 782 case PERIOD_PERIOD_PERIOD_TOKEN:
783 return TokenType.PERIOD_PERIOD_PERIOD; 783 return TokenType.PERIOD_PERIOD_PERIOD;
784 // case GENERIC_METHOD_TYPE_LIST_TOKEN: 784 // case GENERIC_METHOD_TYPE_LIST_TOKEN:
785 // return TokenType.GENERIC_METHOD_TYPE_LIST; 785 // return TokenType.GENERIC_METHOD_TYPE_LIST;
786 // case GENERIC_METHOD_TYPE_ASSIGN_TOKEN: 786 // case GENERIC_METHOD_TYPE_ASSIGN_TOKEN:
787 // return TokenType.GENERIC_METHOD_TYPE_ASSIGN; 787 // return TokenType.GENERIC_METHOD_TYPE_ASSIGN;
788 default: 788 default:
789 return internalError("Unhandled token ${token.info}"); 789 return internalError("Unhandled token ${token.info}");
790 } 790 }
791 } 791 }
OLDNEW
« no previous file with comments | « pkg/front_end/lib/src/fasta/analyzer/ast_builder.dart ('k') | pkg/front_end/lib/src/fasta/kernel/body_builder.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698