| OLD | NEW |
| 1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library fasta.analyzer.token_utils; | 5 library fasta.analyzer.token_utils; |
| 6 | 6 |
| 7 import '../parser/error_kind.dart' show ErrorKind; | 7 import '../parser/error_kind.dart' show ErrorKind; |
| 8 | 8 |
| 9 import '../scanner/error_token.dart' show ErrorToken; | 9 import '../scanner/error_token.dart' show ErrorToken; |
| 10 | 10 |
| (...skipping 198 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 209 default: | 209 default: |
| 210 throw new UnimplementedError('$errorCode'); | 210 throw new UnimplementedError('$errorCode'); |
| 211 } | 211 } |
| 212 } | 212 } |
| 213 } | 213 } |
| 214 | 214 |
| 215 /// Converts a single Fasta comment token to an analyzer comment token. | 215 /// Converts a single Fasta comment token to an analyzer comment token. |
| 216 analyzer.CommentToken toAnalyzerCommentToken(Token token) { | 216 analyzer.CommentToken toAnalyzerCommentToken(Token token) { |
| 217 // TODO(paulberry,ahe): It would be nice if the scanner gave us an | 217 // TODO(paulberry,ahe): It would be nice if the scanner gave us an |
| 218 // easier way to distinguish between the two types of comment. | 218 // easier way to distinguish between the two types of comment. |
| 219 var type = token.value.startsWith('/*') | 219 var type = token.lexeme.startsWith('/*') |
| 220 ? TokenType.MULTI_LINE_COMMENT | 220 ? TokenType.MULTI_LINE_COMMENT |
| 221 : TokenType.SINGLE_LINE_COMMENT; | 221 : TokenType.SINGLE_LINE_COMMENT; |
| 222 return new analyzer.CommentToken(type, token.value, token.charOffset); | 222 return new analyzer.CommentToken(type, token.lexeme, token.charOffset); |
| 223 } | 223 } |
| 224 | 224 |
| 225 /// Converts a stream of Analyzer tokens (starting with [token] and continuing | 225 /// Converts a stream of Analyzer tokens (starting with [token] and continuing |
| 226 /// to EOF) to a stream of Fasta tokens. | 226 /// to EOF) to a stream of Fasta tokens. |
| 227 /// | 227 /// |
| 228 /// TODO(paulberry): Analyzer tokens do not record error conditions, so a round | 228 /// TODO(paulberry): Analyzer tokens do not record error conditions, so a round |
| 229 /// trip through this function and [toAnalyzerTokenStream] will lose error | 229 /// trip through this function and [toAnalyzerTokenStream] will lose error |
| 230 /// information. | 230 /// information. |
| 231 Token fromAnalyzerTokenStream(analyzer.Token analyzerToken) { | 231 Token fromAnalyzerTokenStream(analyzer.Token analyzerToken) { |
| 232 Token tokenHead = new SymbolToken(EOF_INFO, -1); | 232 Token tokenHead = new SymbolToken(EOF_INFO, -1); |
| (...skipping 266 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 499 if (token.info.kind != BAD_INPUT_TOKEN) return false; | 499 if (token.info.kind != BAD_INPUT_TOKEN) return false; |
| 500 // Otherwise keep looking. | 500 // Otherwise keep looking. |
| 501 } | 501 } |
| 502 } | 502 } |
| 503 | 503 |
| 504 analyzer.Token toAnalyzerToken(Token token, | 504 analyzer.Token toAnalyzerToken(Token token, |
| 505 [analyzer.CommentToken commentToken]) { | 505 [analyzer.CommentToken commentToken]) { |
| 506 if (token == null) return null; | 506 if (token == null) return null; |
| 507 analyzer.Token makeStringToken(TokenType tokenType) { | 507 analyzer.Token makeStringToken(TokenType tokenType) { |
| 508 if (commentToken == null) { | 508 if (commentToken == null) { |
| 509 return new analyzer.StringToken(tokenType, token.value, token.charOffset); | 509 return new analyzer.StringToken(tokenType, token.lexeme, token.charOffset)
; |
| 510 } else { | 510 } else { |
| 511 return new analyzer.StringTokenWithComment( | 511 return new analyzer.StringTokenWithComment( |
| 512 tokenType, token.value, token.charOffset, commentToken); | 512 tokenType, token.lexeme, token.charOffset, commentToken); |
| 513 } | 513 } |
| 514 } | 514 } |
| 515 | 515 |
| 516 analyzer.Token makeBeginToken(TokenType tokenType) { | 516 analyzer.Token makeBeginToken(TokenType tokenType) { |
| 517 if (commentToken == null) { | 517 if (commentToken == null) { |
| 518 return new analyzer.BeginToken(tokenType, token.charOffset); | 518 return new analyzer.BeginToken(tokenType, token.charOffset); |
| 519 } else { | 519 } else { |
| 520 return new analyzer.BeginTokenWithComment( | 520 return new analyzer.BeginTokenWithComment( |
| 521 tokenType, token.charOffset, commentToken); | 521 tokenType, token.charOffset, commentToken); |
| 522 } | 522 } |
| (...skipping 259 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 782 case PERIOD_PERIOD_PERIOD_TOKEN: | 782 case PERIOD_PERIOD_PERIOD_TOKEN: |
| 783 return TokenType.PERIOD_PERIOD_PERIOD; | 783 return TokenType.PERIOD_PERIOD_PERIOD; |
| 784 // case GENERIC_METHOD_TYPE_LIST_TOKEN: | 784 // case GENERIC_METHOD_TYPE_LIST_TOKEN: |
| 785 // return TokenType.GENERIC_METHOD_TYPE_LIST; | 785 // return TokenType.GENERIC_METHOD_TYPE_LIST; |
| 786 // case GENERIC_METHOD_TYPE_ASSIGN_TOKEN: | 786 // case GENERIC_METHOD_TYPE_ASSIGN_TOKEN: |
| 787 // return TokenType.GENERIC_METHOD_TYPE_ASSIGN; | 787 // return TokenType.GENERIC_METHOD_TYPE_ASSIGN; |
| 788 default: | 788 default: |
| 789 return internalError("Unhandled token ${token.info}"); | 789 return internalError("Unhandled token ${token.info}"); |
| 790 } | 790 } |
| 791 } | 791 } |
| OLD | NEW |