| OLD | NEW |
| 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library fasta.scanner.token; | 5 library fasta.scanner.token; |
| 6 | 6 |
| 7 import '../../scanner/token.dart' as analyzer; | 7 import '../../scanner/token.dart' as analyzer; |
| 8 import '../../scanner/token.dart' show TokenType; | 8 import '../../scanner/token.dart' show TokenType; |
| 9 | 9 |
| 10 import 'precedence.dart' | |
| 11 show AS_INFO, BAD_INPUT_INFO, EOF_INFO, IS_INFO, KEYWORD_INFO; | |
| 12 | |
| 13 import 'token_constants.dart' show IDENTIFIER_TOKEN; | 10 import 'token_constants.dart' show IDENTIFIER_TOKEN; |
| 14 | 11 |
| 15 import 'string_canonicalizer.dart'; | 12 import 'string_canonicalizer.dart'; |
| 16 | 13 |
| 17 /** | 14 /** |
| 18 * A token that doubles as a linked list. | 15 * A token that doubles as a linked list. |
| 19 */ | 16 */ |
| 20 abstract class Token implements analyzer.TokenWithComment { | 17 abstract class Token implements analyzer.TokenWithComment { |
| 21 /** | 18 /** |
| 22 * The character offset of the start of this token within the source text. | 19 * The character offset of the start of this token within the source text. |
| (...skipping 92 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 115 * token 'foo'. | 112 * token 'foo'. |
| 116 * | 113 * |
| 117 * Use [lexeme] for the text actually parsed by the token. | 114 * Use [lexeme] for the text actually parsed by the token. |
| 118 */ | 115 */ |
| 119 String toString(); | 116 String toString(); |
| 120 | 117 |
| 121 /** | 118 /** |
| 122 * The number of characters parsed by this token. | 119 * The number of characters parsed by this token. |
| 123 */ | 120 */ |
| 124 int get charCount { | 121 int get charCount { |
| 125 if (info == BAD_INPUT_INFO) { | 122 if (info == analyzer.TokenType.BAD_INPUT) { |
| 126 // This is a token that wraps around an error message. Return 1 | 123 // This is a token that wraps around an error message. Return 1 |
| 127 // instead of the size of the length of the error message. | 124 // instead of the size of the length of the error message. |
| 128 return 1; | 125 return 1; |
| 129 } else { | 126 } else { |
| 130 return lexeme.length; | 127 return lexeme.length; |
| 131 } | 128 } |
| 132 } | 129 } |
| 133 | 130 |
| 134 /// The character offset of the end of this token within the source text. | 131 /// The character offset of the end of this token within the source text. |
| 135 int get charEnd => charOffset + charCount; | 132 int get charEnd => charOffset + charCount; |
| 136 | 133 |
| 137 bool get isEof => false; | 134 bool get isEof => false; |
| 138 | 135 |
| 139 bool get isBuiltInIdentifier => false; | 136 bool get isBuiltInIdentifier => false; |
| 140 | 137 |
| 141 @override | 138 @override |
| 142 bool get isOperator => info.isOperator; | 139 bool get isOperator => info.isOperator; |
| 143 | 140 |
| 144 @override | 141 @override |
| 145 bool get isUserDefinableOperator => info.isUserDefinableOperator; | 142 bool get isUserDefinableOperator => info.isUserDefinableOperator; |
| 146 | 143 |
| 147 @override | 144 @override |
| 148 analyzer.TokenType get type { | 145 analyzer.TokenType get type { |
| 149 // Analyzer has a different concept of what is a Keyword type. | 146 // Analyzer has a different concept of what is a Keyword type. |
| 150 return info == AS_INFO || info == IS_INFO ? KEYWORD_INFO : info; | 147 return info == TokenType.AS || info == TokenType.IS |
| 148 ? TokenType.KEYWORD |
| 149 : info; |
| 151 } | 150 } |
| 152 | 151 |
| 153 @override | 152 @override |
| 154 int get offset => charOffset; | 153 int get offset => charOffset; |
| 155 | 154 |
| 156 @override | 155 @override |
| 157 set offset(int newOffset) { | 156 set offset(int newOffset) { |
| 158 charOffset = newOffset; | 157 charOffset = newOffset; |
| 159 } | 158 } |
| 160 | 159 |
| (...skipping 81 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 242 /** | 241 /** |
| 243 * A [SymbolToken] represents the symbol in its precedence info. | 242 * A [SymbolToken] represents the symbol in its precedence info. |
| 244 * Also used for end of file with EOF_INFO. | 243 * Also used for end of file with EOF_INFO. |
| 245 */ | 244 */ |
| 246 class SymbolToken extends Token { | 245 class SymbolToken extends Token { |
| 247 final TokenType info; | 246 final TokenType info; |
| 248 | 247 |
| 249 SymbolToken(this.info, int charOffset) : super(charOffset); | 248 SymbolToken(this.info, int charOffset) : super(charOffset); |
| 250 | 249 |
| 251 factory SymbolToken.eof(int charOffset) { | 250 factory SymbolToken.eof(int charOffset) { |
| 252 var eof = new SyntheticSymbolToken(EOF_INFO, charOffset); | 251 var eof = new SyntheticSymbolToken(analyzer.TokenType.EOF, charOffset); |
| 253 // EOF points to itself so there's always infinite look-ahead. | 252 // EOF points to itself so there's always infinite look-ahead. |
| 254 eof.previousToken = eof; | 253 eof.previousToken = eof; |
| 255 eof.next = eof; | 254 eof.next = eof; |
| 256 return eof; | 255 return eof; |
| 257 } | 256 } |
| 258 | 257 |
| 259 @override | 258 @override |
| 260 String get lexeme => info.value; | 259 String get lexeme => info.value; |
| 261 | 260 |
| 262 @override | 261 @override |
| 263 String get stringValue => info.value; | 262 String get stringValue => info.value; |
| 264 | 263 |
| 265 @override | 264 @override |
| 266 bool isIdentifier() => false; | 265 bool isIdentifier() => false; |
| 267 | 266 |
| 268 @override | 267 @override |
| 269 String toString() => "SymbolToken(${isEof ? '-eof-' : lexeme})"; | 268 String toString() => "SymbolToken(${isEof ? '-eof-' : lexeme})"; |
| 270 | 269 |
| 271 @override | 270 @override |
| 272 bool get isEof => info == EOF_INFO; | 271 bool get isEof => info == analyzer.TokenType.EOF; |
| 273 | 272 |
| 274 @override | 273 @override |
| 275 Token copyWithoutComments() => isEof | 274 Token copyWithoutComments() => isEof |
| 276 ? new SymbolToken.eof(charOffset) | 275 ? new SymbolToken.eof(charOffset) |
| 277 : new SymbolToken(info, charOffset); | 276 : new SymbolToken(info, charOffset); |
| 278 } | 277 } |
| 279 | 278 |
| 280 /** | 279 /** |
| 281 * A [SyntheticSymbolToken] represents the symbol in its precedence info | 280 * A [SyntheticSymbolToken] represents the symbol in its precedence info |
| 282 * which does not exist in the original source. | 281 * which does not exist in the original source. |
| (...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 345 | 344 |
| 346 String toString() => "KeywordToken($lexeme)"; | 345 String toString() => "KeywordToken($lexeme)"; |
| 347 | 346 |
| 348 @override | 347 @override |
| 349 Token copyWithoutComments() => new KeywordToken(keyword, charOffset); | 348 Token copyWithoutComments() => new KeywordToken(keyword, charOffset); |
| 350 | 349 |
| 351 @override | 350 @override |
| 352 analyzer.Keyword value() => keyword; | 351 analyzer.Keyword value() => keyword; |
| 353 | 352 |
| 354 @override | 353 @override |
| 355 analyzer.TokenType get type => KEYWORD_INFO; | 354 analyzer.TokenType get type => TokenType.KEYWORD; |
| 356 } | 355 } |
| 357 | 356 |
| 358 /** | 357 /** |
| 359 * A synthetic keyword token. | 358 * A synthetic keyword token. |
| 360 */ | 359 */ |
| 361 class SyntheticKeywordToken extends KeywordToken | 360 class SyntheticKeywordToken extends KeywordToken |
| 362 implements analyzer.SyntheticKeywordToken { | 361 implements analyzer.SyntheticKeywordToken { |
| 363 /** | 362 /** |
| 364 * Initialize a newly created token to represent the given [keyword] at the | 363 * Initialize a newly created token to represent the given [keyword] at the |
| 365 * given [offset]. | 364 * given [offset]. |
| (...skipping 317 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 683 identical(value, "<=") || | 682 identical(value, "<=") || |
| 684 identical(value, "<") || | 683 identical(value, "<") || |
| 685 identical(value, "&") || | 684 identical(value, "&") || |
| 686 identical(value, "^") || | 685 identical(value, "^") || |
| 687 identical(value, "|"); | 686 identical(value, "|"); |
| 688 } | 687 } |
| 689 | 688 |
| 690 bool isTernaryOperator(String value) => identical(value, "[]="); | 689 bool isTernaryOperator(String value) => identical(value, "[]="); |
| 691 | 690 |
| 692 bool isMinusOperator(String value) => identical(value, "-"); | 691 bool isMinusOperator(String value) => identical(value, "-"); |
| OLD | NEW |