Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(85)

Side by Side Diff: pkg/front_end/lib/src/fasta/scanner/token.dart

Issue 2832403002: merge PrecedenceInfo into TokenType and update all refs (Closed)
Patch Set: rebase Created 3 years, 8 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 library fasta.scanner.token; 5 library fasta.scanner.token;
6 6
7 import '../../scanner/token.dart' as analyzer; 7 import '../../scanner/token.dart' as analyzer;
8 import '../../scanner/token.dart' show TokenType; 8 import '../../scanner/token.dart' show TokenType;
9 9
10 import 'precedence.dart'
11 show AS_INFO, BAD_INPUT_INFO, EOF_INFO, IS_INFO, KEYWORD_INFO;
12
13 import 'token_constants.dart' show IDENTIFIER_TOKEN; 10 import 'token_constants.dart' show IDENTIFIER_TOKEN;
14 11
15 import 'string_canonicalizer.dart'; 12 import 'string_canonicalizer.dart';
16 13
17 /** 14 /**
18 * A token that doubles as a linked list. 15 * A token that doubles as a linked list.
19 */ 16 */
20 abstract class Token implements analyzer.TokenWithComment { 17 abstract class Token implements analyzer.TokenWithComment {
21 /** 18 /**
22 * The character offset of the start of this token within the source text. 19 * The character offset of the start of this token within the source text.
(...skipping 92 matching lines...) Expand 10 before | Expand all | Expand 10 after
115 * token 'foo'. 112 * token 'foo'.
116 * 113 *
117 * Use [lexeme] for the text actually parsed by the token. 114 * Use [lexeme] for the text actually parsed by the token.
118 */ 115 */
119 String toString(); 116 String toString();
120 117
121 /** 118 /**
122 * The number of characters parsed by this token. 119 * The number of characters parsed by this token.
123 */ 120 */
124 int get charCount { 121 int get charCount {
125 if (info == BAD_INPUT_INFO) { 122 if (info == analyzer.TokenType.BAD_INPUT) {
126 // This is a token that wraps around an error message. Return 1 123 // This is a token that wraps around an error message. Return 1
127 // instead of the size of the length of the error message. 124 // instead of the size of the length of the error message.
128 return 1; 125 return 1;
129 } else { 126 } else {
130 return lexeme.length; 127 return lexeme.length;
131 } 128 }
132 } 129 }
133 130
134 /// The character offset of the end of this token within the source text. 131 /// The character offset of the end of this token within the source text.
135 int get charEnd => charOffset + charCount; 132 int get charEnd => charOffset + charCount;
136 133
137 bool get isEof => false; 134 bool get isEof => false;
138 135
139 bool get isBuiltInIdentifier => false; 136 bool get isBuiltInIdentifier => false;
140 137
141 @override 138 @override
142 bool get isOperator => info.isOperator; 139 bool get isOperator => info.isOperator;
143 140
144 @override 141 @override
145 bool get isUserDefinableOperator => info.isUserDefinableOperator; 142 bool get isUserDefinableOperator => info.isUserDefinableOperator;
146 143
147 @override 144 @override
148 analyzer.TokenType get type { 145 analyzer.TokenType get type {
149 // Analyzer has a different concept of what is a Keyword type. 146 // Analyzer has a different concept of what is a Keyword type.
150 return info == AS_INFO || info == IS_INFO ? KEYWORD_INFO : info; 147 return info == TokenType.AS || info == TokenType.IS
148 ? TokenType.KEYWORD
149 : info;
151 } 150 }
152 151
153 @override 152 @override
154 int get offset => charOffset; 153 int get offset => charOffset;
155 154
156 @override 155 @override
157 set offset(int newOffset) { 156 set offset(int newOffset) {
158 charOffset = newOffset; 157 charOffset = newOffset;
159 } 158 }
160 159
(...skipping 81 matching lines...) Expand 10 before | Expand all | Expand 10 after
242 /** 241 /**
243 * A [SymbolToken] represents the symbol in its precedence info. 242 * A [SymbolToken] represents the symbol in its precedence info.
244 * Also used for end of file with EOF_INFO. 243 * Also used for end of file with EOF_INFO.
245 */ 244 */
246 class SymbolToken extends Token { 245 class SymbolToken extends Token {
247 final TokenType info; 246 final TokenType info;
248 247
249 SymbolToken(this.info, int charOffset) : super(charOffset); 248 SymbolToken(this.info, int charOffset) : super(charOffset);
250 249
251 factory SymbolToken.eof(int charOffset) { 250 factory SymbolToken.eof(int charOffset) {
252 var eof = new SyntheticSymbolToken(EOF_INFO, charOffset); 251 var eof = new SyntheticSymbolToken(analyzer.TokenType.EOF, charOffset);
253 // EOF points to itself so there's always infinite look-ahead. 252 // EOF points to itself so there's always infinite look-ahead.
254 eof.previousToken = eof; 253 eof.previousToken = eof;
255 eof.next = eof; 254 eof.next = eof;
256 return eof; 255 return eof;
257 } 256 }
258 257
259 @override 258 @override
260 String get lexeme => info.value; 259 String get lexeme => info.value;
261 260
262 @override 261 @override
263 String get stringValue => info.value; 262 String get stringValue => info.value;
264 263
265 @override 264 @override
266 bool isIdentifier() => false; 265 bool isIdentifier() => false;
267 266
268 @override 267 @override
269 String toString() => "SymbolToken(${isEof ? '-eof-' : lexeme})"; 268 String toString() => "SymbolToken(${isEof ? '-eof-' : lexeme})";
270 269
271 @override 270 @override
272 bool get isEof => info == EOF_INFO; 271 bool get isEof => info == analyzer.TokenType.EOF;
273 272
274 @override 273 @override
275 Token copyWithoutComments() => isEof 274 Token copyWithoutComments() => isEof
276 ? new SymbolToken.eof(charOffset) 275 ? new SymbolToken.eof(charOffset)
277 : new SymbolToken(info, charOffset); 276 : new SymbolToken(info, charOffset);
278 } 277 }
279 278
280 /** 279 /**
281 * A [SyntheticSymbolToken] represents the symbol in its precedence info 280 * A [SyntheticSymbolToken] represents the symbol in its precedence info
282 * which does not exist in the original source. 281 * which does not exist in the original source.
(...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after
345 344
346 String toString() => "KeywordToken($lexeme)"; 345 String toString() => "KeywordToken($lexeme)";
347 346
348 @override 347 @override
349 Token copyWithoutComments() => new KeywordToken(keyword, charOffset); 348 Token copyWithoutComments() => new KeywordToken(keyword, charOffset);
350 349
351 @override 350 @override
352 analyzer.Keyword value() => keyword; 351 analyzer.Keyword value() => keyword;
353 352
354 @override 353 @override
355 analyzer.TokenType get type => KEYWORD_INFO; 354 analyzer.TokenType get type => TokenType.KEYWORD;
356 } 355 }
357 356
358 /** 357 /**
359 * A synthetic keyword token. 358 * A synthetic keyword token.
360 */ 359 */
361 class SyntheticKeywordToken extends KeywordToken 360 class SyntheticKeywordToken extends KeywordToken
362 implements analyzer.SyntheticKeywordToken { 361 implements analyzer.SyntheticKeywordToken {
363 /** 362 /**
364 * Initialize a newly created token to represent the given [keyword] at the 363 * Initialize a newly created token to represent the given [keyword] at the
365 * given [offset]. 364 * given [offset].
(...skipping 317 matching lines...) Expand 10 before | Expand all | Expand 10 after
683 identical(value, "<=") || 682 identical(value, "<=") ||
684 identical(value, "<") || 683 identical(value, "<") ||
685 identical(value, "&") || 684 identical(value, "&") ||
686 identical(value, "^") || 685 identical(value, "^") ||
687 identical(value, "|"); 686 identical(value, "|");
688 } 687 }
689 688
690 bool isTernaryOperator(String value) => identical(value, "[]="); 689 bool isTernaryOperator(String value) => identical(value, "[]=");
691 690
692 bool isMinusOperator(String value) => identical(value, "-"); 691 bool isMinusOperator(String value) => identical(value, "-");
OLDNEW
« no previous file with comments | « pkg/front_end/lib/src/fasta/scanner/recover.dart ('k') | pkg/front_end/lib/src/scanner/token.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698