Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(273)

Side by Side Diff: pkg/front_end/lib/src/fasta/scanner/token.dart

Issue 3010533002: flatten token hierarchy (Closed)
Patch Set: Created 3 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 library fasta.scanner.token; 5 library fasta.scanner.token;
6 6
7 import '../../scanner/token.dart' as analyzer; 7 import '../../scanner/token.dart' as analyzer;
8 import '../../scanner/token.dart' show Token, TokenType; 8 import '../../scanner/token.dart' show Token, TokenType;
9 9
10 import 'token_constants.dart' show IDENTIFIER_TOKEN; 10 import 'token_constants.dart' show IDENTIFIER_TOKEN;
11 11
12 import 'string_canonicalizer.dart'; 12 import 'string_canonicalizer.dart';
13 13
14 /** 14 /**
15 * A String-valued token. Represents identifiers, string literals, 15 * A String-valued token. Represents identifiers, string literals,
16 * number literals, comments, and error tokens, using the corresponding 16 * number literals, comments, and error tokens, using the corresponding
17 * precedence info. 17 * precedence info.
18 */ 18 */
19 class StringToken extends analyzer.TokenWithComment 19 class StringToken extends analyzer.SimpleToken implements analyzer.StringToken {
20 implements analyzer.StringTokenWithComment {
21 /** 20 /**
22 * The length threshold above which substring tokens are computed lazily. 21 * The length threshold above which substring tokens are computed lazily.
23 * 22 *
24 * For string tokens that are substrings of the program source, the actual 23 * For string tokens that are substrings of the program source, the actual
25 * substring extraction is performed lazily. This is beneficial because 24 * substring extraction is performed lazily. This is beneficial because
26 * not all scanned code are actually used. For unused parts, the substrings 25 * not all scanned code are actually used. For unused parts, the substrings
27 * are never computed and allocated. 26 * are never computed and allocated.
28 */ 27 */
29 static const int LAZY_THRESHOLD = 4; 28 static const int LAZY_THRESHOLD = 4;
30 29
(...skipping 105 matching lines...) Expand 10 before | Expand all | Expand 10 after
136 @override 135 @override
137 int get length => 0; 136 int get length => 0;
138 137
139 @override 138 @override
140 Token copy() => new SyntheticStringToken( 139 Token copy() => new SyntheticStringToken(
141 type, valueOrLazySubstring, offset, copyComments(precedingComments)); 140 type, valueOrLazySubstring, offset, copyComments(precedingComments));
142 } 141 }
143 142
144 class CommentToken extends StringToken implements analyzer.CommentToken { 143 class CommentToken extends StringToken implements analyzer.CommentToken {
145 @override 144 @override
146 analyzer.TokenWithComment parent; 145 analyzer.SimpleToken parent;
147 146
148 /** 147 /**
149 * Creates a lazy comment token. If [canonicalize] is true, the string 148 * Creates a lazy comment token. If [canonicalize] is true, the string
150 * is canonicalized before the token is created. 149 * is canonicalized before the token is created.
151 */ 150 */
152 CommentToken.fromSubstring( 151 CommentToken.fromSubstring(
153 TokenType type, String data, int start, int end, int charOffset, 152 TokenType type, String data, int start, int end, int charOffset,
154 {bool canonicalize: false}) 153 {bool canonicalize: false})
155 : super.fromSubstring(type, data, start, end, charOffset, 154 : super.fromSubstring(type, data, start, end, charOffset,
156 canonicalize: canonicalize); 155 canonicalize: canonicalize);
(...skipping 155 matching lines...) Expand 10 before | Expand all | Expand 10 after
312 identical(value, "<=") || 311 identical(value, "<=") ||
313 identical(value, "<") || 312 identical(value, "<") ||
314 identical(value, "&") || 313 identical(value, "&") ||
315 identical(value, "^") || 314 identical(value, "^") ||
316 identical(value, "|"); 315 identical(value, "|");
317 } 316 }
318 317
319 bool isTernaryOperator(String value) => identical(value, "[]="); 318 bool isTernaryOperator(String value) => identical(value, "[]=");
320 319
321 bool isMinusOperator(String value) => identical(value, "-"); 320 bool isMinusOperator(String value) => identical(value, "-");
OLDNEW
« no previous file with comments | « pkg/front_end/lib/src/fasta/scanner/error_token.dart ('k') | pkg/front_end/lib/src/scanner/scanner.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698