OLD | NEW |
1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 library fasta.scanner.token; | 5 library fasta.scanner.token; |
6 | 6 |
7 import 'keyword.dart' show | 7 import 'keyword.dart' show Keyword; |
8 Keyword; | |
9 | 8 |
10 import 'precedence.dart' show | 9 import 'precedence.dart' show BAD_INPUT_INFO, EOF_INFO, PrecedenceInfo; |
11 BAD_INPUT_INFO, | |
12 EOF_INFO, | |
13 PrecedenceInfo; | |
14 | 10 |
15 import 'token_constants.dart' show | 11 import 'token_constants.dart' show IDENTIFIER_TOKEN; |
16 IDENTIFIER_TOKEN; | |
17 | 12 |
18 import 'string_canonicalizer.dart'; | 13 import 'string_canonicalizer.dart'; |
19 | 14 |
20 /** | 15 /** |
21 * A token that doubles as a linked list. | 16 * A token that doubles as a linked list. |
22 */ | 17 */ |
23 abstract class Token { | 18 abstract class Token { |
24 /** | 19 /** |
25 * The character offset of the start of this token within the source text. | 20 * The character offset of the start of this token within the source text. |
26 */ | 21 */ |
(...skipping 157 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
184 var /* String | LazySubtring */ valueOrLazySubstring; | 179 var /* String | LazySubtring */ valueOrLazySubstring; |
185 | 180 |
186 final PrecedenceInfo info; | 181 final PrecedenceInfo info; |
187 | 182 |
188 /** | 183 /** |
189 * Creates a non-lazy string token. If [canonicalize] is true, the string | 184 * Creates a non-lazy string token. If [canonicalize] is true, the string |
190 * is canonicalized before the token is created. | 185 * is canonicalized before the token is created. |
191 */ | 186 */ |
192 StringToken.fromString(this.info, String value, int charOffset, | 187 StringToken.fromString(this.info, String value, int charOffset, |
193 {bool canonicalize: false}) | 188 {bool canonicalize: false}) |
194 : valueOrLazySubstring = canonicalizedString(value, | 189 : valueOrLazySubstring = |
195 0, value.length, canonicalize), | 190 canonicalizedString(value, 0, value.length, canonicalize), |
196 super(charOffset); | 191 super(charOffset); |
197 | 192 |
198 /** | 193 /** |
199 * Creates a lazy string token. If [canonicalize] is true, the string | 194 * Creates a lazy string token. If [canonicalize] is true, the string |
200 * is canonicalized before the token is created. | 195 * is canonicalized before the token is created. |
201 */ | 196 */ |
202 StringToken.fromSubstring( | 197 StringToken.fromSubstring( |
203 this.info, String data, int start, int end, int charOffset, | 198 this.info, String data, int start, int end, int charOffset, |
204 {bool canonicalize: false}) | 199 {bool canonicalize: false}) |
205 : super(charOffset) { | 200 : super(charOffset) { |
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
249 | 244 |
250 /// See [Token.stringValue] for an explanation. | 245 /// See [Token.stringValue] for an explanation. |
251 String get stringValue => null; | 246 String get stringValue => null; |
252 | 247 |
253 bool isIdentifier() => identical(kind, IDENTIFIER_TOKEN); | 248 bool isIdentifier() => identical(kind, IDENTIFIER_TOKEN); |
254 | 249 |
255 String toString() => "StringToken($value)"; | 250 String toString() => "StringToken($value)"; |
256 | 251 |
257 static final StringCanonicalizer canonicalizer = new StringCanonicalizer(); | 252 static final StringCanonicalizer canonicalizer = new StringCanonicalizer(); |
258 | 253 |
259 static String canonicalizedString(String s, int start, int end, | 254 static String canonicalizedString( |
260 bool canonicalize) { | 255 String s, int start, int end, bool canonicalize) { |
261 if (!canonicalize) return s; | 256 if (!canonicalize) return s; |
262 return canonicalizer.canonicalize(s, start, end, false); | 257 return canonicalizer.canonicalize(s, start, end, false); |
263 } | 258 } |
264 | 259 |
265 static String decodeUtf8(List<int> data, int start, int end, bool asciiOnly) { | 260 static String decodeUtf8(List<int> data, int start, int end, bool asciiOnly) { |
266 return canonicalizer.canonicalize(data, start, end, asciiOnly); | 261 return canonicalizer.canonicalize(data, start, end, asciiOnly); |
267 } | 262 } |
268 } | 263 } |
269 | 264 |
270 /** | 265 /** |
(...skipping 85 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
356 identical(value, "<=") || | 351 identical(value, "<=") || |
357 identical(value, "<") || | 352 identical(value, "<") || |
358 identical(value, "&") || | 353 identical(value, "&") || |
359 identical(value, "^") || | 354 identical(value, "^") || |
360 identical(value, "|"); | 355 identical(value, "|"); |
361 } | 356 } |
362 | 357 |
363 bool isTernaryOperator(String value) => identical(value, "[]="); | 358 bool isTernaryOperator(String value) => identical(value, "[]="); |
364 | 359 |
365 bool isMinusOperator(String value) => identical(value, "-"); | 360 bool isMinusOperator(String value) => identical(value, "-"); |
OLD | NEW |