| OLD | NEW |
| 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library fasta.scanner.token; | 5 library fasta.scanner.token; |
| 6 | 6 |
| 7 import 'keyword.dart' show | 7 import 'keyword.dart' show Keyword; |
| 8 Keyword; | |
| 9 | 8 |
| 10 import 'precedence.dart' show | 9 import 'precedence.dart' show BAD_INPUT_INFO, EOF_INFO, PrecedenceInfo; |
| 11 BAD_INPUT_INFO, | |
| 12 EOF_INFO, | |
| 13 PrecedenceInfo; | |
| 14 | 10 |
| 15 import 'token_constants.dart' show | 11 import 'token_constants.dart' show IDENTIFIER_TOKEN; |
| 16 IDENTIFIER_TOKEN; | |
| 17 | 12 |
| 18 import 'string_canonicalizer.dart'; | 13 import 'string_canonicalizer.dart'; |
| 19 | 14 |
| 20 /** | 15 /** |
| 21 * A token that doubles as a linked list. | 16 * A token that doubles as a linked list. |
| 22 */ | 17 */ |
| 23 abstract class Token { | 18 abstract class Token { |
| 24 /** | 19 /** |
| 25 * The character offset of the start of this token within the source text. | 20 * The character offset of the start of this token within the source text. |
| 26 */ | 21 */ |
| (...skipping 157 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 184 var /* String | LazySubtring */ valueOrLazySubstring; | 179 var /* String | LazySubtring */ valueOrLazySubstring; |
| 185 | 180 |
| 186 final PrecedenceInfo info; | 181 final PrecedenceInfo info; |
| 187 | 182 |
| 188 /** | 183 /** |
| 189 * Creates a non-lazy string token. If [canonicalize] is true, the string | 184 * Creates a non-lazy string token. If [canonicalize] is true, the string |
| 190 * is canonicalized before the token is created. | 185 * is canonicalized before the token is created. |
| 191 */ | 186 */ |
| 192 StringToken.fromString(this.info, String value, int charOffset, | 187 StringToken.fromString(this.info, String value, int charOffset, |
| 193 {bool canonicalize: false}) | 188 {bool canonicalize: false}) |
| 194 : valueOrLazySubstring = canonicalizedString(value, | 189 : valueOrLazySubstring = |
| 195 0, value.length, canonicalize), | 190 canonicalizedString(value, 0, value.length, canonicalize), |
| 196 super(charOffset); | 191 super(charOffset); |
| 197 | 192 |
| 198 /** | 193 /** |
| 199 * Creates a lazy string token. If [canonicalize] is true, the string | 194 * Creates a lazy string token. If [canonicalize] is true, the string |
| 200 * is canonicalized before the token is created. | 195 * is canonicalized before the token is created. |
| 201 */ | 196 */ |
| 202 StringToken.fromSubstring( | 197 StringToken.fromSubstring( |
| 203 this.info, String data, int start, int end, int charOffset, | 198 this.info, String data, int start, int end, int charOffset, |
| 204 {bool canonicalize: false}) | 199 {bool canonicalize: false}) |
| 205 : super(charOffset) { | 200 : super(charOffset) { |
| (...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 249 | 244 |
| 250 /// See [Token.stringValue] for an explanation. | 245 /// See [Token.stringValue] for an explanation. |
| 251 String get stringValue => null; | 246 String get stringValue => null; |
| 252 | 247 |
| 253 bool isIdentifier() => identical(kind, IDENTIFIER_TOKEN); | 248 bool isIdentifier() => identical(kind, IDENTIFIER_TOKEN); |
| 254 | 249 |
| 255 String toString() => "StringToken($value)"; | 250 String toString() => "StringToken($value)"; |
| 256 | 251 |
| 257 static final StringCanonicalizer canonicalizer = new StringCanonicalizer(); | 252 static final StringCanonicalizer canonicalizer = new StringCanonicalizer(); |
| 258 | 253 |
| 259 static String canonicalizedString(String s, int start, int end, | 254 static String canonicalizedString( |
| 260 bool canonicalize) { | 255 String s, int start, int end, bool canonicalize) { |
| 261 if (!canonicalize) return s; | 256 if (!canonicalize) return s; |
| 262 return canonicalizer.canonicalize(s, start, end, false); | 257 return canonicalizer.canonicalize(s, start, end, false); |
| 263 } | 258 } |
| 264 | 259 |
| 265 static String decodeUtf8(List<int> data, int start, int end, bool asciiOnly) { | 260 static String decodeUtf8(List<int> data, int start, int end, bool asciiOnly) { |
| 266 return canonicalizer.canonicalize(data, start, end, asciiOnly); | 261 return canonicalizer.canonicalize(data, start, end, asciiOnly); |
| 267 } | 262 } |
| 268 } | 263 } |
| 269 | 264 |
| 270 /** | 265 /** |
| (...skipping 85 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 356 identical(value, "<=") || | 351 identical(value, "<=") || |
| 357 identical(value, "<") || | 352 identical(value, "<") || |
| 358 identical(value, "&") || | 353 identical(value, "&") || |
| 359 identical(value, "^") || | 354 identical(value, "^") || |
| 360 identical(value, "|"); | 355 identical(value, "|"); |
| 361 } | 356 } |
| 362 | 357 |
| 363 bool isTernaryOperator(String value) => identical(value, "[]="); | 358 bool isTernaryOperator(String value) => identical(value, "[]="); |
| 364 | 359 |
| 365 bool isMinusOperator(String value) => identical(value, "-"); | 360 bool isMinusOperator(String value) => identical(value, "-"); |
| OLD | NEW |