| OLD | NEW |
| 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library dart2js.tokens; | 5 library dart2js.tokens; |
| 6 | 6 |
| 7 import 'dart:convert' show | 7 import 'dart:convert' show UTF8; |
| 8 UTF8; | 8 import 'dart:collection' show HashSet; |
| 9 import 'dart:collection' show | |
| 10 HashSet; | |
| 11 | 9 |
| 12 import '../common.dart'; | 10 import '../common.dart'; |
| 13 import '../util/util.dart' show | 11 import '../util/util.dart' show computeHashCode; |
| 14 computeHashCode; | |
| 15 | 12 |
| 16 import 'keyword.dart' show | 13 import 'keyword.dart' show Keyword; |
| 17 Keyword; | 14 import 'precedence.dart' show PrecedenceInfo; |
| 18 import 'precedence.dart' show | 15 import 'precedence_constants.dart' as Precedence show BAD_INPUT_INFO; |
| 19 PrecedenceInfo; | 16 import 'token_constants.dart' as Tokens show IDENTIFIER_TOKEN; |
| 20 import 'precedence_constants.dart' as Precedence show | |
| 21 BAD_INPUT_INFO; | |
| 22 import 'token_constants.dart' as Tokens show | |
| 23 IDENTIFIER_TOKEN; | |
| 24 | 17 |
| 25 /** | 18 /** |
| 26 * A token that doubles as a linked list. | 19 * A token that doubles as a linked list. |
| 27 */ | 20 */ |
| 28 abstract class Token implements Spannable { | 21 abstract class Token implements Spannable { |
| 29 /** | 22 /** |
| 30 * The character offset of the start of this token within the source text. | 23 * The character offset of the start of this token within the source text. |
| 31 */ | 24 */ |
| 32 final int charOffset; | 25 final int charOffset; |
| 33 | 26 |
| (...skipping 91 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 125 final Token end; | 118 final Token end; |
| 126 | 119 |
| 127 TokenPair(this.begin, this.end); | 120 TokenPair(this.begin, this.end); |
| 128 } | 121 } |
| 129 | 122 |
| 130 /** | 123 /** |
| 131 * A [SymbolToken] represents the symbol in its precendence info. | 124 * A [SymbolToken] represents the symbol in its precendence info. |
| 132 * Also used for end of file with EOF_INFO. | 125 * Also used for end of file with EOF_INFO. |
| 133 */ | 126 */ |
| 134 class SymbolToken extends Token { | 127 class SymbolToken extends Token { |
| 135 | |
| 136 final PrecedenceInfo info; | 128 final PrecedenceInfo info; |
| 137 | 129 |
| 138 SymbolToken(this.info, int charOffset) : super(charOffset); | 130 SymbolToken(this.info, int charOffset) : super(charOffset); |
| 139 | 131 |
| 140 String get value => info.value; | 132 String get value => info.value; |
| 141 | 133 |
| 142 String get stringValue => info.value; | 134 String get stringValue => info.value; |
| 143 | 135 |
| 144 bool isIdentifier() => false; | 136 bool isIdentifier() => false; |
| 145 | 137 |
| (...skipping 26 matching lines...) Expand all Loading... |
| 172 String get value => keyword.syntax; | 164 String get value => keyword.syntax; |
| 173 | 165 |
| 174 String get stringValue => keyword.syntax; | 166 String get stringValue => keyword.syntax; |
| 175 | 167 |
| 176 bool isIdentifier() => keyword.isPseudo || keyword.isBuiltIn; | 168 bool isIdentifier() => keyword.isPseudo || keyword.isBuiltIn; |
| 177 | 169 |
| 178 String toString() => "KeywordToken($value)"; | 170 String toString() => "KeywordToken($value)"; |
| 179 } | 171 } |
| 180 | 172 |
| 181 abstract class ErrorToken extends Token { | 173 abstract class ErrorToken extends Token { |
| 182 ErrorToken(int charOffset) | 174 ErrorToken(int charOffset) : super(charOffset); |
| 183 : super(charOffset); | |
| 184 | 175 |
| 185 PrecedenceInfo get info => Precedence.BAD_INPUT_INFO; | 176 PrecedenceInfo get info => Precedence.BAD_INPUT_INFO; |
| 186 | 177 |
| 187 String get value { | 178 String get value { |
| 188 throw new SpannableAssertionFailure(this, assertionMessage); | 179 throw new SpannableAssertionFailure(this, assertionMessage); |
| 189 } | 180 } |
| 190 | 181 |
| 191 String get stringValue => null; | 182 String get stringValue => null; |
| 192 | 183 |
| 193 bool isIdentifier() => false; | 184 bool isIdentifier() => false; |
| 194 | 185 |
| 195 String get assertionMessage; | 186 String get assertionMessage; |
| 196 } | 187 } |
| 197 | 188 |
| 198 class BadInputToken extends ErrorToken { | 189 class BadInputToken extends ErrorToken { |
| 199 final int character; | 190 final int character; |
| 200 | 191 |
| 201 BadInputToken(this.character, int charOffset) | 192 BadInputToken(this.character, int charOffset) : super(charOffset); |
| 202 : super(charOffset); | |
| 203 | 193 |
| 204 String toString() => "BadInputToken($character)"; | 194 String toString() => "BadInputToken($character)"; |
| 205 | 195 |
| 206 String get assertionMessage { | 196 String get assertionMessage { |
| 207 return 'Character U+${character.toRadixString(16)} not allowed here.'; | 197 return 'Character U+${character.toRadixString(16)} not allowed here.'; |
| 208 } | 198 } |
| 209 } | 199 } |
| 210 | 200 |
| 211 class UnterminatedToken extends ErrorToken { | 201 class UnterminatedToken extends ErrorToken { |
| 212 final String start; | 202 final String start; |
| (...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 252 | 242 |
| 253 var /* String | LazySubtring */ valueOrLazySubstring; | 243 var /* String | LazySubtring */ valueOrLazySubstring; |
| 254 | 244 |
| 255 final PrecedenceInfo info; | 245 final PrecedenceInfo info; |
| 256 | 246 |
| 257 /** | 247 /** |
| 258 * Creates a non-lazy string token. If [canonicalize] is true, the string | 248 * Creates a non-lazy string token. If [canonicalize] is true, the string |
| 259 * is canonicalized before the token is created. | 249 * is canonicalized before the token is created. |
| 260 */ | 250 */ |
| 261 StringToken.fromString(this.info, String value, int charOffset, | 251 StringToken.fromString(this.info, String value, int charOffset, |
| 262 {bool canonicalize : false}) | 252 {bool canonicalize: false}) |
| 263 : valueOrLazySubstring = canonicalizedString(value, canonicalize), | 253 : valueOrLazySubstring = canonicalizedString(value, canonicalize), |
| 264 super(charOffset); | 254 super(charOffset); |
| 265 | 255 |
| 266 /** | 256 /** |
| 267 * Creates a lazy string token. If [canonicalize] is true, the string | 257 * Creates a lazy string token. If [canonicalize] is true, the string |
| 268 * is canonicalized before the token is created. | 258 * is canonicalized before the token is created. |
| 269 */ | 259 */ |
| 270 StringToken.fromSubstring(this.info, String data, int start, int end, | 260 StringToken.fromSubstring( |
| 271 int charOffset, {bool canonicalize : false}) | 261 this.info, String data, int start, int end, int charOffset, |
| 262 {bool canonicalize: false}) |
| 272 : super(charOffset) { | 263 : super(charOffset) { |
| 273 int length = end - start; | 264 int length = end - start; |
| 274 if (length <= LAZY_THRESHOLD) { | 265 if (length <= LAZY_THRESHOLD) { |
| 275 valueOrLazySubstring = canonicalizedString(data.substring(start, end), | 266 valueOrLazySubstring = |
| 276 canonicalize); | 267 canonicalizedString(data.substring(start, end), canonicalize); |
| 277 } else { | 268 } else { |
| 278 valueOrLazySubstring = | 269 valueOrLazySubstring = |
| 279 new LazySubstring(data, start, length, canonicalize); | 270 new LazySubstring(data, start, length, canonicalize); |
| 280 } | 271 } |
| 281 } | 272 } |
| 282 | 273 |
| 283 /** | 274 /** |
| 284 * Creates a lazy string token. If [asciiOnly] is false, the byte array | 275 * Creates a lazy string token. If [asciiOnly] is false, the byte array |
| 285 * is passed through a UTF-8 decoder. | 276 * is passed through a UTF-8 decoder. |
| 286 */ | 277 */ |
| 287 StringToken.fromUtf8Bytes(this.info, List<int> data, int start, int end, | 278 StringToken.fromUtf8Bytes(this.info, List<int> data, int start, int end, |
| 288 bool asciiOnly, int charOffset) | 279 bool asciiOnly, int charOffset) |
| 289 : super(charOffset) { | 280 : super(charOffset) { |
| 290 int length = end - start; | 281 int length = end - start; |
| 291 if (length <= LAZY_THRESHOLD) { | 282 if (length <= LAZY_THRESHOLD) { |
| 292 valueOrLazySubstring = decodeUtf8(data, start, end, asciiOnly); | 283 valueOrLazySubstring = decodeUtf8(data, start, end, asciiOnly); |
| 293 } else { | 284 } else { |
| 294 valueOrLazySubstring = new LazySubstring(data, start, length, asciiOnly); | 285 valueOrLazySubstring = new LazySubstring(data, start, length, asciiOnly); |
| 295 } | 286 } |
| 296 } | 287 } |
| 297 | 288 |
| 298 String get value { | 289 String get value { |
| 299 if (valueOrLazySubstring is String) { | 290 if (valueOrLazySubstring is String) { |
| 300 return valueOrLazySubstring; | 291 return valueOrLazySubstring; |
| 301 } else { | 292 } else { |
| 302 assert(valueOrLazySubstring is LazySubstring); | 293 assert(valueOrLazySubstring is LazySubstring); |
| 303 var data = valueOrLazySubstring.data; | 294 var data = valueOrLazySubstring.data; |
| 304 int start = valueOrLazySubstring.start; | 295 int start = valueOrLazySubstring.start; |
| 305 int end = start + valueOrLazySubstring.length; | 296 int end = start + valueOrLazySubstring.length; |
| 306 if (data is String) { | 297 if (data is String) { |
| 307 valueOrLazySubstring = canonicalizedString( | 298 valueOrLazySubstring = canonicalizedString( |
| 308 data.substring(start, end), valueOrLazySubstring.boolValue); | 299 data.substring(start, end), valueOrLazySubstring.boolValue); |
| 309 } else { | 300 } else { |
| 310 valueOrLazySubstring = decodeUtf8( | 301 valueOrLazySubstring = |
| 311 data, start, end, valueOrLazySubstring.boolValue); | 302 decodeUtf8(data, start, end, valueOrLazySubstring.boolValue); |
| 312 } | 303 } |
| 313 return valueOrLazySubstring; | 304 return valueOrLazySubstring; |
| 314 } | 305 } |
| 315 } | 306 } |
| 316 | 307 |
| 317 /// See [Token.stringValue] for an explanation. | 308 /// See [Token.stringValue] for an explanation. |
| 318 String get stringValue => null; | 309 String get stringValue => null; |
| 319 | 310 |
| 320 bool isIdentifier() => identical(kind, Tokens.IDENTIFIER_TOKEN); | 311 bool isIdentifier() => identical(kind, Tokens.IDENTIFIER_TOKEN); |
| 321 | 312 |
| 322 String toString() => "StringToken($value)"; | 313 String toString() => "StringToken($value)"; |
| 323 | 314 |
| 324 static final HashSet<String> canonicalizedSubstrings = | 315 static final HashSet<String> canonicalizedSubstrings = new HashSet<String>(); |
| 325 new HashSet<String>(); | |
| 326 | 316 |
| 327 static String canonicalizedString(String s, bool canonicalize) { | 317 static String canonicalizedString(String s, bool canonicalize) { |
| 328 if (!canonicalize) return s; | 318 if (!canonicalize) return s; |
| 329 var result = canonicalizedSubstrings.lookup(s); | 319 var result = canonicalizedSubstrings.lookup(s); |
| 330 if (result != null) return result; | 320 if (result != null) return result; |
| 331 canonicalizedSubstrings.add(s); | 321 canonicalizedSubstrings.add(s); |
| 332 return s; | 322 return s; |
| 333 } | 323 } |
| 334 | 324 |
| 335 static String decodeUtf8(List<int> data, int start, int end, bool asciiOnly) { | 325 static String decodeUtf8(List<int> data, int start, int end, bool asciiOnly) { |
| (...skipping 66 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 402 class FullLazySubstring extends LazySubstring { | 392 class FullLazySubstring extends LazySubstring { |
| 403 final data; | 393 final data; |
| 404 final int start; | 394 final int start; |
| 405 final int length; | 395 final int length; |
| 406 final bool boolValue; | 396 final bool boolValue; |
| 407 FullLazySubstring(this.data, this.start, this.length, this.boolValue) | 397 FullLazySubstring(this.data, this.start, this.length, this.boolValue) |
| 408 : super.internal(); | 398 : super.internal(); |
| 409 } | 399 } |
| 410 | 400 |
| 411 bool isUserDefinableOperator(String value) { | 401 bool isUserDefinableOperator(String value) { |
| 412 return | 402 return isBinaryOperator(value) || |
| 413 isBinaryOperator(value) || | |
| 414 isMinusOperator(value) || | 403 isMinusOperator(value) || |
| 415 isTernaryOperator(value) || | 404 isTernaryOperator(value) || |
| 416 isUnaryOperator(value); | 405 isUnaryOperator(value); |
| 417 } | 406 } |
| 418 | 407 |
| 419 bool isUnaryOperator(String value) => value == '~'; | 408 bool isUnaryOperator(String value) => value == '~'; |
| 420 | 409 |
| 421 bool isBinaryOperator(String value) { | 410 bool isBinaryOperator(String value) { |
| 422 return | 411 return value == '==' || |
| 423 value == '==' || | |
| 424 value == '[]' || | 412 value == '[]' || |
| 425 value == '*' || | 413 value == '*' || |
| 426 value == '/' || | 414 value == '/' || |
| 427 value == '%' || | 415 value == '%' || |
| 428 value == '~/' || | 416 value == '~/' || |
| 429 value == '+' || | 417 value == '+' || |
| 430 value == '<<' || | 418 value == '<<' || |
| 431 value == '>>' || | 419 value == '>>' || |
| 432 value == '>=' || | 420 value == '>=' || |
| 433 value == '>' || | 421 value == '>' || |
| 434 value == '<=' || | 422 value == '<=' || |
| 435 value == '<' || | 423 value == '<' || |
| 436 value == '&' || | 424 value == '&' || |
| 437 value == '^' || | 425 value == '^' || |
| 438 value == '|'; | 426 value == '|'; |
| 439 } | 427 } |
| 440 | 428 |
| 441 bool isTernaryOperator(String value) => value == '[]='; | 429 bool isTernaryOperator(String value) => value == '[]='; |
| 442 | 430 |
| 443 bool isMinusOperator(String value) => value == '-'; | 431 bool isMinusOperator(String value) => value == '-'; |
| OLD | NEW |