| OLD | NEW |
| 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 part of scanner; | 5 part of scanner; |
| 6 | 6 |
| 7 const int EOF_TOKEN = 0; | 7 const int EOF_TOKEN = 0; |
| 8 | 8 |
| 9 const int KEYWORD_TOKEN = $k; | 9 const int KEYWORD_TOKEN = $k; |
| 10 const int IDENTIFIER_TOKEN = $a; | 10 const int IDENTIFIER_TOKEN = $a; |
| (...skipping 367 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 378 if (!canonicalize) return s; | 378 if (!canonicalize) return s; |
| 379 var result = canonicalizedSubstrings.lookup(s); | 379 var result = canonicalizedSubstrings.lookup(s); |
| 380 if (result != null) return result; | 380 if (result != null) return result; |
| 381 canonicalizedSubstrings.add(s); | 381 canonicalizedSubstrings.add(s); |
| 382 return s; | 382 return s; |
| 383 } | 383 } |
| 384 | 384 |
| 385 static String decodeUtf8(List<int> data, int start, int end, bool asciiOnly) { | 385 static String decodeUtf8(List<int> data, int start, int end, bool asciiOnly) { |
| 386 var s; | 386 var s; |
| 387 if (asciiOnly) { | 387 if (asciiOnly) { |
| 388 // getRange returns an iterator, it does not copy the data. | 388 s = new String.fromCharCodes(data, start, end); |
| 389 s = new String.fromCharCodes(data.getRange(start, end)); | |
| 390 } else { | 389 } else { |
| 391 // TODO(lry): this is measurably slow. Also sublist is copied eagerly. | 390 s = UTF8.decoder.convert(data, start, end); |
| 392 var bytes = data.sublist(start, end); | |
| 393 s = UTF8.decode(bytes); | |
| 394 } | 391 } |
| 395 return canonicalizedString(s, true); | 392 return canonicalizedString(s, true); |
| 396 } | 393 } |
| 397 } | 394 } |
| 398 | 395 |
| 399 /** | 396 /** |
| 400 * This class represents the necessary information to compute a substring | 397 * This class represents the necessary information to compute a substring |
| 401 * lazily. The substring can either originate from a string or from | 398 * lazily. The substring can either originate from a string or from |
| 402 * a [:List<int>:] of UTF-8 bytes. | 399 * a [:List<int>:] of UTF-8 bytes. |
| 403 */ | 400 */ |
| (...skipping 318 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 722 | 719 |
| 723 const PrecedenceInfo STRING_INTERPOLATION_IDENTIFIER_INFO = | 720 const PrecedenceInfo STRING_INTERPOLATION_IDENTIFIER_INFO = |
| 724 const PrecedenceInfo('\$', 0, | 721 const PrecedenceInfo('\$', 0, |
| 725 STRING_INTERPOLATION_IDENTIFIER_TOKEN); | 722 STRING_INTERPOLATION_IDENTIFIER_TOKEN); |
| 726 | 723 |
| 727 const PrecedenceInfo HEXADECIMAL_INFO = | 724 const PrecedenceInfo HEXADECIMAL_INFO = |
| 728 const PrecedenceInfo('hexadecimal', 0, HEXADECIMAL_TOKEN); | 725 const PrecedenceInfo('hexadecimal', 0, HEXADECIMAL_TOKEN); |
| 729 | 726 |
| 730 const PrecedenceInfo COMMENT_INFO = | 727 const PrecedenceInfo COMMENT_INFO = |
| 731 const PrecedenceInfo('comment', 0, COMMENT_TOKEN); | 728 const PrecedenceInfo('comment', 0, COMMENT_TOKEN); |
| OLD | NEW |