| OLD | NEW |
| 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library dart2js.scanner.utf8; | 5 library dart2js.scanner.utf8; |
| 6 | 6 |
| 7 import 'dart:convert' show UNICODE_BOM_CHARACTER_RUNE, UTF8; | 7 import 'dart:convert' show UNICODE_BOM_CHARACTER_RUNE, UTF8; |
| 8 | 8 |
| 9 import '../io/source_file.dart' show SourceFile; | 9 import '../io/source_file.dart' show SourceFile; |
| 10 import '../tokens/precedence.dart' show PrecedenceInfo; | 10 import '../tokens/precedence.dart' show PrecedenceInfo; |
| 11 import '../tokens/token.dart' show StringToken, Token; | 11 import '../tokens/token.dart' show StringToken, Token; |
| 12 | |
| 13 import 'array_based_scanner.dart' show ArrayBasedScanner; | 12 import 'array_based_scanner.dart' show ArrayBasedScanner; |
| 14 | 13 |
| 15 /** | 14 /** |
| 16 * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens | 15 * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens |
| 17 * that points to substrings. | 16 * that points to substrings. |
| 18 */ | 17 */ |
| 19 class Utf8BytesScanner extends ArrayBasedScanner { | 18 class Utf8BytesScanner extends ArrayBasedScanner { |
| 20 /** | 19 /** |
| 21 * The file content. | 20 * The file content. |
| 22 * | 21 * |
| (...skipping 183 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 206 | 205 |
| 207 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, | 206 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, |
| 208 [int extraOffset = 0]) { | 207 [int extraOffset = 0]) { |
| 209 tail.next = new StringToken.fromUtf8Bytes( | 208 tail.next = new StringToken.fromUtf8Bytes( |
| 210 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); | 209 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); |
| 211 tail = tail.next; | 210 tail = tail.next; |
| 212 } | 211 } |
| 213 | 212 |
| 214 bool atEndOfFile() => byteOffset >= bytes.length - 1; | 213 bool atEndOfFile() => byteOffset >= bytes.length - 1; |
| 215 } | 214 } |
| OLD | NEW |