OLD | NEW |
1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 part of scanner; | 5 library dart2js.scanner.utf8; |
| 6 |
| 7 import 'dart:convert' show |
| 8 UNICODE_BOM_CHARACTER_RUNE, |
| 9 UTF8; |
| 10 |
| 11 import '../io/source_file.dart' show |
| 12 SourceFile; |
| 13 |
| 14 import 'array_based_scanner.dart' show |
| 15 ArrayBasedScanner; |
| 16 import 'token.dart' show |
| 17 PrecedenceInfo, |
| 18 StringToken, |
| 19 Token; |
6 | 20 |
7 /** | 21 /** |
8 * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens | 22 * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens |
9 * that points to substrings. | 23 * that points to substrings. |
10 */ | 24 */ |
11 class Utf8BytesScanner extends ArrayBasedScanner { | 25 class Utf8BytesScanner extends ArrayBasedScanner { |
12 /** | 26 /** |
13 * The file content. | 27 * The file content. |
14 * | 28 * |
15 * The content is zero-terminated. | 29 * The content is zero-terminated. |
(...skipping 182 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
198 | 212 |
199 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, | 213 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, |
200 [int extraOffset = 0]) { | 214 [int extraOffset = 0]) { |
201 tail.next = new StringToken.fromUtf8Bytes( | 215 tail.next = new StringToken.fromUtf8Bytes( |
202 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); | 216 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); |
203 tail = tail.next; | 217 tail = tail.next; |
204 } | 218 } |
205 | 219 |
206 bool atEndOfFile() => byteOffset >= bytes.length - 1; | 220 bool atEndOfFile() => byteOffset >= bytes.length - 1; |
207 } | 221 } |
OLD | NEW |