OLD | NEW |
1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 library dart2js.scanner.utf8; | 5 library dart2js.scanner.utf8; |
6 | 6 |
7 import 'dart:convert' show | 7 import 'dart:convert' show |
8 UNICODE_BOM_CHARACTER_RUNE, | 8 UNICODE_BOM_CHARACTER_RUNE, |
9 UTF8; | 9 UTF8; |
10 | 10 |
11 import '../io/source_file.dart' show | 11 import '../io/source_file.dart' show |
12 SourceFile; | 12 SourceFile; |
| 13 import '../tokens/precedence.dart' show |
| 14 PrecedenceInfo; |
13 import '../tokens/token.dart' show | 15 import '../tokens/token.dart' show |
14 PrecedenceInfo, | |
15 StringToken, | 16 StringToken, |
16 Token; | 17 Token; |
17 | 18 |
18 import 'array_based_scanner.dart' show | 19 import 'array_based_scanner.dart' show |
19 ArrayBasedScanner; | 20 ArrayBasedScanner; |
20 | 21 |
21 /** | 22 /** |
22 * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens | 23 * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens |
23 * that points to substrings. | 24 * that points to substrings. |
24 */ | 25 */ |
(...skipping 187 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
212 | 213 |
213 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, | 214 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, |
214 [int extraOffset = 0]) { | 215 [int extraOffset = 0]) { |
215 tail.next = new StringToken.fromUtf8Bytes( | 216 tail.next = new StringToken.fromUtf8Bytes( |
216 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); | 217 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); |
217 tail = tail.next; | 218 tail = tail.next; |
218 } | 219 } |
219 | 220 |
220 bool atEndOfFile() => byteOffset >= bytes.length - 1; | 221 bool atEndOfFile() => byteOffset >= bytes.length - 1; |
221 } | 222 } |
OLD | NEW |