OLD | NEW |
1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 library dart2js.scanner.utf8; | 5 library dart2js.scanner.utf8; |
6 | 6 |
7 import 'dart:convert' show | 7 import 'dart:convert' show UNICODE_BOM_CHARACTER_RUNE, UTF8; |
8 UNICODE_BOM_CHARACTER_RUNE, | |
9 UTF8; | |
10 | 8 |
11 import '../io/source_file.dart' show | 9 import '../io/source_file.dart' show SourceFile; |
12 SourceFile; | 10 import '../tokens/precedence.dart' show PrecedenceInfo; |
13 import '../tokens/precedence.dart' show | 11 import '../tokens/token.dart' show StringToken, Token; |
14 PrecedenceInfo; | |
15 import '../tokens/token.dart' show | |
16 StringToken, | |
17 Token; | |
18 | 12 |
19 import 'array_based_scanner.dart' show | 13 import 'array_based_scanner.dart' show ArrayBasedScanner; |
20 ArrayBasedScanner; | |
21 | 14 |
22 /** | 15 /** |
23 * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens | 16 * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens |
24 * that points to substrings. | 17 * that points to substrings. |
25 */ | 18 */ |
26 class Utf8BytesScanner extends ArrayBasedScanner { | 19 class Utf8BytesScanner extends ArrayBasedScanner { |
27 /** | 20 /** |
28 * The file content. | 21 * The file content. |
29 * | 22 * |
30 * The content is zero-terminated. | 23 * The content is zero-terminated. |
(...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
93 } | 86 } |
94 | 87 |
95 /** | 88 /** |
96 * Creates a new Utf8BytesScanner from a list of UTF-8 bytes. | 89 * Creates a new Utf8BytesScanner from a list of UTF-8 bytes. |
97 * | 90 * |
98 * The last element of the list is expected to be '0' to signal the end of | 91 * The last element of the list is expected to be '0' to signal the end of |
99 * the file. If this is not the case, the entire array is copied before | 92 * the file. If this is not the case, the entire array is copied before |
100 * scanning. | 93 * scanning. |
101 */ | 94 */ |
102 Utf8BytesScanner.fromBytes(List<int> zeroTerminatedBytes, | 95 Utf8BytesScanner.fromBytes(List<int> zeroTerminatedBytes, |
103 {bool includeComments: false}) | 96 {bool includeComments: false}) |
104 : this.bytes = zeroTerminatedBytes, | 97 : this.bytes = zeroTerminatedBytes, |
105 super(null, includeComments) { | 98 super(null, includeComments) { |
106 assert(bytes.last == 0); | 99 assert(bytes.last == 0); |
107 } | 100 } |
108 | 101 |
109 bool _containsBomAt(int offset) { | 102 bool _containsBomAt(int offset) { |
110 const BOM_UTF8 = const [0xEF, 0xBB, 0xBF]; | 103 const BOM_UTF8 = const [0xEF, 0xBB, 0xBF]; |
111 | 104 |
112 return offset + 3 < bytes.length && | 105 return offset + 3 < bytes.length && |
113 bytes[offset] == BOM_UTF8[0] && | 106 bytes[offset] == BOM_UTF8[0] && |
(...skipping 91 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
205 return byteOffset - utf8Slack - 1; | 198 return byteOffset - utf8Slack - 1; |
206 } else { | 199 } else { |
207 return byteOffset - utf8Slack; | 200 return byteOffset - utf8Slack; |
208 } | 201 } |
209 } | 202 } |
210 | 203 |
211 Token firstToken() => tokens.next; | 204 Token firstToken() => tokens.next; |
212 Token previousToken() => tail; | 205 Token previousToken() => tail; |
213 | 206 |
214 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, | 207 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, |
215 [int extraOffset = 0]) { | 208 [int extraOffset = 0]) { |
216 tail.next = new StringToken.fromUtf8Bytes( | 209 tail.next = new StringToken.fromUtf8Bytes( |
217 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); | 210 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); |
218 tail = tail.next; | 211 tail = tail.next; |
219 } | 212 } |
220 | 213 |
221 bool atEndOfFile() => byteOffset >= bytes.length - 1; | 214 bool atEndOfFile() => byteOffset >= bytes.length - 1; |
222 } | 215 } |
OLD | NEW |