OLD | NEW |
1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 library fasta.scanner.utf8_bytes_scanner; | 5 library fasta.scanner.utf8_bytes_scanner; |
6 | 6 |
7 import 'dart:convert' show | 7 import 'dart:convert' show |
8 UNICODE_BOM_CHARACTER_RUNE, | 8 UNICODE_BOM_CHARACTER_RUNE, |
9 UTF8; | 9 UTF8; |
10 | 10 |
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
80 /** | 80 /** |
81 * Creates a new Utf8BytesScanner. The source file is expected to be a | 81 * Creates a new Utf8BytesScanner. The source file is expected to be a |
82 * [Utf8BytesSourceFile] that holds a list of UTF-8 bytes. Otherwise the | 82 * [Utf8BytesSourceFile] that holds a list of UTF-8 bytes. Otherwise the |
83 * string text of the source file is decoded. | 83 * string text of the source file is decoded. |
84 * | 84 * |
85 * The list of UTF-8 bytes [file.slowUtf8Bytes()] is expected to return an | 85 * The list of UTF-8 bytes [file.slowUtf8Bytes()] is expected to return an |
86 * array whose last element is '0' to signal the end of the file. If this | 86 * array whose last element is '0' to signal the end of the file. If this |
87 * is not the case, the entire array is copied before scanning. | 87 * is not the case, the entire array is copied before scanning. |
88 */ | 88 */ |
89 Utf8BytesScanner(this.bytes, {bool includeComments: false}) | 89 Utf8BytesScanner(this.bytes, {bool includeComments: false}) |
90 : super(includeComments) { | 90 : super(includeComments, numberOfBytesHint: bytes.length) { |
91 assert(bytes.last == 0); | 91 assert(bytes.last == 0); |
92 // Skip a leading BOM. | 92 // Skip a leading BOM. |
93 if (containsBomAt(0)) byteOffset += 3; | 93 if (containsBomAt(0)) byteOffset += 3; |
94 } | 94 } |
95 | 95 |
96 bool containsBomAt(int offset) { | 96 bool containsBomAt(int offset) { |
97 const BOM_UTF8 = const [0xEF, 0xBB, 0xBF]; | 97 const BOM_UTF8 = const [0xEF, 0xBB, 0xBF]; |
98 | 98 |
99 return offset + 3 < bytes.length && | 99 return offset + 3 < bytes.length && |
100 bytes[offset] == BOM_UTF8[0] && | 100 bytes[offset] == BOM_UTF8[0] && |
(...skipping 106 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
207 | 207 |
208 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, | 208 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, |
209 [int extraOffset = 0]) { | 209 [int extraOffset = 0]) { |
210 tail.next = new StringToken.fromUtf8Bytes( | 210 tail.next = new StringToken.fromUtf8Bytes( |
211 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); | 211 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); |
212 tail = tail.next; | 212 tail = tail.next; |
213 } | 213 } |
214 | 214 |
215 bool atEndOfFile() => byteOffset >= bytes.length - 1; | 215 bool atEndOfFile() => byteOffset >= bytes.length - 1; |
216 } | 216 } |
OLD | NEW |