| OLD | NEW |
| 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library fasta.scanner.utf8_bytes_scanner; | 5 library fasta.scanner.utf8_bytes_scanner; |
| 6 | 6 |
| 7 import 'dart:convert' show | 7 import 'dart:convert' show |
| 8 UNICODE_BOM_CHARACTER_RUNE, | 8 UNICODE_BOM_CHARACTER_RUNE, |
| 9 UTF8; | 9 UTF8; |
| 10 | 10 |
| (...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 80 /** | 80 /** |
| 81 * Creates a new Utf8BytesScanner. The source file is expected to be a | 81 * Creates a new Utf8BytesScanner. The source file is expected to be a |
| 82 * [Utf8BytesSourceFile] that holds a list of UTF-8 bytes. Otherwise the | 82 * [Utf8BytesSourceFile] that holds a list of UTF-8 bytes. Otherwise the |
| 83 * string text of the source file is decoded. | 83 * string text of the source file is decoded. |
| 84 * | 84 * |
| 85 * The list of UTF-8 bytes [file.slowUtf8Bytes()] is expected to return an | 85 * The list of UTF-8 bytes [file.slowUtf8Bytes()] is expected to return an |
| 86 * array whose last element is '0' to signal the end of the file. If this | 86 * array whose last element is '0' to signal the end of the file. If this |
| 87 * is not the case, the entire array is copied before scanning. | 87 * is not the case, the entire array is copied before scanning. |
| 88 */ | 88 */ |
| 89 Utf8BytesScanner(this.bytes, {bool includeComments: false}) | 89 Utf8BytesScanner(this.bytes, {bool includeComments: false}) |
| 90 : super(includeComments) { | 90 : super(includeComments, numberOfBytesHint: bytes.length) { |
| 91 assert(bytes.last == 0); | 91 assert(bytes.last == 0); |
| 92 // Skip a leading BOM. | 92 // Skip a leading BOM. |
| 93 if (containsBomAt(0)) byteOffset += 3; | 93 if (containsBomAt(0)) byteOffset += 3; |
| 94 } | 94 } |
| 95 | 95 |
| 96 bool containsBomAt(int offset) { | 96 bool containsBomAt(int offset) { |
| 97 const BOM_UTF8 = const [0xEF, 0xBB, 0xBF]; | 97 const BOM_UTF8 = const [0xEF, 0xBB, 0xBF]; |
| 98 | 98 |
| 99 return offset + 3 < bytes.length && | 99 return offset + 3 < bytes.length && |
| 100 bytes[offset] == BOM_UTF8[0] && | 100 bytes[offset] == BOM_UTF8[0] && |
| (...skipping 106 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 207 | 207 |
| 208 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, | 208 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, |
| 209 [int extraOffset = 0]) { | 209 [int extraOffset = 0]) { |
| 210 tail.next = new StringToken.fromUtf8Bytes( | 210 tail.next = new StringToken.fromUtf8Bytes( |
| 211 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); | 211 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); |
| 212 tail = tail.next; | 212 tail = tail.next; |
| 213 } | 213 } |
| 214 | 214 |
| 215 bool atEndOfFile() => byteOffset >= bytes.length - 1; | 215 bool atEndOfFile() => byteOffset >= bytes.length - 1; |
| 216 } | 216 } |
| OLD | NEW |