| OLD | NEW |
| 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library fasta.scanner.utf8_bytes_scanner; | 5 library fasta.scanner.utf8_bytes_scanner; |
| 6 | 6 |
| 7 import 'dart:convert' show UNICODE_BOM_CHARACTER_RUNE, UTF8; | 7 import 'dart:convert' show UNICODE_BOM_CHARACTER_RUNE, UTF8; |
| 8 | 8 |
| 9 import '../../scanner/token.dart' show TokenType; | 9 import '../../scanner/token.dart' show TokenType; |
| 10 | 10 |
| (...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 73 /** | 73 /** |
| 74 * Creates a new Utf8BytesScanner. The source file is expected to be a | 74 * Creates a new Utf8BytesScanner. The source file is expected to be a |
| 75 * [Utf8BytesSourceFile] that holds a list of UTF-8 bytes. Otherwise the | 75 * [Utf8BytesSourceFile] that holds a list of UTF-8 bytes. Otherwise the |
| 76 * string text of the source file is decoded. | 76 * string text of the source file is decoded. |
| 77 * | 77 * |
| 78 * The list of UTF-8 bytes [file.slowUtf8Bytes()] is expected to return an | 78 * The list of UTF-8 bytes [file.slowUtf8Bytes()] is expected to return an |
| 79 * array whose last element is '0' to signal the end of the file. If this | 79 * array whose last element is '0' to signal the end of the file. If this |
| 80 * is not the case, the entire array is copied before scanning. | 80 * is not the case, the entire array is copied before scanning. |
| 81 */ | 81 */ |
| 82 Utf8BytesScanner(this.bytes, | 82 Utf8BytesScanner(this.bytes, |
| 83 {bool includeComments: false, bool scanGenericMethodComments: false}) | 83 {bool includeComments: false, |
| 84 bool scanGenericMethodComments: false, |
| 85 bool scanLazyAssignmentOperators: false}) |
| 84 : super(includeComments, scanGenericMethodComments, | 86 : super(includeComments, scanGenericMethodComments, |
| 87 scanLazyAssignmentOperators, |
| 85 numberOfBytesHint: bytes.length) { | 88 numberOfBytesHint: bytes.length) { |
| 86 assert(bytes.last == 0); | 89 assert(bytes.last == 0); |
| 87 // Skip a leading BOM. | 90 // Skip a leading BOM. |
| 88 if (containsBomAt(0)) byteOffset += 3; | 91 if (containsBomAt(0)) byteOffset += 3; |
| 89 } | 92 } |
| 90 | 93 |
| 91 bool containsBomAt(int offset) { | 94 bool containsBomAt(int offset) { |
| 92 const BOM_UTF8 = const [0xEF, 0xBB, 0xBF]; | 95 const BOM_UTF8 = const [0xEF, 0xBB, 0xBF]; |
| 93 | 96 |
| 94 return offset + 3 < bytes.length && | 97 return offset + 3 < bytes.length && |
| (...skipping 119 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 214 | 217 |
| 215 @override | 218 @override |
| 216 DartDocToken createDartDocToken(TokenType type, int start, bool asciiOnly, | 219 DartDocToken createDartDocToken(TokenType type, int start, bool asciiOnly, |
| 217 [int extraOffset = 0]) { | 220 [int extraOffset = 0]) { |
| 218 return new DartDocToken.fromUtf8Bytes( | 221 return new DartDocToken.fromUtf8Bytes( |
| 219 type, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); | 222 type, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); |
| 220 } | 223 } |
| 221 | 224 |
| 222 bool atEndOfFile() => byteOffset >= bytes.length - 1; | 225 bool atEndOfFile() => byteOffset >= bytes.length - 1; |
| 223 } | 226 } |
| OLD | NEW |