| OLD | NEW |
| 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library dart2js.scanner.utf8; | 5 library dart2js.scanner.utf8; |
| 6 | 6 |
| 7 import 'dart:convert' show | 7 import 'dart:convert' show UNICODE_BOM_CHARACTER_RUNE, UTF8; |
| 8 UNICODE_BOM_CHARACTER_RUNE, | |
| 9 UTF8; | |
| 10 | 8 |
| 11 import '../io/source_file.dart' show | 9 import '../io/source_file.dart' show SourceFile; |
| 12 SourceFile; | 10 import '../tokens/precedence.dart' show PrecedenceInfo; |
| 13 import '../tokens/precedence.dart' show | 11 import '../tokens/token.dart' show StringToken, Token; |
| 14 PrecedenceInfo; | |
| 15 import '../tokens/token.dart' show | |
| 16 StringToken, | |
| 17 Token; | |
| 18 | 12 |
| 19 import 'array_based_scanner.dart' show | 13 import 'array_based_scanner.dart' show ArrayBasedScanner; |
| 20 ArrayBasedScanner; | |
| 21 | 14 |
| 22 /** | 15 /** |
| 23 * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens | 16 * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens |
| 24 * that points to substrings. | 17 * that points to substrings. |
| 25 */ | 18 */ |
| 26 class Utf8BytesScanner extends ArrayBasedScanner { | 19 class Utf8BytesScanner extends ArrayBasedScanner { |
| 27 /** | 20 /** |
| 28 * The file content. | 21 * The file content. |
| 29 * | 22 * |
| 30 * The content is zero-terminated. | 23 * The content is zero-terminated. |
| (...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 93 } | 86 } |
| 94 | 87 |
| 95 /** | 88 /** |
| 96 * Creates a new Utf8BytesScanner from a list of UTF-8 bytes. | 89 * Creates a new Utf8BytesScanner from a list of UTF-8 bytes. |
| 97 * | 90 * |
| 98 * The last element of the list is expected to be '0' to signal the end of | 91 * The last element of the list is expected to be '0' to signal the end of |
| 99 * the file. If this is not the case, the entire array is copied before | 92 * the file. If this is not the case, the entire array is copied before |
| 100 * scanning. | 93 * scanning. |
| 101 */ | 94 */ |
| 102 Utf8BytesScanner.fromBytes(List<int> zeroTerminatedBytes, | 95 Utf8BytesScanner.fromBytes(List<int> zeroTerminatedBytes, |
| 103 {bool includeComments: false}) | 96 {bool includeComments: false}) |
| 104 : this.bytes = zeroTerminatedBytes, | 97 : this.bytes = zeroTerminatedBytes, |
| 105 super(null, includeComments) { | 98 super(null, includeComments) { |
| 106 assert(bytes.last == 0); | 99 assert(bytes.last == 0); |
| 107 } | 100 } |
| 108 | 101 |
| 109 bool _containsBomAt(int offset) { | 102 bool _containsBomAt(int offset) { |
| 110 const BOM_UTF8 = const [0xEF, 0xBB, 0xBF]; | 103 const BOM_UTF8 = const [0xEF, 0xBB, 0xBF]; |
| 111 | 104 |
| 112 return offset + 3 < bytes.length && | 105 return offset + 3 < bytes.length && |
| 113 bytes[offset] == BOM_UTF8[0] && | 106 bytes[offset] == BOM_UTF8[0] && |
| (...skipping 91 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 205 return byteOffset - utf8Slack - 1; | 198 return byteOffset - utf8Slack - 1; |
| 206 } else { | 199 } else { |
| 207 return byteOffset - utf8Slack; | 200 return byteOffset - utf8Slack; |
| 208 } | 201 } |
| 209 } | 202 } |
| 210 | 203 |
| 211 Token firstToken() => tokens.next; | 204 Token firstToken() => tokens.next; |
| 212 Token previousToken() => tail; | 205 Token previousToken() => tail; |
| 213 | 206 |
| 214 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, | 207 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, |
| 215 [int extraOffset = 0]) { | 208 [int extraOffset = 0]) { |
| 216 tail.next = new StringToken.fromUtf8Bytes( | 209 tail.next = new StringToken.fromUtf8Bytes( |
| 217 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); | 210 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); |
| 218 tail = tail.next; | 211 tail = tail.next; |
| 219 } | 212 } |
| 220 | 213 |
| 221 bool atEndOfFile() => byteOffset >= bytes.length - 1; | 214 bool atEndOfFile() => byteOffset >= bytes.length - 1; |
| 222 } | 215 } |
| OLD | NEW |