| OLD | NEW |
| 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 part of scanner; | 5 part of scanner; |
| 6 | 6 |
| 7 /** | 7 /** |
| 8 * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens | 8 * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens |
| 9 * that points to substrings. | 9 * that points to substrings. |
| 10 */ | 10 */ |
| (...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 62 | 62 |
| 63 /** | 63 /** |
| 64 * Creates a new Utf8BytesScanner. The source file is expected to be a | 64 * Creates a new Utf8BytesScanner. The source file is expected to be a |
| 65 * [Utf8BytesSourceFile] that holds a list of UTF-8 bytes. Otherwise the | 65 * [Utf8BytesSourceFile] that holds a list of UTF-8 bytes. Otherwise the |
| 66 * string text of the source file is decoded. | 66 * string text of the source file is decoded. |
| 67 * | 67 * |
| 68 * The list of UTF-8 bytes [file.slowUtf8Bytes()] is expected to return an | 68 * The list of UTF-8 bytes [file.slowUtf8Bytes()] is expected to return an |
| 69 * array whose last element is '0' to signal the end of the file. If this | 69 * array whose last element is '0' to signal the end of the file. If this |
| 70 * is not the case, the entire array is copied before scanning. | 70 * is not the case, the entire array is copied before scanning. |
| 71 */ | 71 */ |
| 72 Utf8BytesScanner(SourceFile file, {bool includeComments: false}) | 72 Utf8BytesScanner(SourceFile file, {bool includeComments: false, |
| 73 bool enableNullAwareOperators: false}) |
| 73 : bytes = file.slowUtf8ZeroTerminatedBytes(), | 74 : bytes = file.slowUtf8ZeroTerminatedBytes(), |
| 74 super(file, includeComments) { | 75 super(file, includeComments, enableNullAwareOperators) { |
| 75 assert(bytes.last == 0); | 76 assert(bytes.last == 0); |
| 76 // Skip a leading BOM. | 77 // Skip a leading BOM. |
| 77 if (_containsBomAt(0)) byteOffset += 3; | 78 if (_containsBomAt(0)) byteOffset += 3; |
| 78 } | 79 } |
| 79 | 80 |
| 80 /** | 81 /** |
| 81 * Creates a new Utf8BytesScanner from a list of UTF-8 bytes. | 82 * Creates a new Utf8BytesScanner from a list of UTF-8 bytes. |
| 82 * | 83 * |
| 83 * The last element of the list is expected to be '0' to signal the end of | 84 * The last element of the list is expected to be '0' to signal the end of |
| 84 * the file. If this is not the case, the entire array is copied before | 85 * the file. If this is not the case, the entire array is copied before |
| (...skipping 113 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 198 | 199 |
| 199 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, | 200 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, |
| 200 [int extraOffset = 0]) { | 201 [int extraOffset = 0]) { |
| 201 tail.next = new StringToken.fromUtf8Bytes( | 202 tail.next = new StringToken.fromUtf8Bytes( |
| 202 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); | 203 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); |
| 203 tail = tail.next; | 204 tail = tail.next; |
| 204 } | 205 } |
| 205 | 206 |
| 206 bool atEndOfFile() => byteOffset >= bytes.length - 1; | 207 bool atEndOfFile() => byteOffset >= bytes.length - 1; |
| 207 } | 208 } |
| OLD | NEW |