| OLD | NEW |
| 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 part of scanner; | 5 part of scanner; |
| 6 | 6 |
| 7 /** | 7 /** |
| 8 * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens | 8 * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens |
| 9 * that points to substrings. | 9 * that points to substrings. |
| 10 */ | 10 */ |
| 11 class Utf8BytesScanner extends ArrayBasedScanner { | 11 class Utf8BytesScanner extends ArrayBasedScanner { |
| 12 /** The file content. */ | 12 /** The file content. */ |
| 13 List<int> bytes; | 13 List<int> bytes; |
| 14 | 14 |
| 15 /** | 15 /** |
| 16 * Points to the offset of the byte last returned by [advance]. | 16 * Points to the offset of the last byte returned by [advance]. |
| 17 * | 17 * |
| 18 * After invoking [currentAsUnicode], the [byteOffset] points to the last | 18 * After invoking [currentAsUnicode], the [byteOffset] points to the last |
| 19 * byte that is part of the (unicode or ASCII) character. That way, [advance] | 19 * byte that is part of the (unicode or ASCII) character. That way, [advance] |
| 20 * can always increase the byte offset by 1. | 20 * can always increase the byte offset by 1. |
| 21 */ | 21 */ |
| 22 int byteOffset = -1; | 22 int byteOffset = -1; |
| 23 | 23 |
| 24 /** | 24 /** |
| 25 * The getter [scanOffset] is expected to return the index where the current | 25 * The getter [scanOffset] is expected to return the index where the current |
| 26 * character *starts*. In case of a non-ascii character, after invoking | 26 * character *starts*. In case of a non-ascii character, after invoking |
| (...skipping 150 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 177 if (stringOffsetSlackOffset == byteOffset) { | 177 if (stringOffsetSlackOffset == byteOffset) { |
| 178 return byteOffset - utf8Slack - 1; | 178 return byteOffset - utf8Slack - 1; |
| 179 } else { | 179 } else { |
| 180 return byteOffset - utf8Slack; | 180 return byteOffset - utf8Slack; |
| 181 } | 181 } |
| 182 } | 182 } |
| 183 | 183 |
| 184 Token firstToken() => tokens.next; | 184 Token firstToken() => tokens.next; |
| 185 Token previousToken() => tail; | 185 Token previousToken() => tail; |
| 186 | 186 |
| 187 | |
| 188 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, | 187 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, |
| 189 [int extraOffset = 0]) { | 188 [int extraOffset = 0]) { |
| 190 tail.next = new StringToken.fromUtf8Bytes( | 189 tail.next = new StringToken.fromUtf8Bytes( |
| 191 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); | 190 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); |
| 192 tail = tail.next; | 191 tail = tail.next; |
| 193 } | 192 } |
| 194 } | 193 } |
| OLD | NEW |