| OLD | NEW |
| 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library fasta.scanner.utf8_bytes_scanner; | 5 library fasta.scanner.utf8_bytes_scanner; |
| 6 | 6 |
| 7 import 'dart:convert' show UNICODE_BOM_CHARACTER_RUNE, UTF8; | 7 import 'dart:convert' show UNICODE_BOM_CHARACTER_RUNE, UTF8; |
| 8 | 8 |
| 9 import '../../scanner/token.dart' show TokenType; |
| 10 |
| 9 import '../scanner.dart' show unicodeReplacementCharacter; | 11 import '../scanner.dart' show unicodeReplacementCharacter; |
| 10 | 12 |
| 11 import 'precedence.dart' show PrecedenceInfo; | |
| 12 | |
| 13 import 'token.dart' show CommentToken, DartDocToken, StringToken; | 13 import 'token.dart' show CommentToken, DartDocToken, StringToken; |
| 14 | 14 |
| 15 import 'array_based_scanner.dart' show ArrayBasedScanner; | 15 import 'array_based_scanner.dart' show ArrayBasedScanner; |
| 16 | 16 |
| 17 /** | 17 /** |
| 18 * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens | 18 * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens |
| 19 * that points to substrings. | 19 * that points to substrings. |
| 20 */ | 20 */ |
| 21 class Utf8BytesScanner extends ArrayBasedScanner { | 21 class Utf8BytesScanner extends ArrayBasedScanner { |
| 22 /** | 22 /** |
| (...skipping 166 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 189 | 189 |
| 190 int get stringOffset { | 190 int get stringOffset { |
| 191 if (stringOffsetSlackOffset == byteOffset) { | 191 if (stringOffsetSlackOffset == byteOffset) { |
| 192 return byteOffset - utf8Slack - 1; | 192 return byteOffset - utf8Slack - 1; |
| 193 } else { | 193 } else { |
| 194 return byteOffset - utf8Slack; | 194 return byteOffset - utf8Slack; |
| 195 } | 195 } |
| 196 } | 196 } |
| 197 | 197 |
| 198 @override | 198 @override |
| 199 StringToken createSubstringToken( | 199 StringToken createSubstringToken(TokenType info, int start, bool asciiOnly, |
| 200 PrecedenceInfo info, int start, bool asciiOnly, | |
| 201 [int extraOffset = 0]) { | 200 [int extraOffset = 0]) { |
| 202 return new StringToken.fromUtf8Bytes( | 201 return new StringToken.fromUtf8Bytes( |
| 203 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); | 202 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); |
| 204 } | 203 } |
| 205 | 204 |
| 206 @override | 205 @override |
| 207 CommentToken createCommentToken( | 206 CommentToken createCommentToken(TokenType info, int start, bool asciiOnly, |
| 208 PrecedenceInfo info, int start, bool asciiOnly, | |
| 209 [int extraOffset = 0]) { | 207 [int extraOffset = 0]) { |
| 210 return new CommentToken.fromUtf8Bytes( | 208 return new CommentToken.fromUtf8Bytes( |
| 211 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); | 209 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); |
| 212 } | 210 } |
| 213 | 211 |
| 214 @override | 212 @override |
| 215 DartDocToken createDartDocToken( | 213 DartDocToken createDartDocToken(TokenType info, int start, bool asciiOnly, |
| 216 PrecedenceInfo info, int start, bool asciiOnly, | |
| 217 [int extraOffset = 0]) { | 214 [int extraOffset = 0]) { |
| 218 return new DartDocToken.fromUtf8Bytes( | 215 return new DartDocToken.fromUtf8Bytes( |
| 219 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); | 216 info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); |
| 220 } | 217 } |
| 221 | 218 |
| 222 bool atEndOfFile() => byteOffset >= bytes.length - 1; | 219 bool atEndOfFile() => byteOffset >= bytes.length - 1; |
| 223 } | 220 } |
| OLD | NEW |