| OLD | NEW |
| 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library fasta.scanner.utf8_bytes_scanner; | 5 library fasta.scanner.utf8_bytes_scanner; |
| 6 | 6 |
| 7 import 'dart:convert' show UNICODE_BOM_CHARACTER_RUNE, UTF8; | 7 import 'dart:convert' show UNICODE_BOM_CHARACTER_RUNE, UTF8; |
| 8 | 8 |
| 9 import '../../scanner/token.dart' show TokenType; | 9 import '../../scanner/token.dart' show SyntheticStringToken, TokenType; |
| 10 |
| 11 import '../../scanner/token.dart' as analyzer show StringToken; |
| 10 | 12 |
| 11 import '../scanner.dart' show unicodeReplacementCharacter; | 13 import '../scanner.dart' show unicodeReplacementCharacter; |
| 12 | 14 |
| 13 import 'token.dart' show CommentToken, DartDocToken, StringToken; | 15 import 'token.dart' show CommentToken, DartDocToken, StringToken; |
| 14 | 16 |
| 15 import 'array_based_scanner.dart' show ArrayBasedScanner; | 17 import 'array_based_scanner.dart' show ArrayBasedScanner; |
| 16 | 18 |
| 17 /** | 19 /** |
| 18 * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens | 20 * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens |
| 19 * that points to substrings. | 21 * that points to substrings. |
| (...skipping 171 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 191 | 193 |
| 192 int get stringOffset { | 194 int get stringOffset { |
| 193 if (stringOffsetSlackOffset == byteOffset) { | 195 if (stringOffsetSlackOffset == byteOffset) { |
| 194 return byteOffset - utf8Slack - 1; | 196 return byteOffset - utf8Slack - 1; |
| 195 } else { | 197 } else { |
| 196 return byteOffset - utf8Slack; | 198 return byteOffset - utf8Slack; |
| 197 } | 199 } |
| 198 } | 200 } |
| 199 | 201 |
| 200 @override | 202 @override |
| 201 StringToken createSubstringToken(TokenType type, int start, bool asciiOnly, | 203 analyzer.StringToken createSubstringToken( |
| 204 TokenType type, int start, bool asciiOnly, |
| 202 [int extraOffset = 0]) { | 205 [int extraOffset = 0]) { |
| 203 return new StringToken.fromUtf8Bytes( | 206 return new StringToken.fromUtf8Bytes( |
| 204 type, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart, | 207 type, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart, |
| 205 precedingComments: comments); | 208 precedingComments: comments); |
| 206 } | 209 } |
| 207 | 210 |
| 208 @override | 211 @override |
| 212 analyzer.StringToken createSyntheticSubstringToken( |
| 213 TokenType type, int start, bool asciiOnly, String closingQuotes) { |
| 214 String source = StringToken.decodeUtf8(bytes, start, byteOffset, asciiOnly); |
| 215 return new SyntheticStringToken( |
| 216 type, source + closingQuotes, start, source.length); |
| 217 } |
| 218 |
| 219 @override |
| 209 CommentToken createCommentToken(TokenType type, int start, bool asciiOnly, | 220 CommentToken createCommentToken(TokenType type, int start, bool asciiOnly, |
| 210 [int extraOffset = 0]) { | 221 [int extraOffset = 0]) { |
| 211 return new CommentToken.fromUtf8Bytes( | 222 return new CommentToken.fromUtf8Bytes( |
| 212 type, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); | 223 type, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); |
| 213 } | 224 } |
| 214 | 225 |
| 215 @override | 226 @override |
| 216 DartDocToken createDartDocToken(TokenType type, int start, bool asciiOnly, | 227 DartDocToken createDartDocToken(TokenType type, int start, bool asciiOnly, |
| 217 [int extraOffset = 0]) { | 228 [int extraOffset = 0]) { |
| 218 return new DartDocToken.fromUtf8Bytes( | 229 return new DartDocToken.fromUtf8Bytes( |
| 219 type, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); | 230 type, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); |
| 220 } | 231 } |
| 221 | 232 |
| 222 bool atEndOfFile() => byteOffset >= bytes.length - 1; | 233 bool atEndOfFile() => byteOffset >= bytes.length - 1; |
| 223 } | 234 } |
| OLD | NEW |