OLD | NEW |
1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 library fasta.scanner.utf8_bytes_scanner; | 5 library fasta.scanner.utf8_bytes_scanner; |
6 | 6 |
7 import 'dart:convert' show UNICODE_BOM_CHARACTER_RUNE, UTF8; | 7 import 'dart:convert' show UNICODE_BOM_CHARACTER_RUNE, UTF8; |
8 | 8 |
9 import '../../scanner/token.dart' show TokenType; | 9 import '../../scanner/token.dart' show SyntheticStringToken, TokenType; |
| 10 |
| 11 import '../../scanner/token.dart' as analyzer show StringToken; |
10 | 12 |
11 import '../scanner.dart' show unicodeReplacementCharacter; | 13 import '../scanner.dart' show unicodeReplacementCharacter; |
12 | 14 |
13 import 'token.dart' show CommentToken, DartDocToken, StringToken; | 15 import 'token.dart' show CommentToken, DartDocToken, StringToken; |
14 | 16 |
15 import 'array_based_scanner.dart' show ArrayBasedScanner; | 17 import 'array_based_scanner.dart' show ArrayBasedScanner; |
16 | 18 |
17 /** | 19 /** |
18 * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens | 20 * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens |
19 * that points to substrings. | 21 * that points to substrings. |
(...skipping 171 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
191 | 193 |
192 int get stringOffset { | 194 int get stringOffset { |
193 if (stringOffsetSlackOffset == byteOffset) { | 195 if (stringOffsetSlackOffset == byteOffset) { |
194 return byteOffset - utf8Slack - 1; | 196 return byteOffset - utf8Slack - 1; |
195 } else { | 197 } else { |
196 return byteOffset - utf8Slack; | 198 return byteOffset - utf8Slack; |
197 } | 199 } |
198 } | 200 } |
199 | 201 |
200 @override | 202 @override |
201 StringToken createSubstringToken(TokenType type, int start, bool asciiOnly, | 203 analyzer.StringToken createSubstringToken( |
| 204 TokenType type, int start, bool asciiOnly, |
202 [int extraOffset = 0]) { | 205 [int extraOffset = 0]) { |
203 return new StringToken.fromUtf8Bytes( | 206 return new StringToken.fromUtf8Bytes( |
204 type, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart, | 207 type, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart, |
205 precedingComments: comments); | 208 precedingComments: comments); |
206 } | 209 } |
207 | 210 |
208 @override | 211 @override |
| 212 analyzer.StringToken createSyntheticSubstringToken( |
| 213 TokenType type, int start, bool asciiOnly, String closingQuotes) { |
| 214 String source = StringToken.decodeUtf8(bytes, start, byteOffset, asciiOnly); |
| 215 return new SyntheticStringToken( |
| 216 type, source + closingQuotes, start, source.length); |
| 217 } |
| 218 |
| 219 @override |
209 CommentToken createCommentToken(TokenType type, int start, bool asciiOnly, | 220 CommentToken createCommentToken(TokenType type, int start, bool asciiOnly, |
210 [int extraOffset = 0]) { | 221 [int extraOffset = 0]) { |
211 return new CommentToken.fromUtf8Bytes( | 222 return new CommentToken.fromUtf8Bytes( |
212 type, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); | 223 type, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); |
213 } | 224 } |
214 | 225 |
215 @override | 226 @override |
216 DartDocToken createDartDocToken(TokenType type, int start, bool asciiOnly, | 227 DartDocToken createDartDocToken(TokenType type, int start, bool asciiOnly, |
217 [int extraOffset = 0]) { | 228 [int extraOffset = 0]) { |
218 return new DartDocToken.fromUtf8Bytes( | 229 return new DartDocToken.fromUtf8Bytes( |
219 type, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); | 230 type, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); |
220 } | 231 } |
221 | 232 |
222 bool atEndOfFile() => byteOffset >= bytes.length - 1; | 233 bool atEndOfFile() => byteOffset >= bytes.length - 1; |
223 } | 234 } |
OLD | NEW |