Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(258)

Side by Side Diff: pkg/front_end/lib/src/fasta/scanner/utf8_bytes_scanner.dart

Issue 2915093002: improve fasta unterminated string recovery (Closed)
Patch Set: fix dartdoc Created 3 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 library fasta.scanner.utf8_bytes_scanner; 5 library fasta.scanner.utf8_bytes_scanner;
6 6
7 import 'dart:convert' show UNICODE_BOM_CHARACTER_RUNE, UTF8; 7 import 'dart:convert' show UNICODE_BOM_CHARACTER_RUNE, UTF8;
8 8
9 import '../../scanner/token.dart' show TokenType; 9 import '../../scanner/token.dart' show SyntheticStringToken, TokenType;
10
11 import '../../scanner/token.dart' as analyzer show StringToken;
10 12
11 import '../scanner.dart' show unicodeReplacementCharacter; 13 import '../scanner.dart' show unicodeReplacementCharacter;
12 14
13 import 'token.dart' show CommentToken, DartDocToken, StringToken; 15 import 'token.dart' show CommentToken, DartDocToken, StringToken;
14 16
15 import 'array_based_scanner.dart' show ArrayBasedScanner; 17 import 'array_based_scanner.dart' show ArrayBasedScanner;
16 18
17 /** 19 /**
18 * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens 20 * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens
19 * that points to substrings. 21 * that points to substrings.
(...skipping 171 matching lines...) Expand 10 before | Expand all | Expand 10 after
191 193
192 int get stringOffset { 194 int get stringOffset {
193 if (stringOffsetSlackOffset == byteOffset) { 195 if (stringOffsetSlackOffset == byteOffset) {
194 return byteOffset - utf8Slack - 1; 196 return byteOffset - utf8Slack - 1;
195 } else { 197 } else {
196 return byteOffset - utf8Slack; 198 return byteOffset - utf8Slack;
197 } 199 }
198 } 200 }
199 201
200 @override 202 @override
201 StringToken createSubstringToken(TokenType type, int start, bool asciiOnly, 203 analyzer.StringToken createSubstringToken(
204 TokenType type, int start, bool asciiOnly,
202 [int extraOffset = 0]) { 205 [int extraOffset = 0]) {
203 return new StringToken.fromUtf8Bytes( 206 return new StringToken.fromUtf8Bytes(
204 type, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart, 207 type, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart,
205 precedingComments: comments); 208 precedingComments: comments);
206 } 209 }
207 210
208 @override 211 @override
212 analyzer.StringToken createSyntheticSubstringToken(
213 TokenType type, int start, bool asciiOnly, String closingQuotes) {
214 String source = StringToken.decodeUtf8(bytes, start, byteOffset, asciiOnly);
215 return new SyntheticStringToken(
216 type, source + closingQuotes, start, source.length);
217 }
218
219 @override
209 CommentToken createCommentToken(TokenType type, int start, bool asciiOnly, 220 CommentToken createCommentToken(TokenType type, int start, bool asciiOnly,
210 [int extraOffset = 0]) { 221 [int extraOffset = 0]) {
211 return new CommentToken.fromUtf8Bytes( 222 return new CommentToken.fromUtf8Bytes(
212 type, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); 223 type, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart);
213 } 224 }
214 225
215 @override 226 @override
216 DartDocToken createDartDocToken(TokenType type, int start, bool asciiOnly, 227 DartDocToken createDartDocToken(TokenType type, int start, bool asciiOnly,
217 [int extraOffset = 0]) { 228 [int extraOffset = 0]) {
218 return new DartDocToken.fromUtf8Bytes( 229 return new DartDocToken.fromUtf8Bytes(
219 type, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart); 230 type, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart);
220 } 231 }
221 232
222 bool atEndOfFile() => byteOffset >= bytes.length - 1; 233 bool atEndOfFile() => byteOffset >= bytes.length - 1;
223 } 234 }
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698