Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(173)

Side by Side Diff: pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart

Issue 2915093002: improve fasta unterminated string recovery (Closed)
Patch Set: fix dartdoc Created 3 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 library fasta.scanner.abstract_scanner; 5 library fasta.scanner.abstract_scanner;
6 6
7 import 'dart:collection' show ListMixin; 7 import 'dart:collection' show ListMixin;
8 8
9 import 'dart:typed_data' show Uint16List, Uint32List; 9 import 'dart:typed_data' show Uint16List, Uint32List;
10 10
(...skipping 144 matching lines...) Expand 10 before | Expand all | Expand 10 after
155 * [:scanOffset:] plus the [:extraOffset:]. For example, if the current 155 * [:scanOffset:] plus the [:extraOffset:]. For example, if the current
156 * scanOffset is 10, then [:appendSubstringToken(5, -1):] will append the 156 * scanOffset is 10, then [:appendSubstringToken(5, -1):] will append the
157 * substring string [5,9). 157 * substring string [5,9).
158 * 158 *
159 * Note that [extraOffset] can only be used if the covered character(s) are 159 * Note that [extraOffset] can only be used if the covered character(s) are
160 * known to be ASCII. 160 * known to be ASCII.
161 */ 161 */
162 void appendSubstringToken(TokenType type, int start, bool asciiOnly, 162 void appendSubstringToken(TokenType type, int start, bool asciiOnly,
163 [int extraOffset]); 163 [int extraOffset]);
164 164
165 /**
166 * Appends a substring from the scan offset [start] to the current
167 * [scanOffset] plus [closingQuotes]. The closing quote(s) will be added
168 * to the unterminated string literal's lexeme but the returned
169 * token's length will *not* include those closing quotes
170 * so as to be true to the original source.
171 */
172 void appendSyntheticSubstringToken(
173 TokenType type, int start, bool asciiOnly, String closingQuotes);
174
165 /** Documentation in subclass [ArrayBasedScanner]. */ 175 /** Documentation in subclass [ArrayBasedScanner]. */
166 void appendPrecedenceToken(TokenType type); 176 void appendPrecedenceToken(TokenType type);
167 177
168 /** Documentation in subclass [ArrayBasedScanner]. */ 178 /** Documentation in subclass [ArrayBasedScanner]. */
169 int select(int choice, TokenType yes, TokenType no); 179 int select(int choice, TokenType yes, TokenType no);
170 180
171 /** Documentation in subclass [ArrayBasedScanner]. */ 181 /** Documentation in subclass [ArrayBasedScanner]. */
172 void appendKeywordToken(Keyword keyword); 182 void appendKeywordToken(Keyword keyword);
173 183
174 /** Documentation in subclass [ArrayBasedScanner]. */ 184 /** Documentation in subclass [ArrayBasedScanner]. */
(...skipping 842 matching lines...) Expand 10 before | Expand all | Expand 10 after
1017 next = tokenizeStringInterpolation(start, asciiOnly); 1027 next = tokenizeStringInterpolation(start, asciiOnly);
1018 start = scanOffset; 1028 start = scanOffset;
1019 asciiOnly = true; 1029 asciiOnly = true;
1020 continue; 1030 continue;
1021 } 1031 }
1022 if (next <= $CR && 1032 if (next <= $CR &&
1023 (identical(next, $LF) || 1033 (identical(next, $LF) ||
1024 identical(next, $CR) || 1034 identical(next, $CR) ||
1025 identical(next, $EOF))) { 1035 identical(next, $EOF))) {
1026 if (!asciiOnly) handleUnicode(start); 1036 if (!asciiOnly) handleUnicode(start);
1027 return unterminatedString(quoteChar); 1037 return unterminatedString(quoteChar, start, asciiOnly);
1028 } 1038 }
1029 if (next > 127) asciiOnly = false; 1039 if (next > 127) asciiOnly = false;
1030 next = advance(); 1040 next = advance();
1031 } 1041 }
1032 if (!asciiOnly) handleUnicode(start); 1042 if (!asciiOnly) handleUnicode(start);
1033 // Advance past the quote character. 1043 // Advance past the quote character.
1034 next = advance(); 1044 next = advance();
1035 appendSubstringToken(TokenType.STRING, start, asciiOnly); 1045 appendSubstringToken(TokenType.STRING, start, asciiOnly);
1036 return next; 1046 return next;
1037 } 1047 }
(...skipping 44 matching lines...) Expand 10 before | Expand all | Expand 10 after
1082 int tokenizeSingleLineRawString(int next, int quoteChar, int start) { 1092 int tokenizeSingleLineRawString(int next, int quoteChar, int start) {
1083 bool asciiOnly = true; 1093 bool asciiOnly = true;
1084 while (next != $EOF) { 1094 while (next != $EOF) {
1085 if (identical(next, quoteChar)) { 1095 if (identical(next, quoteChar)) {
1086 if (!asciiOnly) handleUnicode(start); 1096 if (!asciiOnly) handleUnicode(start);
1087 next = advance(); 1097 next = advance();
1088 appendSubstringToken(TokenType.STRING, start, asciiOnly); 1098 appendSubstringToken(TokenType.STRING, start, asciiOnly);
1089 return next; 1099 return next;
1090 } else if (identical(next, $LF) || identical(next, $CR)) { 1100 } else if (identical(next, $LF) || identical(next, $CR)) {
1091 if (!asciiOnly) handleUnicode(start); 1101 if (!asciiOnly) handleUnicode(start);
1092 return unterminatedRawString(quoteChar); 1102 return unterminatedRawString(quoteChar, start, asciiOnly);
1093 } else if (next > 127) { 1103 } else if (next > 127) {
1094 asciiOnly = false; 1104 asciiOnly = false;
1095 } 1105 }
1096 next = advance(); 1106 next = advance();
1097 } 1107 }
1098 if (!asciiOnly) handleUnicode(start); 1108 if (!asciiOnly) handleUnicode(start);
1099 return unterminatedRawString(quoteChar); 1109 return unterminatedRawString(quoteChar, start, asciiOnly);
1100 } 1110 }
1101 1111
1102 int tokenizeMultiLineRawString(int quoteChar, int start) { 1112 int tokenizeMultiLineRawString(int quoteChar, int start) {
1103 bool asciiOnlyString = true; 1113 bool asciiOnlyString = true;
1104 bool asciiOnlyLine = true; 1114 bool asciiOnlyLine = true;
1105 int unicodeStart = start; 1115 int unicodeStart = start;
1106 int next = advance(); // Advance past the (last) quote (of three). 1116 int next = advance(); // Advance past the (last) quote (of three).
1107 outer: 1117 outer:
1108 while (!identical(next, $EOF)) { 1118 while (!identical(next, $EOF)) {
1109 while (!identical(next, quoteChar)) { 1119 while (!identical(next, quoteChar)) {
(...skipping 17 matching lines...) Expand all
1127 next = advance(); 1137 next = advance();
1128 if (identical(next, quoteChar)) { 1138 if (identical(next, quoteChar)) {
1129 if (!asciiOnlyLine) handleUnicode(unicodeStart); 1139 if (!asciiOnlyLine) handleUnicode(unicodeStart);
1130 next = advance(); 1140 next = advance();
1131 appendSubstringToken(TokenType.STRING, start, asciiOnlyString); 1141 appendSubstringToken(TokenType.STRING, start, asciiOnlyString);
1132 return next; 1142 return next;
1133 } 1143 }
1134 } 1144 }
1135 } 1145 }
1136 if (!asciiOnlyLine) handleUnicode(unicodeStart); 1146 if (!asciiOnlyLine) handleUnicode(unicodeStart);
1137 return unterminatedRawMultiLineString(quoteChar); 1147 return unterminatedRawMultiLineString(quoteChar, start, asciiOnlyLine);
1138 } 1148 }
1139 1149
1140 int tokenizeMultiLineString(int quoteChar, int start, bool raw) { 1150 int tokenizeMultiLineString(int quoteChar, int start, bool raw) {
1141 if (raw) return tokenizeMultiLineRawString(quoteChar, start); 1151 if (raw) return tokenizeMultiLineRawString(quoteChar, start);
1142 bool asciiOnlyString = true; 1152 bool asciiOnlyString = true;
1143 bool asciiOnlyLine = true; 1153 bool asciiOnlyLine = true;
1144 int unicodeStart = start; 1154 int unicodeStart = start;
1145 int next = advance(); // Advance past the (last) quote (of three). 1155 int next = advance(); // Advance past the (last) quote (of three).
1146 while (!identical(next, $EOF)) { 1156 while (!identical(next, $EOF)) {
1147 if (identical(next, $$)) { 1157 if (identical(next, $$)) {
(...skipping 30 matching lines...) Expand all
1178 unicodeStart = scanOffset; 1188 unicodeStart = scanOffset;
1179 } 1189 }
1180 lineFeedInMultiline(); 1190 lineFeedInMultiline();
1181 } else if (next > 127) { 1191 } else if (next > 127) {
1182 asciiOnlyString = false; 1192 asciiOnlyString = false;
1183 asciiOnlyLine = false; 1193 asciiOnlyLine = false;
1184 } 1194 }
1185 next = advance(); 1195 next = advance();
1186 } 1196 }
1187 if (!asciiOnlyLine) handleUnicode(unicodeStart); 1197 if (!asciiOnlyLine) handleUnicode(unicodeStart);
1188 return unterminatedMultiLineString(quoteChar); 1198 return unterminatedMultiLineString(quoteChar, start, asciiOnlyString);
1189 } 1199 }
1190 1200
1191 int unexpected(int character) { 1201 int unexpected(int character) {
1192 appendErrorToken(buildUnexpectedCharacterToken(character, tokenStart)); 1202 appendErrorToken(buildUnexpectedCharacterToken(character, tokenStart));
1193 return advanceAfterError(true); 1203 return advanceAfterError(true);
1194 } 1204 }
1195 1205
1196 int unterminated(String prefix, {bool shouldAdvance: true}) { 1206 int unterminated(String prefix, {bool shouldAdvance: true}) {
1197 appendErrorToken(new UnterminatedToken(prefix, tokenStart, stringOffset)); 1207 appendErrorToken(new UnterminatedToken(prefix, tokenStart, stringOffset));
1198 return advanceAfterError(shouldAdvance); 1208 return advanceAfterError(shouldAdvance);
1199 } 1209 }
1200 1210
1201 int unterminatedString(int quoteChar) { 1211 void terminateUnterminatedString(
1202 return unterminated(new String.fromCharCodes([quoteChar])); 1212 int start, bool asciiOnlyString, String closingQuotes) {
1213 appendSyntheticSubstringToken(
1214 TokenType.STRING, start, asciiOnlyString, closingQuotes);
1215 beginToken();
1203 } 1216 }
1204 1217
1205 int unterminatedRawString(int quoteChar) { 1218 int unterminatedString(int quoteChar, int start, bool asciiOnlyString) {
1206 return unterminated('r${new String.fromCharCodes([quoteChar])}'); 1219 String suffix = new String.fromCharCodes([quoteChar]);
1220 terminateUnterminatedString(start, asciiOnlyString, suffix);
1221 return unterminated(suffix);
1207 } 1222 }
1208 1223
1209 int unterminatedMultiLineString(int quoteChar) { 1224 int unterminatedRawString(int quoteChar, int start, bool asciiOnlyString) {
1210 return unterminated( 1225 String suffix = new String.fromCharCodes([quoteChar]);
1211 new String.fromCharCodes([quoteChar, quoteChar, quoteChar])); 1226 terminateUnterminatedString(start, asciiOnlyString, suffix);
1227 return unterminated('r$suffix');
1212 } 1228 }
1213 1229
1214 int unterminatedRawMultiLineString(int quoteChar) { 1230 int unterminatedMultiLineString(
1215 return unterminated( 1231 int quoteChar, int start, bool asciiOnlyString) {
1216 'r${new String.fromCharCodes([quoteChar, quoteChar, quoteChar])}'); 1232 String suffix = new String.fromCharCodes([quoteChar, quoteChar, quoteChar]);
1233 terminateUnterminatedString(start, asciiOnlyString, suffix);
1234 return unterminated(suffix);
1235 }
1236
1237 int unterminatedRawMultiLineString(
1238 int quoteChar, int start, bool asciiOnlyString) {
ahe 2017/06/02 12:26:43 Since these are for error handling, you could poss
danrubel 2017/06/07 15:51:41 Sounds reasonable. Fixed in https://codereview.chr
1239 String suffix = new String.fromCharCodes([quoteChar, quoteChar, quoteChar]);
1240 terminateUnterminatedString(start, asciiOnlyString, suffix);
1241 return unterminated('r$suffix');
1217 } 1242 }
1218 1243
1219 int advanceAfterError(bool shouldAdvance) { 1244 int advanceAfterError(bool shouldAdvance) {
1220 if (atEndOfFile()) return $EOF; 1245 if (atEndOfFile()) return $EOF;
1221 if (shouldAdvance) { 1246 if (shouldAdvance) {
1222 return advance(); // Ensure progress. 1247 return advance(); // Ensure progress.
1223 } else { 1248 } else {
1224 return -1; 1249 return -1;
1225 } 1250 }
1226 } 1251 }
(...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after
1305 switchToUint32(newLength); 1330 switchToUint32(newLength);
1306 } 1331 }
1307 } 1332 }
1308 1333
1309 void switchToUint32(int newLength) { 1334 void switchToUint32(int newLength) {
1310 final newArray = new Uint32List(newLength); 1335 final newArray = new Uint32List(newLength);
1311 newArray.setRange(0, arrayLength, array); 1336 newArray.setRange(0, arrayLength, array);
1312 array = newArray; 1337 array = newArray;
1313 } 1338 }
1314 } 1339 }
OLDNEW
« no previous file with comments | « no previous file | pkg/front_end/lib/src/fasta/scanner/array_based_scanner.dart » ('j') | pkg/front_end/test/scanner_test.dart » ('J')

Powered by Google App Engine
This is Rietveld 408576698