OLD | NEW |
---|---|
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 library fasta.scanner.abstract_scanner; | 5 library fasta.scanner.abstract_scanner; |
6 | 6 |
7 import 'dart:collection' show ListMixin; | 7 import 'dart:collection' show ListMixin; |
8 | 8 |
9 import 'dart:typed_data' show Uint16List, Uint32List; | 9 import 'dart:typed_data' show Uint16List, Uint32List; |
10 | 10 |
(...skipping 144 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
155 * [:scanOffset:] plus the [:extraOffset:]. For example, if the current | 155 * [:scanOffset:] plus the [:extraOffset:]. For example, if the current |
156 * scanOffset is 10, then [:appendSubstringToken(5, -1):] will append the | 156 * scanOffset is 10, then [:appendSubstringToken(5, -1):] will append the |
157 * substring string [5,9). | 157 * substring string [5,9). |
158 * | 158 * |
159 * Note that [extraOffset] can only be used if the covered character(s) are | 159 * Note that [extraOffset] can only be used if the covered character(s) are |
160 * known to be ASCII. | 160 * known to be ASCII. |
161 */ | 161 */ |
162 void appendSubstringToken(TokenType type, int start, bool asciiOnly, | 162 void appendSubstringToken(TokenType type, int start, bool asciiOnly, |
163 [int extraOffset]); | 163 [int extraOffset]); |
164 | 164 |
165 /** | |
166 * Appends a substring from the scan offset [start] to the current | |
167 * [scanOffset] plus [closingQuotes]. The closing quote(s) will be added | |
168 * to the unterminated string literal's lexeme but the returned | |
169 * token's length will *not* include those closing quotes | |
170 * so as to be true to the original source. | |
171 */ | |
172 void appendSyntheticSubstringToken( | |
173 TokenType type, int start, bool asciiOnly, String closingQuotes); | |
174 | |
165 /** Documentation in subclass [ArrayBasedScanner]. */ | 175 /** Documentation in subclass [ArrayBasedScanner]. */ |
166 void appendPrecedenceToken(TokenType type); | 176 void appendPrecedenceToken(TokenType type); |
167 | 177 |
168 /** Documentation in subclass [ArrayBasedScanner]. */ | 178 /** Documentation in subclass [ArrayBasedScanner]. */ |
169 int select(int choice, TokenType yes, TokenType no); | 179 int select(int choice, TokenType yes, TokenType no); |
170 | 180 |
171 /** Documentation in subclass [ArrayBasedScanner]. */ | 181 /** Documentation in subclass [ArrayBasedScanner]. */ |
172 void appendKeywordToken(Keyword keyword); | 182 void appendKeywordToken(Keyword keyword); |
173 | 183 |
174 /** Documentation in subclass [ArrayBasedScanner]. */ | 184 /** Documentation in subclass [ArrayBasedScanner]. */ |
(...skipping 842 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1017 next = tokenizeStringInterpolation(start, asciiOnly); | 1027 next = tokenizeStringInterpolation(start, asciiOnly); |
1018 start = scanOffset; | 1028 start = scanOffset; |
1019 asciiOnly = true; | 1029 asciiOnly = true; |
1020 continue; | 1030 continue; |
1021 } | 1031 } |
1022 if (next <= $CR && | 1032 if (next <= $CR && |
1023 (identical(next, $LF) || | 1033 (identical(next, $LF) || |
1024 identical(next, $CR) || | 1034 identical(next, $CR) || |
1025 identical(next, $EOF))) { | 1035 identical(next, $EOF))) { |
1026 if (!asciiOnly) handleUnicode(start); | 1036 if (!asciiOnly) handleUnicode(start); |
1027 return unterminatedString(quoteChar); | 1037 return unterminatedString(quoteChar, start, asciiOnly); |
1028 } | 1038 } |
1029 if (next > 127) asciiOnly = false; | 1039 if (next > 127) asciiOnly = false; |
1030 next = advance(); | 1040 next = advance(); |
1031 } | 1041 } |
1032 if (!asciiOnly) handleUnicode(start); | 1042 if (!asciiOnly) handleUnicode(start); |
1033 // Advance past the quote character. | 1043 // Advance past the quote character. |
1034 next = advance(); | 1044 next = advance(); |
1035 appendSubstringToken(TokenType.STRING, start, asciiOnly); | 1045 appendSubstringToken(TokenType.STRING, start, asciiOnly); |
1036 return next; | 1046 return next; |
1037 } | 1047 } |
(...skipping 44 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1082 int tokenizeSingleLineRawString(int next, int quoteChar, int start) { | 1092 int tokenizeSingleLineRawString(int next, int quoteChar, int start) { |
1083 bool asciiOnly = true; | 1093 bool asciiOnly = true; |
1084 while (next != $EOF) { | 1094 while (next != $EOF) { |
1085 if (identical(next, quoteChar)) { | 1095 if (identical(next, quoteChar)) { |
1086 if (!asciiOnly) handleUnicode(start); | 1096 if (!asciiOnly) handleUnicode(start); |
1087 next = advance(); | 1097 next = advance(); |
1088 appendSubstringToken(TokenType.STRING, start, asciiOnly); | 1098 appendSubstringToken(TokenType.STRING, start, asciiOnly); |
1089 return next; | 1099 return next; |
1090 } else if (identical(next, $LF) || identical(next, $CR)) { | 1100 } else if (identical(next, $LF) || identical(next, $CR)) { |
1091 if (!asciiOnly) handleUnicode(start); | 1101 if (!asciiOnly) handleUnicode(start); |
1092 return unterminatedRawString(quoteChar); | 1102 return unterminatedRawString(quoteChar, start, asciiOnly); |
1093 } else if (next > 127) { | 1103 } else if (next > 127) { |
1094 asciiOnly = false; | 1104 asciiOnly = false; |
1095 } | 1105 } |
1096 next = advance(); | 1106 next = advance(); |
1097 } | 1107 } |
1098 if (!asciiOnly) handleUnicode(start); | 1108 if (!asciiOnly) handleUnicode(start); |
1099 return unterminatedRawString(quoteChar); | 1109 return unterminatedRawString(quoteChar, start, asciiOnly); |
1100 } | 1110 } |
1101 | 1111 |
1102 int tokenizeMultiLineRawString(int quoteChar, int start) { | 1112 int tokenizeMultiLineRawString(int quoteChar, int start) { |
1103 bool asciiOnlyString = true; | 1113 bool asciiOnlyString = true; |
1104 bool asciiOnlyLine = true; | 1114 bool asciiOnlyLine = true; |
1105 int unicodeStart = start; | 1115 int unicodeStart = start; |
1106 int next = advance(); // Advance past the (last) quote (of three). | 1116 int next = advance(); // Advance past the (last) quote (of three). |
1107 outer: | 1117 outer: |
1108 while (!identical(next, $EOF)) { | 1118 while (!identical(next, $EOF)) { |
1109 while (!identical(next, quoteChar)) { | 1119 while (!identical(next, quoteChar)) { |
(...skipping 17 matching lines...) Expand all Loading... | |
1127 next = advance(); | 1137 next = advance(); |
1128 if (identical(next, quoteChar)) { | 1138 if (identical(next, quoteChar)) { |
1129 if (!asciiOnlyLine) handleUnicode(unicodeStart); | 1139 if (!asciiOnlyLine) handleUnicode(unicodeStart); |
1130 next = advance(); | 1140 next = advance(); |
1131 appendSubstringToken(TokenType.STRING, start, asciiOnlyString); | 1141 appendSubstringToken(TokenType.STRING, start, asciiOnlyString); |
1132 return next; | 1142 return next; |
1133 } | 1143 } |
1134 } | 1144 } |
1135 } | 1145 } |
1136 if (!asciiOnlyLine) handleUnicode(unicodeStart); | 1146 if (!asciiOnlyLine) handleUnicode(unicodeStart); |
1137 return unterminatedRawMultiLineString(quoteChar); | 1147 return unterminatedRawMultiLineString(quoteChar, start, asciiOnlyLine); |
1138 } | 1148 } |
1139 | 1149 |
1140 int tokenizeMultiLineString(int quoteChar, int start, bool raw) { | 1150 int tokenizeMultiLineString(int quoteChar, int start, bool raw) { |
1141 if (raw) return tokenizeMultiLineRawString(quoteChar, start); | 1151 if (raw) return tokenizeMultiLineRawString(quoteChar, start); |
1142 bool asciiOnlyString = true; | 1152 bool asciiOnlyString = true; |
1143 bool asciiOnlyLine = true; | 1153 bool asciiOnlyLine = true; |
1144 int unicodeStart = start; | 1154 int unicodeStart = start; |
1145 int next = advance(); // Advance past the (last) quote (of three). | 1155 int next = advance(); // Advance past the (last) quote (of three). |
1146 while (!identical(next, $EOF)) { | 1156 while (!identical(next, $EOF)) { |
1147 if (identical(next, $$)) { | 1157 if (identical(next, $$)) { |
(...skipping 30 matching lines...) Expand all Loading... | |
1178 unicodeStart = scanOffset; | 1188 unicodeStart = scanOffset; |
1179 } | 1189 } |
1180 lineFeedInMultiline(); | 1190 lineFeedInMultiline(); |
1181 } else if (next > 127) { | 1191 } else if (next > 127) { |
1182 asciiOnlyString = false; | 1192 asciiOnlyString = false; |
1183 asciiOnlyLine = false; | 1193 asciiOnlyLine = false; |
1184 } | 1194 } |
1185 next = advance(); | 1195 next = advance(); |
1186 } | 1196 } |
1187 if (!asciiOnlyLine) handleUnicode(unicodeStart); | 1197 if (!asciiOnlyLine) handleUnicode(unicodeStart); |
1188 return unterminatedMultiLineString(quoteChar); | 1198 return unterminatedMultiLineString(quoteChar, start, asciiOnlyString); |
1189 } | 1199 } |
1190 | 1200 |
1191 int unexpected(int character) { | 1201 int unexpected(int character) { |
1192 appendErrorToken(buildUnexpectedCharacterToken(character, tokenStart)); | 1202 appendErrorToken(buildUnexpectedCharacterToken(character, tokenStart)); |
1193 return advanceAfterError(true); | 1203 return advanceAfterError(true); |
1194 } | 1204 } |
1195 | 1205 |
1196 int unterminated(String prefix, {bool shouldAdvance: true}) { | 1206 int unterminated(String prefix, {bool shouldAdvance: true}) { |
1197 appendErrorToken(new UnterminatedToken(prefix, tokenStart, stringOffset)); | 1207 appendErrorToken(new UnterminatedToken(prefix, tokenStart, stringOffset)); |
1198 return advanceAfterError(shouldAdvance); | 1208 return advanceAfterError(shouldAdvance); |
1199 } | 1209 } |
1200 | 1210 |
1201 int unterminatedString(int quoteChar) { | 1211 void terminateUnterminatedString( |
1202 return unterminated(new String.fromCharCodes([quoteChar])); | 1212 int start, bool asciiOnlyString, String closingQuotes) { |
1213 appendSyntheticSubstringToken( | |
1214 TokenType.STRING, start, asciiOnlyString, closingQuotes); | |
1215 beginToken(); | |
1203 } | 1216 } |
1204 | 1217 |
1205 int unterminatedRawString(int quoteChar) { | 1218 int unterminatedString(int quoteChar, int start, bool asciiOnlyString) { |
1206 return unterminated('r${new String.fromCharCodes([quoteChar])}'); | 1219 String suffix = new String.fromCharCodes([quoteChar]); |
1220 terminateUnterminatedString(start, asciiOnlyString, suffix); | |
1221 return unterminated(suffix); | |
1207 } | 1222 } |
1208 | 1223 |
1209 int unterminatedMultiLineString(int quoteChar) { | 1224 int unterminatedRawString(int quoteChar, int start, bool asciiOnlyString) { |
1210 return unterminated( | 1225 String suffix = new String.fromCharCodes([quoteChar]); |
1211 new String.fromCharCodes([quoteChar, quoteChar, quoteChar])); | 1226 terminateUnterminatedString(start, asciiOnlyString, suffix); |
1227 return unterminated('r$suffix'); | |
1212 } | 1228 } |
1213 | 1229 |
1214 int unterminatedRawMultiLineString(int quoteChar) { | 1230 int unterminatedMultiLineString( |
1215 return unterminated( | 1231 int quoteChar, int start, bool asciiOnlyString) { |
1216 'r${new String.fromCharCodes([quoteChar, quoteChar, quoteChar])}'); | 1232 String suffix = new String.fromCharCodes([quoteChar, quoteChar, quoteChar]); |
1233 terminateUnterminatedString(start, asciiOnlyString, suffix); | |
1234 return unterminated(suffix); | |
1235 } | |
1236 | |
1237 int unterminatedRawMultiLineString( | |
1238 int quoteChar, int start, bool asciiOnlyString) { | |
ahe
2017/06/02 12:26:43
Since these are for error handling, you could poss
danrubel
2017/06/07 15:51:41
Sounds reasonable. Fixed in https://codereview.chr
| |
1239 String suffix = new String.fromCharCodes([quoteChar, quoteChar, quoteChar]); | |
1240 terminateUnterminatedString(start, asciiOnlyString, suffix); | |
1241 return unterminated('r$suffix'); | |
1217 } | 1242 } |
1218 | 1243 |
1219 int advanceAfterError(bool shouldAdvance) { | 1244 int advanceAfterError(bool shouldAdvance) { |
1220 if (atEndOfFile()) return $EOF; | 1245 if (atEndOfFile()) return $EOF; |
1221 if (shouldAdvance) { | 1246 if (shouldAdvance) { |
1222 return advance(); // Ensure progress. | 1247 return advance(); // Ensure progress. |
1223 } else { | 1248 } else { |
1224 return -1; | 1249 return -1; |
1225 } | 1250 } |
1226 } | 1251 } |
(...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1305 switchToUint32(newLength); | 1330 switchToUint32(newLength); |
1306 } | 1331 } |
1307 } | 1332 } |
1308 | 1333 |
1309 void switchToUint32(int newLength) { | 1334 void switchToUint32(int newLength) { |
1310 final newArray = new Uint32List(newLength); | 1335 final newArray = new Uint32List(newLength); |
1311 newArray.setRange(0, arrayLength, array); | 1336 newArray.setRange(0, arrayLength, array); |
1312 array = newArray; | 1337 array = newArray; |
1313 } | 1338 } |
1314 } | 1339 } |
OLD | NEW |