| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library fasta.scanner.abstract_scanner; | 5 library fasta.scanner.abstract_scanner; |
| 6 | 6 |
| 7 import 'dart:collection' show ListMixin; | 7 import 'dart:collection' show ListMixin; |
| 8 | 8 |
| 9 import 'dart:typed_data' show Uint16List, Uint32List; | 9 import 'dart:typed_data' show Uint16List, Uint32List; |
| 10 | 10 |
| (...skipping 1016 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1027 next = tokenizeStringInterpolation(start, asciiOnly); | 1027 next = tokenizeStringInterpolation(start, asciiOnly); |
| 1028 start = scanOffset; | 1028 start = scanOffset; |
| 1029 asciiOnly = true; | 1029 asciiOnly = true; |
| 1030 continue; | 1030 continue; |
| 1031 } | 1031 } |
| 1032 if (next <= $CR && | 1032 if (next <= $CR && |
| 1033 (identical(next, $LF) || | 1033 (identical(next, $LF) || |
| 1034 identical(next, $CR) || | 1034 identical(next, $CR) || |
| 1035 identical(next, $EOF))) { | 1035 identical(next, $EOF))) { |
| 1036 if (!asciiOnly) handleUnicode(start); | 1036 if (!asciiOnly) handleUnicode(start); |
| 1037 return unterminatedString(quoteChar, start, | 1037 unterminatedString(quoteChar, start, |
| 1038 asciiOnly: asciiOnly, isMultiLine: false, isRaw: false); | 1038 asciiOnly: asciiOnly, isMultiLine: false, isRaw: false); |
| 1039 return next; |
| 1039 } | 1040 } |
| 1040 if (next > 127) asciiOnly = false; | 1041 if (next > 127) asciiOnly = false; |
| 1041 next = advance(); | 1042 next = advance(); |
| 1042 } | 1043 } |
| 1043 if (!asciiOnly) handleUnicode(start); | 1044 if (!asciiOnly) handleUnicode(start); |
| 1044 // Advance past the quote character. | 1045 // Advance past the quote character. |
| 1045 next = advance(); | 1046 next = advance(); |
| 1046 appendSubstringToken(TokenType.STRING, start, asciiOnly); | 1047 appendSubstringToken(TokenType.STRING, start, asciiOnly); |
| 1047 return next; | 1048 return next; |
| 1048 } | 1049 } |
| (...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1095 int tokenizeSingleLineRawString(int next, int quoteChar, int start) { | 1096 int tokenizeSingleLineRawString(int next, int quoteChar, int start) { |
| 1096 bool asciiOnly = true; | 1097 bool asciiOnly = true; |
| 1097 while (next != $EOF) { | 1098 while (next != $EOF) { |
| 1098 if (identical(next, quoteChar)) { | 1099 if (identical(next, quoteChar)) { |
| 1099 if (!asciiOnly) handleUnicode(start); | 1100 if (!asciiOnly) handleUnicode(start); |
| 1100 next = advance(); | 1101 next = advance(); |
| 1101 appendSubstringToken(TokenType.STRING, start, asciiOnly); | 1102 appendSubstringToken(TokenType.STRING, start, asciiOnly); |
| 1102 return next; | 1103 return next; |
| 1103 } else if (identical(next, $LF) || identical(next, $CR)) { | 1104 } else if (identical(next, $LF) || identical(next, $CR)) { |
| 1104 if (!asciiOnly) handleUnicode(start); | 1105 if (!asciiOnly) handleUnicode(start); |
| 1105 return unterminatedString(quoteChar, start, | 1106 unterminatedString(quoteChar, start, |
| 1106 asciiOnly: asciiOnly, isMultiLine: false, isRaw: true); | 1107 asciiOnly: asciiOnly, isMultiLine: false, isRaw: true); |
| 1108 return next; |
| 1107 } else if (next > 127) { | 1109 } else if (next > 127) { |
| 1108 asciiOnly = false; | 1110 asciiOnly = false; |
| 1109 } | 1111 } |
| 1110 next = advance(); | 1112 next = advance(); |
| 1111 } | 1113 } |
| 1112 if (!asciiOnly) handleUnicode(start); | 1114 if (!asciiOnly) handleUnicode(start); |
| 1113 return unterminatedString(quoteChar, start, | 1115 unterminatedString(quoteChar, start, |
| 1114 asciiOnly: asciiOnly, isMultiLine: false, isRaw: true); | 1116 asciiOnly: asciiOnly, isMultiLine: false, isRaw: true); |
| 1117 return next; |
| 1115 } | 1118 } |
| 1116 | 1119 |
| 1117 int tokenizeMultiLineRawString(int quoteChar, int start) { | 1120 int tokenizeMultiLineRawString(int quoteChar, int start) { |
| 1118 bool asciiOnlyString = true; | 1121 bool asciiOnlyString = true; |
| 1119 bool asciiOnlyLine = true; | 1122 bool asciiOnlyLine = true; |
| 1120 int unicodeStart = start; | 1123 int unicodeStart = start; |
| 1121 int next = advance(); // Advance past the (last) quote (of three). | 1124 int next = advance(); // Advance past the (last) quote (of three). |
| 1122 outer: | 1125 outer: |
| 1123 while (!identical(next, $EOF)) { | 1126 while (!identical(next, $EOF)) { |
| 1124 while (!identical(next, quoteChar)) { | 1127 while (!identical(next, quoteChar)) { |
| (...skipping 17 matching lines...) Expand all Loading... |
| 1142 next = advance(); | 1145 next = advance(); |
| 1143 if (identical(next, quoteChar)) { | 1146 if (identical(next, quoteChar)) { |
| 1144 if (!asciiOnlyLine) handleUnicode(unicodeStart); | 1147 if (!asciiOnlyLine) handleUnicode(unicodeStart); |
| 1145 next = advance(); | 1148 next = advance(); |
| 1146 appendSubstringToken(TokenType.STRING, start, asciiOnlyString); | 1149 appendSubstringToken(TokenType.STRING, start, asciiOnlyString); |
| 1147 return next; | 1150 return next; |
| 1148 } | 1151 } |
| 1149 } | 1152 } |
| 1150 } | 1153 } |
| 1151 if (!asciiOnlyLine) handleUnicode(unicodeStart); | 1154 if (!asciiOnlyLine) handleUnicode(unicodeStart); |
| 1152 return unterminatedString(quoteChar, start, | 1155 unterminatedString(quoteChar, start, |
| 1153 asciiOnly: asciiOnlyLine, isMultiLine: true, isRaw: true); | 1156 asciiOnly: asciiOnlyLine, isMultiLine: true, isRaw: true); |
| 1157 return next; |
| 1154 } | 1158 } |
| 1155 | 1159 |
| 1156 int tokenizeMultiLineString(int quoteChar, int start, bool raw) { | 1160 int tokenizeMultiLineString(int quoteChar, int start, bool raw) { |
| 1157 if (raw) return tokenizeMultiLineRawString(quoteChar, start); | 1161 if (raw) return tokenizeMultiLineRawString(quoteChar, start); |
| 1158 bool asciiOnlyString = true; | 1162 bool asciiOnlyString = true; |
| 1159 bool asciiOnlyLine = true; | 1163 bool asciiOnlyLine = true; |
| 1160 int unicodeStart = start; | 1164 int unicodeStart = start; |
| 1161 int next = advance(); // Advance past the (last) quote (of three). | 1165 int next = advance(); // Advance past the (last) quote (of three). |
| 1162 while (!identical(next, $EOF)) { | 1166 while (!identical(next, $EOF)) { |
| 1163 if (identical(next, $$)) { | 1167 if (identical(next, $$)) { |
| (...skipping 30 matching lines...) Expand all Loading... |
| 1194 unicodeStart = scanOffset; | 1198 unicodeStart = scanOffset; |
| 1195 } | 1199 } |
| 1196 lineFeedInMultiline(); | 1200 lineFeedInMultiline(); |
| 1197 } else if (next > 127) { | 1201 } else if (next > 127) { |
| 1198 asciiOnlyString = false; | 1202 asciiOnlyString = false; |
| 1199 asciiOnlyLine = false; | 1203 asciiOnlyLine = false; |
| 1200 } | 1204 } |
| 1201 next = advance(); | 1205 next = advance(); |
| 1202 } | 1206 } |
| 1203 if (!asciiOnlyLine) handleUnicode(unicodeStart); | 1207 if (!asciiOnlyLine) handleUnicode(unicodeStart); |
| 1204 return unterminatedString(quoteChar, start, | 1208 unterminatedString(quoteChar, start, |
| 1205 asciiOnly: asciiOnlyString, isMultiLine: true, isRaw: false); | 1209 asciiOnly: asciiOnlyString, isMultiLine: true, isRaw: false); |
| 1210 return next; |
| 1206 } | 1211 } |
| 1207 | 1212 |
| 1208 int unexpected(int character) { | 1213 int unexpected(int character) { |
| 1209 appendErrorToken(buildUnexpectedCharacterToken(character, tokenStart)); | 1214 appendErrorToken(buildUnexpectedCharacterToken(character, tokenStart)); |
| 1210 return advanceAfterError(true); | 1215 return advanceAfterError(true); |
| 1211 } | 1216 } |
| 1212 | 1217 |
| 1213 int unterminated(String prefix, {bool shouldAdvance: true}) { | 1218 int unterminated(String prefix, {bool shouldAdvance: true}) { |
| 1214 appendErrorToken(new UnterminatedToken(prefix, tokenStart, stringOffset)); | 1219 appendErrorToken(new UnterminatedToken(prefix, tokenStart, stringOffset)); |
| 1215 return advanceAfterError(shouldAdvance); | 1220 return advanceAfterError(shouldAdvance); |
| 1216 } | 1221 } |
| 1217 | 1222 |
| 1218 int unterminatedString(int quoteChar, int start, | 1223 void unterminatedString(int quoteChar, int start, |
| 1219 {bool asciiOnly, bool isMultiLine, bool isRaw}) { | 1224 {bool asciiOnly, bool isMultiLine, bool isRaw}) { |
| 1220 String suffix = new String.fromCharCodes( | 1225 String suffix = new String.fromCharCodes( |
| 1221 isMultiLine ? [quoteChar, quoteChar, quoteChar] : [quoteChar]); | 1226 isMultiLine ? [quoteChar, quoteChar, quoteChar] : [quoteChar]); |
| 1222 String prefix = isRaw ? 'r$suffix' : suffix; | 1227 String prefix = isRaw ? 'r$suffix' : suffix; |
| 1223 | 1228 |
| 1224 appendSyntheticSubstringToken(TokenType.STRING, start, asciiOnly, suffix); | 1229 appendSyntheticSubstringToken(TokenType.STRING, start, asciiOnly, suffix); |
| 1225 beginToken(); | 1230 unterminated(prefix, shouldAdvance: false); |
| 1226 return unterminated(prefix); | |
| 1227 } | 1231 } |
| 1228 | 1232 |
| 1229 int advanceAfterError(bool shouldAdvance) { | 1233 int advanceAfterError(bool shouldAdvance) { |
| 1230 if (atEndOfFile()) return $EOF; | 1234 if (atEndOfFile()) return $EOF; |
| 1231 if (shouldAdvance) { | 1235 if (shouldAdvance) { |
| 1232 return advance(); // Ensure progress. | 1236 return advance(); // Ensure progress. |
| 1233 } else { | 1237 } else { |
| 1234 return -1; | 1238 return -1; |
| 1235 } | 1239 } |
| 1236 } | 1240 } |
| (...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1315 switchToUint32(newLength); | 1319 switchToUint32(newLength); |
| 1316 } | 1320 } |
| 1317 } | 1321 } |
| 1318 | 1322 |
| 1319 void switchToUint32(int newLength) { | 1323 void switchToUint32(int newLength) { |
| 1320 final newArray = new Uint32List(newLength); | 1324 final newArray = new Uint32List(newLength); |
| 1321 newArray.setRange(0, arrayLength, array); | 1325 newArray.setRange(0, arrayLength, array); |
| 1322 array = newArray; | 1326 array = newArray; |
| 1323 } | 1327 } |
| 1324 } | 1328 } |
| OLD | NEW |