| OLD | NEW |
| 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library engine.scanner_test; | 5 library analyzer.test.generated.scanner_test; |
| 6 | 6 |
| 7 import 'package:analyzer/src/generated/error.dart'; | 7 import 'package:analyzer/dart/ast/token.dart'; |
| 8 import 'package:analyzer/src/generated/scanner.dart'; | 8 import 'package:analyzer/error/error.dart'; |
| 9 import 'package:analyzer/error/listener.dart'; |
| 10 import 'package:analyzer/src/dart/ast/token.dart'; |
| 11 import 'package:analyzer/src/dart/scanner/reader.dart'; |
| 12 import 'package:analyzer/src/dart/scanner/scanner.dart'; |
| 9 import 'package:analyzer/src/generated/source.dart'; | 13 import 'package:analyzer/src/generated/source.dart'; |
| 14 import 'package:test_reflective_loader/test_reflective_loader.dart'; |
| 10 import 'package:unittest/unittest.dart'; | 15 import 'package:unittest/unittest.dart'; |
| 11 | 16 |
| 12 import '../reflective_tests.dart'; | |
| 13 import '../utils.dart'; | 17 import '../utils.dart'; |
| 14 import 'test_support.dart'; | 18 import 'test_support.dart'; |
| 15 | 19 |
| 16 main() { | 20 main() { |
| 17 initializeTestEnvironment(); | 21 initializeTestEnvironment(); |
| 18 runReflectiveTests(CharSequenceReaderTest); | 22 defineReflectiveTests(CharSequenceReaderTest); |
| 19 runReflectiveTests(KeywordStateTest); | 23 defineReflectiveTests(KeywordStateTest); |
| 20 runReflectiveTests(ScannerTest); | 24 defineReflectiveTests(ScannerTest); |
| 21 runReflectiveTests(TokenTypeTest); | 25 defineReflectiveTests(TokenTypeTest); |
| 22 } | 26 } |
| 23 | 27 |
| 24 class CharacterRangeReaderTest extends EngineTestCase { | 28 class CharacterRangeReaderTest extends EngineTestCase { |
| 25 void test_advance() { | 29 void test_advance() { |
| 26 CharSequenceReader baseReader = new CharSequenceReader("xyzzy"); | 30 CharSequenceReader baseReader = new CharSequenceReader("xyzzy"); |
| 27 CharacterRangeReader reader = new CharacterRangeReader(baseReader, 1, 4); | 31 CharacterRangeReader reader = new CharacterRangeReader(baseReader, 1, 4); |
| 28 expect(reader.advance(), 0x79); | 32 expect(reader.advance(), 0x79); |
| 29 expect(reader.advance(), 0x80); | 33 expect(reader.advance(), 0x80); |
| 30 expect(reader.advance(), 0x80); | 34 expect(reader.advance(), 0x80); |
| 31 expect(reader.advance(), -1); | 35 expect(reader.advance(), -1); |
| (...skipping 150 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 182 } | 186 } |
| 183 | 187 |
| 184 void test_ampersand() { | 188 void test_ampersand() { |
| 185 _assertToken(TokenType.AMPERSAND, "&"); | 189 _assertToken(TokenType.AMPERSAND, "&"); |
| 186 } | 190 } |
| 187 | 191 |
| 188 void test_ampersand_ampersand() { | 192 void test_ampersand_ampersand() { |
| 189 _assertToken(TokenType.AMPERSAND_AMPERSAND, "&&"); | 193 _assertToken(TokenType.AMPERSAND_AMPERSAND, "&&"); |
| 190 } | 194 } |
| 191 | 195 |
| 196 void test_ampersand_ampersand_eq() { |
| 197 _assertToken(TokenType.AMPERSAND_AMPERSAND_EQ, "&&=", |
| 198 lazyAssignmentOperators: true); |
| 199 } |
| 200 |
| 192 void test_ampersand_eq() { | 201 void test_ampersand_eq() { |
| 193 _assertToken(TokenType.AMPERSAND_EQ, "&="); | 202 _assertToken(TokenType.AMPERSAND_EQ, "&="); |
| 194 } | 203 } |
| 195 | 204 |
| 196 void test_at() { | 205 void test_at() { |
| 197 _assertToken(TokenType.AT, "@"); | 206 _assertToken(TokenType.AT, "@"); |
| 198 } | 207 } |
| 199 | 208 |
| 200 void test_backping() { | 209 void test_backping() { |
| 201 _assertToken(TokenType.BACKPING, "`"); | 210 _assertToken(TokenType.BACKPING, "`"); |
| (...skipping 12 matching lines...) Expand all Loading... |
| 214 } | 223 } |
| 215 | 224 |
| 216 void test_bar() { | 225 void test_bar() { |
| 217 _assertToken(TokenType.BAR, "|"); | 226 _assertToken(TokenType.BAR, "|"); |
| 218 } | 227 } |
| 219 | 228 |
| 220 void test_bar_bar() { | 229 void test_bar_bar() { |
| 221 _assertToken(TokenType.BAR_BAR, "||"); | 230 _assertToken(TokenType.BAR_BAR, "||"); |
| 222 } | 231 } |
| 223 | 232 |
| 233 void test_bar_bar_eq() { |
| 234 _assertToken(TokenType.BAR_BAR_EQ, "||=", lazyAssignmentOperators: true); |
| 235 } |
| 236 |
| 224 void test_bar_eq() { | 237 void test_bar_eq() { |
| 225 _assertToken(TokenType.BAR_EQ, "|="); | 238 _assertToken(TokenType.BAR_EQ, "|="); |
| 226 } | 239 } |
| 227 | 240 |
| 228 void test_caret() { | 241 void test_caret() { |
| 229 _assertToken(TokenType.CARET, "^"); | 242 _assertToken(TokenType.CARET, "^"); |
| 230 } | 243 } |
| 231 | 244 |
| 232 void test_caret_eq() { | 245 void test_caret_eq() { |
| 233 _assertToken(TokenType.CARET_EQ, "^="); | 246 _assertToken(TokenType.CARET_EQ, "^="); |
| (...skipping 23 matching lines...) Expand all Loading... |
| 257 Scanner scanner = new Scanner( | 270 Scanner scanner = new Scanner( |
| 258 null, | 271 null, |
| 259 new CharSequenceReader("/* comment */ "), | 272 new CharSequenceReader("/* comment */ "), |
| 260 AnalysisErrorListener.NULL_LISTENER); | 273 AnalysisErrorListener.NULL_LISTENER); |
| 261 scanner.preserveComments = false; | 274 scanner.preserveComments = false; |
| 262 Token token = scanner.tokenize(); | 275 Token token = scanner.tokenize(); |
| 263 expect(token, isNotNull); | 276 expect(token, isNotNull); |
| 264 expect(token.precedingComments, isNull); | 277 expect(token.precedingComments, isNull); |
| 265 } | 278 } |
| 266 | 279 |
| 280 void test_comment_generic_method_type_assign() { |
| 281 _assertComment(TokenType.MULTI_LINE_COMMENT, "/*=comment*/"); |
| 282 _assertComment(TokenType.GENERIC_METHOD_TYPE_ASSIGN, "/*=comment*/", |
| 283 genericMethodComments: true); |
| 284 } |
| 285 |
| 286 void test_comment_generic_method_type_list() { |
| 287 _assertComment(TokenType.MULTI_LINE_COMMENT, "/*<comment>*/"); |
| 288 _assertComment(TokenType.GENERIC_METHOD_TYPE_LIST, "/*<comment>*/", |
| 289 genericMethodComments: true); |
| 290 } |
| 291 |
| 267 void test_comment_multi() { | 292 void test_comment_multi() { |
| 268 _assertComment(TokenType.MULTI_LINE_COMMENT, "/* comment */"); | 293 _assertComment(TokenType.MULTI_LINE_COMMENT, "/* comment */"); |
| 269 } | 294 } |
| 270 | 295 |
| 271 void test_comment_multi_lineEnds() { | 296 void test_comment_multi_lineEnds() { |
| 272 String code = r''' | 297 String code = r''' |
| 273 /** | 298 /** |
| 274 * aa | 299 * aa |
| 275 * bbb | 300 * bbb |
| 276 * c | 301 * c |
| (...skipping 764 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1041 | 1066 |
| 1042 void test_tilde_slash_eq() { | 1067 void test_tilde_slash_eq() { |
| 1043 _assertToken(TokenType.TILDE_SLASH_EQ, "~/="); | 1068 _assertToken(TokenType.TILDE_SLASH_EQ, "~/="); |
| 1044 } | 1069 } |
| 1045 | 1070 |
| 1046 void test_unclosedPairInInterpolation() { | 1071 void test_unclosedPairInInterpolation() { |
| 1047 GatheringErrorListener listener = new GatheringErrorListener(); | 1072 GatheringErrorListener listener = new GatheringErrorListener(); |
| 1048 _scanWithListener("'\${(}'", listener); | 1073 _scanWithListener("'\${(}'", listener); |
| 1049 } | 1074 } |
| 1050 | 1075 |
| 1051 void _assertComment(TokenType commentType, String source) { | 1076 void _assertComment(TokenType commentType, String source, |
| 1077 {bool genericMethodComments: false}) { |
| 1052 // | 1078 // |
| 1053 // Test without a trailing end-of-line marker | 1079 // Test without a trailing end-of-line marker |
| 1054 // | 1080 // |
| 1055 Token token = _scan(source); | 1081 Token token = _scan(source, genericMethodComments: genericMethodComments); |
| 1056 expect(token, isNotNull); | 1082 expect(token, isNotNull); |
| 1057 expect(token.type, TokenType.EOF); | 1083 expect(token.type, TokenType.EOF); |
| 1058 Token comment = token.precedingComments; | 1084 Token comment = token.precedingComments; |
| 1059 expect(comment, isNotNull); | 1085 expect(comment, isNotNull); |
| 1060 expect(comment.type, commentType); | 1086 expect(comment.type, commentType); |
| 1061 expect(comment.offset, 0); | 1087 expect(comment.offset, 0); |
| 1062 expect(comment.length, source.length); | 1088 expect(comment.length, source.length); |
| 1063 expect(comment.lexeme, source); | 1089 expect(comment.lexeme, source); |
| 1064 // | 1090 // |
| 1065 // Test with a trailing end-of-line marker | 1091 // Test with a trailing end-of-line marker |
| 1066 // | 1092 // |
| 1067 token = _scan("$source\n"); | 1093 token = _scan("$source\n", genericMethodComments: genericMethodComments); |
| 1068 expect(token, isNotNull); | 1094 expect(token, isNotNull); |
| 1069 expect(token.type, TokenType.EOF); | 1095 expect(token.type, TokenType.EOF); |
| 1070 comment = token.precedingComments; | 1096 comment = token.precedingComments; |
| 1071 expect(comment, isNotNull); | 1097 expect(comment, isNotNull); |
| 1072 expect(comment.type, commentType); | 1098 expect(comment.type, commentType); |
| 1073 expect(comment.offset, 0); | 1099 expect(comment.offset, 0); |
| 1074 expect(comment.length, source.length); | 1100 expect(comment.length, source.length); |
| 1075 expect(comment.lexeme, source); | 1101 expect(comment.lexeme, source); |
| 1076 } | 1102 } |
| 1077 | 1103 |
| (...skipping 76 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1154 reason: 'Line number in location $i'); | 1180 reason: 'Line number in location $i'); |
| 1155 expect(location.columnNumber, expectedLocation._columnNumber, | 1181 expect(location.columnNumber, expectedLocation._columnNumber, |
| 1156 reason: 'Column number in location $i'); | 1182 reason: 'Column number in location $i'); |
| 1157 } | 1183 } |
| 1158 } | 1184 } |
| 1159 | 1185 |
| 1160 /** | 1186 /** |
| 1161 * Assert that the token scanned from the given [source] has the | 1187 * Assert that the token scanned from the given [source] has the |
| 1162 * [expectedType]. | 1188 * [expectedType]. |
| 1163 */ | 1189 */ |
| 1164 Token _assertToken(TokenType expectedType, String source) { | 1190 Token _assertToken(TokenType expectedType, String source, |
| 1165 Token originalToken = _scan(source); | 1191 {bool lazyAssignmentOperators: false}) { |
| 1192 Token originalToken = |
| 1193 _scan(source, lazyAssignmentOperators: lazyAssignmentOperators); |
| 1166 expect(originalToken, isNotNull); | 1194 expect(originalToken, isNotNull); |
| 1167 expect(originalToken.type, expectedType); | 1195 expect(originalToken.type, expectedType); |
| 1168 expect(originalToken.offset, 0); | 1196 expect(originalToken.offset, 0); |
| 1169 expect(originalToken.length, source.length); | 1197 expect(originalToken.length, source.length); |
| 1170 expect(originalToken.lexeme, source); | 1198 expect(originalToken.lexeme, source); |
| 1171 if (expectedType == TokenType.SCRIPT_TAG) { | 1199 if (expectedType == TokenType.SCRIPT_TAG) { |
| 1172 // Adding space before the script tag is not allowed, and adding text at | 1200 // Adding space before the script tag is not allowed, and adding text at |
| 1173 // the end changes nothing. | 1201 // the end changes nothing. |
| 1174 return originalToken; | 1202 return originalToken; |
| 1175 } else if (expectedType == TokenType.SINGLE_LINE_COMMENT) { | 1203 } else if (expectedType == TokenType.SINGLE_LINE_COMMENT) { |
| 1176 // Adding space to an end-of-line comment changes the comment. | 1204 // Adding space to an end-of-line comment changes the comment. |
| 1177 Token tokenWithSpaces = _scan(" $source"); | 1205 Token tokenWithSpaces = |
| 1206 _scan(" $source", lazyAssignmentOperators: lazyAssignmentOperators); |
| 1178 expect(tokenWithSpaces, isNotNull); | 1207 expect(tokenWithSpaces, isNotNull); |
| 1179 expect(tokenWithSpaces.type, expectedType); | 1208 expect(tokenWithSpaces.type, expectedType); |
| 1180 expect(tokenWithSpaces.offset, 1); | 1209 expect(tokenWithSpaces.offset, 1); |
| 1181 expect(tokenWithSpaces.length, source.length); | 1210 expect(tokenWithSpaces.length, source.length); |
| 1182 expect(tokenWithSpaces.lexeme, source); | 1211 expect(tokenWithSpaces.lexeme, source); |
| 1183 return originalToken; | 1212 return originalToken; |
| 1184 } else if (expectedType == TokenType.INT || | 1213 } else if (expectedType == TokenType.INT || |
| 1185 expectedType == TokenType.DOUBLE) { | 1214 expectedType == TokenType.DOUBLE) { |
| 1186 Token tokenWithLowerD = _scan("${source}d"); | 1215 Token tokenWithLowerD = |
| 1216 _scan("${source}d", lazyAssignmentOperators: lazyAssignmentOperators); |
| 1187 expect(tokenWithLowerD, isNotNull); | 1217 expect(tokenWithLowerD, isNotNull); |
| 1188 expect(tokenWithLowerD.type, expectedType); | 1218 expect(tokenWithLowerD.type, expectedType); |
| 1189 expect(tokenWithLowerD.offset, 0); | 1219 expect(tokenWithLowerD.offset, 0); |
| 1190 expect(tokenWithLowerD.length, source.length); | 1220 expect(tokenWithLowerD.length, source.length); |
| 1191 expect(tokenWithLowerD.lexeme, source); | 1221 expect(tokenWithLowerD.lexeme, source); |
| 1192 Token tokenWithUpperD = _scan("${source}D"); | 1222 Token tokenWithUpperD = |
| 1223 _scan("${source}D", lazyAssignmentOperators: lazyAssignmentOperators); |
| 1193 expect(tokenWithUpperD, isNotNull); | 1224 expect(tokenWithUpperD, isNotNull); |
| 1194 expect(tokenWithUpperD.type, expectedType); | 1225 expect(tokenWithUpperD.type, expectedType); |
| 1195 expect(tokenWithUpperD.offset, 0); | 1226 expect(tokenWithUpperD.offset, 0); |
| 1196 expect(tokenWithUpperD.length, source.length); | 1227 expect(tokenWithUpperD.length, source.length); |
| 1197 expect(tokenWithUpperD.lexeme, source); | 1228 expect(tokenWithUpperD.lexeme, source); |
| 1198 } | 1229 } |
| 1199 Token tokenWithSpaces = _scan(" $source "); | 1230 Token tokenWithSpaces = |
| 1231 _scan(" $source ", lazyAssignmentOperators: lazyAssignmentOperators); |
| 1200 expect(tokenWithSpaces, isNotNull); | 1232 expect(tokenWithSpaces, isNotNull); |
| 1201 expect(tokenWithSpaces.type, expectedType); | 1233 expect(tokenWithSpaces.type, expectedType); |
| 1202 expect(tokenWithSpaces.offset, 1); | 1234 expect(tokenWithSpaces.offset, 1); |
| 1203 expect(tokenWithSpaces.length, source.length); | 1235 expect(tokenWithSpaces.length, source.length); |
| 1204 expect(tokenWithSpaces.lexeme, source); | 1236 expect(tokenWithSpaces.lexeme, source); |
| 1205 expect(originalToken.next.type, TokenType.EOF); | 1237 expect(originalToken.next.type, TokenType.EOF); |
| 1206 return originalToken; | 1238 return originalToken; |
| 1207 } | 1239 } |
| 1208 | 1240 |
| 1209 /** | 1241 /** |
| (...skipping 16 matching lines...) Expand all Loading... |
| 1226 expect(token.length, expectedToken.length, | 1258 expect(token.length, expectedToken.length, |
| 1227 reason: "Wrong length for token $i"); | 1259 reason: "Wrong length for token $i"); |
| 1228 expect(token.lexeme, expectedToken.lexeme, | 1260 expect(token.lexeme, expectedToken.lexeme, |
| 1229 reason: "Wrong lexeme for token $i"); | 1261 reason: "Wrong lexeme for token $i"); |
| 1230 token = token.next; | 1262 token = token.next; |
| 1231 expect(token, isNotNull); | 1263 expect(token, isNotNull); |
| 1232 } | 1264 } |
| 1233 expect(token.type, TokenType.EOF); | 1265 expect(token.type, TokenType.EOF); |
| 1234 } | 1266 } |
| 1235 | 1267 |
| 1236 Token _scan(String source) { | 1268 Token _scan(String source, |
| 1269 {bool genericMethodComments: false, |
| 1270 bool lazyAssignmentOperators: false}) { |
| 1237 GatheringErrorListener listener = new GatheringErrorListener(); | 1271 GatheringErrorListener listener = new GatheringErrorListener(); |
| 1238 Token token = _scanWithListener(source, listener); | 1272 Token token = _scanWithListener(source, listener, |
| 1273 genericMethodComments: genericMethodComments, |
| 1274 lazyAssignmentOperators: lazyAssignmentOperators); |
| 1239 listener.assertNoErrors(); | 1275 listener.assertNoErrors(); |
| 1240 return token; | 1276 return token; |
| 1241 } | 1277 } |
| 1242 | 1278 |
| 1243 Token _scanWithListener(String source, GatheringErrorListener listener) { | 1279 Token _scanWithListener(String source, GatheringErrorListener listener, |
| 1280 {bool genericMethodComments: false, |
| 1281 bool lazyAssignmentOperators: false}) { |
| 1244 Scanner scanner = | 1282 Scanner scanner = |
| 1245 new Scanner(null, new CharSequenceReader(source), listener); | 1283 new Scanner(null, new CharSequenceReader(source), listener); |
| 1284 scanner.scanGenericMethodComments = genericMethodComments; |
| 1285 scanner.scanLazyAssignmentOperators = lazyAssignmentOperators; |
| 1246 Token result = scanner.tokenize(); | 1286 Token result = scanner.tokenize(); |
| 1247 listener.setLineInfo(new TestSource(), scanner.lineStarts); | 1287 listener.setLineInfo(new TestSource(), scanner.lineStarts); |
| 1248 return result; | 1288 return result; |
| 1249 } | 1289 } |
| 1250 } | 1290 } |
| 1251 | 1291 |
| 1252 /** | 1292 /** |
| 1253 * An `ExpectedLocation` encodes information about the expected location of a | 1293 * An `ExpectedLocation` encodes information about the expected location of a |
| 1254 * given offset in source code. | 1294 * given offset in source code. |
| 1255 */ | 1295 */ |
| (...skipping 134 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1390 expect(TokenType.LT_LT.isUserDefinableOperator, isTrue); | 1430 expect(TokenType.LT_LT.isUserDefinableOperator, isTrue); |
| 1391 expect(TokenType.MINUS.isUserDefinableOperator, isTrue); | 1431 expect(TokenType.MINUS.isUserDefinableOperator, isTrue); |
| 1392 expect(TokenType.PERCENT.isUserDefinableOperator, isTrue); | 1432 expect(TokenType.PERCENT.isUserDefinableOperator, isTrue); |
| 1393 expect(TokenType.PLUS.isUserDefinableOperator, isTrue); | 1433 expect(TokenType.PLUS.isUserDefinableOperator, isTrue); |
| 1394 expect(TokenType.SLASH.isUserDefinableOperator, isTrue); | 1434 expect(TokenType.SLASH.isUserDefinableOperator, isTrue); |
| 1395 expect(TokenType.STAR.isUserDefinableOperator, isTrue); | 1435 expect(TokenType.STAR.isUserDefinableOperator, isTrue); |
| 1396 expect(TokenType.TILDE.isUserDefinableOperator, isTrue); | 1436 expect(TokenType.TILDE.isUserDefinableOperator, isTrue); |
| 1397 expect(TokenType.TILDE_SLASH.isUserDefinableOperator, isTrue); | 1437 expect(TokenType.TILDE_SLASH.isUserDefinableOperator, isTrue); |
| 1398 } | 1438 } |
| 1399 } | 1439 } |
| OLD | NEW |