OLD | NEW |
1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 library engine.scanner_test; | 5 library engine.scanner_test; |
6 | 6 |
7 import 'package:analyzer/src/generated/error.dart'; | 7 import 'package:analyzer/src/generated/error.dart'; |
8 import 'package:analyzer/src/generated/scanner.dart'; | 8 import 'package:analyzer/src/generated/scanner.dart'; |
9 import 'package:analyzer/src/generated/source.dart'; | 9 import 'package:analyzer/src/generated/source.dart'; |
10 import 'package:unittest/unittest.dart'; | 10 import 'package:unittest/unittest.dart'; |
(...skipping 154 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
165 expect(state, isNotNull); | 165 expect(state, isNotNull); |
166 } | 166 } |
167 } | 167 } |
168 } | 168 } |
169 } | 169 } |
170 | 170 |
171 @reflectiveTest | 171 @reflectiveTest |
172 class ScannerTest { | 172 class ScannerTest { |
173 void fail_incomplete_string_interpolation() { | 173 void fail_incomplete_string_interpolation() { |
174 // https://code.google.com/p/dart/issues/detail?id=18073 | 174 // https://code.google.com/p/dart/issues/detail?id=18073 |
175 _assertErrorAndTokens( | 175 _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, 9, |
176 ScannerErrorCode.UNTERMINATED_STRING_LITERAL, | 176 "\"foo \${bar", [ |
177 9, | 177 new StringToken(TokenType.STRING, "\"foo ", 0), |
178 "\"foo \${bar", | 178 new StringToken(TokenType.STRING_INTERPOLATION_EXPRESSION, "\${", 5), |
179 [ | 179 new StringToken(TokenType.IDENTIFIER, "bar", 7) |
180 new StringToken(TokenType.STRING, "\"foo ", 0), | 180 ]); |
181 new StringToken(TokenType.STRING_INTERPOLATION_EXPRESSION, "\${", 5)
, | |
182 new StringToken(TokenType.IDENTIFIER, "bar", 7)]); | |
183 } | 181 } |
184 | 182 |
185 void test_ampersand() { | 183 void test_ampersand() { |
186 _assertToken(TokenType.AMPERSAND, "&"); | 184 _assertToken(TokenType.AMPERSAND, "&"); |
187 } | 185 } |
188 | 186 |
189 void test_ampersand_ampersand() { | 187 void test_ampersand_ampersand() { |
190 _assertToken(TokenType.AMPERSAND_AMPERSAND, "&&"); | 188 _assertToken(TokenType.AMPERSAND_AMPERSAND, "&&"); |
191 } | 189 } |
192 | 190 |
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
248 | 246 |
249 void test_colon() { | 247 void test_colon() { |
250 _assertToken(TokenType.COLON, ":"); | 248 _assertToken(TokenType.COLON, ":"); |
251 } | 249 } |
252 | 250 |
253 void test_comma() { | 251 void test_comma() { |
254 _assertToken(TokenType.COMMA, ","); | 252 _assertToken(TokenType.COMMA, ","); |
255 } | 253 } |
256 | 254 |
257 void test_comment_disabled_multi() { | 255 void test_comment_disabled_multi() { |
258 Scanner scanner = new Scanner( | 256 Scanner scanner = new Scanner(null, |
259 null, | |
260 new CharSequenceReader("/* comment */ "), | 257 new CharSequenceReader("/* comment */ "), |
261 AnalysisErrorListener.NULL_LISTENER); | 258 AnalysisErrorListener.NULL_LISTENER); |
262 scanner.preserveComments = false; | 259 scanner.preserveComments = false; |
263 Token token = scanner.tokenize(); | 260 Token token = scanner.tokenize(); |
264 expect(token, isNotNull); | 261 expect(token, isNotNull); |
265 expect(token.precedingComments, isNull); | 262 expect(token.precedingComments, isNull); |
266 } | 263 } |
267 | 264 |
268 void test_comment_multi() { | 265 void test_comment_multi() { |
269 _assertComment(TokenType.MULTI_LINE_COMMENT, "/* comment */"); | 266 _assertComment(TokenType.MULTI_LINE_COMMENT, "/* comment */"); |
270 } | 267 } |
271 | 268 |
272 void test_comment_multi_unterminated() { | 269 void test_comment_multi_unterminated() { |
273 _assertError(ScannerErrorCode.UNTERMINATED_MULTI_LINE_COMMENT, 3, "/* x"); | 270 _assertError(ScannerErrorCode.UNTERMINATED_MULTI_LINE_COMMENT, 3, "/* x"); |
274 } | 271 } |
275 | 272 |
276 void test_comment_nested() { | 273 void test_comment_nested() { |
277 _assertComment( | 274 _assertComment( |
278 TokenType.MULTI_LINE_COMMENT, | 275 TokenType.MULTI_LINE_COMMENT, "/* comment /* within a */ comment */"); |
279 "/* comment /* within a */ comment */"); | |
280 } | 276 } |
281 | 277 |
282 void test_comment_single() { | 278 void test_comment_single() { |
283 _assertComment(TokenType.SINGLE_LINE_COMMENT, "// comment"); | 279 _assertComment(TokenType.SINGLE_LINE_COMMENT, "// comment"); |
284 } | 280 } |
285 | 281 |
286 void test_double_both_E() { | 282 void test_double_both_E() { |
287 _assertToken(TokenType.DOUBLE, "0.123E4"); | 283 _assertToken(TokenType.DOUBLE, "0.123E4"); |
288 } | 284 } |
289 | 285 |
(...skipping 284 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
574 void test_keyword_while() { | 570 void test_keyword_while() { |
575 _assertKeywordToken("while"); | 571 _assertKeywordToken("while"); |
576 } | 572 } |
577 | 573 |
578 void test_keyword_with() { | 574 void test_keyword_with() { |
579 _assertKeywordToken("with"); | 575 _assertKeywordToken("with"); |
580 } | 576 } |
581 | 577 |
582 void test_lineInfo_multilineComment() { | 578 void test_lineInfo_multilineComment() { |
583 String source = "/*\r *\r */"; | 579 String source = "/*\r *\r */"; |
584 _assertLineInfo( | 580 _assertLineInfo(source, [ |
585 source, | 581 new ScannerTest_ExpectedLocation(0, 1, 1), |
586 [ | 582 new ScannerTest_ExpectedLocation(4, 2, 2), |
587 new ScannerTest_ExpectedLocation(0, 1, 1), | 583 new ScannerTest_ExpectedLocation(source.length - 1, 3, 3) |
588 new ScannerTest_ExpectedLocation(4, 2, 2), | 584 ]); |
589 new ScannerTest_ExpectedLocation(source.length - 1, 3, 3)]); | |
590 } | 585 } |
591 | 586 |
592 void test_lineInfo_multilineString() { | 587 void test_lineInfo_multilineString() { |
593 String source = "'''a\r\nbc\r\nd'''"; | 588 String source = "'''a\r\nbc\r\nd'''"; |
594 _assertLineInfo( | 589 _assertLineInfo(source, [ |
595 source, | 590 new ScannerTest_ExpectedLocation(0, 1, 1), |
596 [ | 591 new ScannerTest_ExpectedLocation(7, 2, 2), |
597 new ScannerTest_ExpectedLocation(0, 1, 1), | 592 new ScannerTest_ExpectedLocation(source.length - 1, 3, 4) |
598 new ScannerTest_ExpectedLocation(7, 2, 2), | 593 ]); |
599 new ScannerTest_ExpectedLocation(source.length - 1, 3, 4)]); | |
600 } | 594 } |
601 | 595 |
602 void test_lineInfo_simpleClass() { | 596 void test_lineInfo_simpleClass() { |
603 String source = | 597 String source = |
604 "class Test {\r\n String s = '...';\r\n int get x => s.MISSING_GET
TER;\r\n}"; | 598 "class Test {\r\n String s = '...';\r\n int get x => s.MISSING_GET
TER;\r\n}"; |
605 _assertLineInfo( | 599 _assertLineInfo(source, [ |
606 source, | 600 new ScannerTest_ExpectedLocation(0, 1, 1), |
607 [ | 601 new ScannerTest_ExpectedLocation(source.indexOf("MISSING_GETTER"), 3, 20), |
608 new ScannerTest_ExpectedLocation(0, 1, 1), | 602 new ScannerTest_ExpectedLocation(source.length - 1, 4, 1) |
609 new ScannerTest_ExpectedLocation(source.indexOf("MISSING_GETTER"), 3
, 20), | 603 ]); |
610 new ScannerTest_ExpectedLocation(source.length - 1, 4, 1)]); | |
611 } | 604 } |
612 | 605 |
613 void test_lineInfo_slashN() { | 606 void test_lineInfo_slashN() { |
614 String source = "class Test {\n}"; | 607 String source = "class Test {\n}"; |
615 _assertLineInfo( | 608 _assertLineInfo(source, [ |
616 source, | 609 new ScannerTest_ExpectedLocation(0, 1, 1), |
617 [ | 610 new ScannerTest_ExpectedLocation(source.indexOf("}"), 2, 1) |
618 new ScannerTest_ExpectedLocation(0, 1, 1), | 611 ]); |
619 new ScannerTest_ExpectedLocation(source.indexOf("}"), 2, 1)]); | |
620 } | 612 } |
621 | 613 |
622 void test_lt() { | 614 void test_lt() { |
623 _assertToken(TokenType.LT, "<"); | 615 _assertToken(TokenType.LT, "<"); |
624 } | 616 } |
625 | 617 |
626 void test_lt_eq() { | 618 void test_lt_eq() { |
627 _assertToken(TokenType.LT_EQ, "<="); | 619 _assertToken(TokenType.LT_EQ, "<="); |
628 } | 620 } |
629 | 621 |
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
669 | 661 |
670 void test_percent_eq() { | 662 void test_percent_eq() { |
671 _assertToken(TokenType.PERCENT_EQ, "%="); | 663 _assertToken(TokenType.PERCENT_EQ, "%="); |
672 } | 664 } |
673 | 665 |
674 void test_period() { | 666 void test_period() { |
675 _assertToken(TokenType.PERIOD, "."); | 667 _assertToken(TokenType.PERIOD, "."); |
676 } | 668 } |
677 | 669 |
678 void test_periodAfterNumberNotIncluded_identifier() { | 670 void test_periodAfterNumberNotIncluded_identifier() { |
679 _assertTokens( | 671 _assertTokens("42.isEven()", [ |
680 "42.isEven()", | 672 new StringToken(TokenType.INT, "42", 0), |
681 [ | 673 new Token(TokenType.PERIOD, 2), |
682 new StringToken(TokenType.INT, "42", 0), | 674 new StringToken(TokenType.IDENTIFIER, "isEven", 3), |
683 new Token(TokenType.PERIOD, 2), | 675 new Token(TokenType.OPEN_PAREN, 9), |
684 new StringToken(TokenType.IDENTIFIER, "isEven", 3), | 676 new Token(TokenType.CLOSE_PAREN, 10) |
685 new Token(TokenType.OPEN_PAREN, 9), | 677 ]); |
686 new Token(TokenType.CLOSE_PAREN, 10)]); | |
687 } | 678 } |
688 | 679 |
689 void test_periodAfterNumberNotIncluded_period() { | 680 void test_periodAfterNumberNotIncluded_period() { |
690 _assertTokens( | 681 _assertTokens("42..isEven()", [ |
691 "42..isEven()", | 682 new StringToken(TokenType.INT, "42", 0), |
692 [ | 683 new Token(TokenType.PERIOD_PERIOD, 2), |
693 new StringToken(TokenType.INT, "42", 0), | 684 new StringToken(TokenType.IDENTIFIER, "isEven", 4), |
694 new Token(TokenType.PERIOD_PERIOD, 2), | 685 new Token(TokenType.OPEN_PAREN, 10), |
695 new StringToken(TokenType.IDENTIFIER, "isEven", 4), | 686 new Token(TokenType.CLOSE_PAREN, 11) |
696 new Token(TokenType.OPEN_PAREN, 10), | 687 ]); |
697 new Token(TokenType.CLOSE_PAREN, 11)]); | |
698 } | 688 } |
699 | 689 |
700 void test_period_period() { | 690 void test_period_period() { |
701 _assertToken(TokenType.PERIOD_PERIOD, ".."); | 691 _assertToken(TokenType.PERIOD_PERIOD, ".."); |
702 } | 692 } |
703 | 693 |
704 void test_period_period_period() { | 694 void test_period_period_period() { |
705 _assertToken(TokenType.PERIOD_PERIOD_PERIOD, "..."); | 695 _assertToken(TokenType.PERIOD_PERIOD_PERIOD, "..."); |
706 } | 696 } |
707 | 697 |
(...skipping 74 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
782 | 772 |
783 void test_string_multi_embeddedQuotes() { | 773 void test_string_multi_embeddedQuotes() { |
784 _assertToken(TokenType.STRING, "\"\"\"line1\n\"\"\nline2\"\"\""); | 774 _assertToken(TokenType.STRING, "\"\"\"line1\n\"\"\nline2\"\"\""); |
785 } | 775 } |
786 | 776 |
787 void test_string_multi_embeddedQuotes_escapedChar() { | 777 void test_string_multi_embeddedQuotes_escapedChar() { |
788 _assertToken(TokenType.STRING, "\"\"\"a\"\"\\tb\"\"\""); | 778 _assertToken(TokenType.STRING, "\"\"\"a\"\"\\tb\"\"\""); |
789 } | 779 } |
790 | 780 |
791 void test_string_multi_interpolation_block() { | 781 void test_string_multi_interpolation_block() { |
792 _assertTokens( | 782 _assertTokens("\"Hello \${name}!\"", [ |
793 "\"Hello \${name}!\"", | 783 new StringToken(TokenType.STRING, "\"Hello ", 0), |
794 [ | 784 new StringToken(TokenType.STRING_INTERPOLATION_EXPRESSION, "\${", 7), |
795 new StringToken(TokenType.STRING, "\"Hello ", 0), | 785 new StringToken(TokenType.IDENTIFIER, "name", 9), |
796 new StringToken(TokenType.STRING_INTERPOLATION_EXPRESSION, "\${", 7)
, | 786 new Token(TokenType.CLOSE_CURLY_BRACKET, 13), |
797 new StringToken(TokenType.IDENTIFIER, "name", 9), | 787 new StringToken(TokenType.STRING, "!\"", 14) |
798 new Token(TokenType.CLOSE_CURLY_BRACKET, 13), | 788 ]); |
799 new StringToken(TokenType.STRING, "!\"", 14)]); | |
800 } | 789 } |
801 | 790 |
802 void test_string_multi_interpolation_identifier() { | 791 void test_string_multi_interpolation_identifier() { |
803 _assertTokens( | 792 _assertTokens("\"Hello \$name!\"", [ |
804 "\"Hello \$name!\"", | 793 new StringToken(TokenType.STRING, "\"Hello ", 0), |
805 [ | 794 new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 7), |
806 new StringToken(TokenType.STRING, "\"Hello ", 0), | 795 new StringToken(TokenType.IDENTIFIER, "name", 8), |
807 new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 7), | 796 new StringToken(TokenType.STRING, "!\"", 12) |
808 new StringToken(TokenType.IDENTIFIER, "name", 8), | 797 ]); |
809 new StringToken(TokenType.STRING, "!\"", 12)]); | |
810 } | 798 } |
811 | 799 |
812 void test_string_multi_single() { | 800 void test_string_multi_single() { |
813 _assertToken(TokenType.STRING, "'''string'''"); | 801 _assertToken(TokenType.STRING, "'''string'''"); |
814 } | 802 } |
815 | 803 |
816 void test_string_multi_slashEnter() { | 804 void test_string_multi_slashEnter() { |
817 _assertToken(TokenType.STRING, "'''\\\n'''"); | 805 _assertToken(TokenType.STRING, "'''\\\n'''"); |
818 } | 806 } |
819 | 807 |
820 void test_string_multi_unterminated() { | 808 void test_string_multi_unterminated() { |
821 _assertErrorAndTokens( | 809 _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, 8, |
822 ScannerErrorCode.UNTERMINATED_STRING_LITERAL, | 810 "'''string", [new StringToken(TokenType.STRING, "'''string", 0)]); |
823 8, | |
824 "'''string", | |
825 [new StringToken(TokenType.STRING, "'''string", 0)]); | |
826 } | 811 } |
827 | 812 |
828 void test_string_multi_unterminated_interpolation_block() { | 813 void test_string_multi_unterminated_interpolation_block() { |
829 _assertErrorAndTokens( | 814 _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, 8, |
830 ScannerErrorCode.UNTERMINATED_STRING_LITERAL, | 815 "'''\${name", [ |
831 8, | 816 new StringToken(TokenType.STRING, "'''", 0), |
832 "'''\${name", | 817 new StringToken(TokenType.STRING_INTERPOLATION_EXPRESSION, "\${", 3), |
833 [ | 818 new StringToken(TokenType.IDENTIFIER, "name", 5), |
834 new StringToken(TokenType.STRING, "'''", 0), | 819 new StringToken(TokenType.STRING, "", 9) |
835 new StringToken(TokenType.STRING_INTERPOLATION_EXPRESSION, "\${", 3)
, | 820 ]); |
836 new StringToken(TokenType.IDENTIFIER, "name", 5), | |
837 new StringToken(TokenType.STRING, "", 9)]); | |
838 } | 821 } |
839 | 822 |
840 void test_string_multi_unterminated_interpolation_identifier() { | 823 void test_string_multi_unterminated_interpolation_identifier() { |
841 _assertErrorAndTokens( | 824 _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, 7, |
842 ScannerErrorCode.UNTERMINATED_STRING_LITERAL, | 825 "'''\$name", [ |
843 7, | 826 new StringToken(TokenType.STRING, "'''", 0), |
844 "'''\$name", | 827 new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 3), |
845 [ | 828 new StringToken(TokenType.IDENTIFIER, "name", 4), |
846 new StringToken(TokenType.STRING, "'''", 0), | 829 new StringToken(TokenType.STRING, "", 8) |
847 new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 3), | 830 ]); |
848 new StringToken(TokenType.IDENTIFIER, "name", 4), | |
849 new StringToken(TokenType.STRING, "", 8)]); | |
850 } | 831 } |
851 | 832 |
852 void test_string_raw_multi_double() { | 833 void test_string_raw_multi_double() { |
853 _assertToken(TokenType.STRING, "r\"\"\"line1\nline2\"\"\""); | 834 _assertToken(TokenType.STRING, "r\"\"\"line1\nline2\"\"\""); |
854 } | 835 } |
855 | 836 |
856 void test_string_raw_multi_single() { | 837 void test_string_raw_multi_single() { |
857 _assertToken(TokenType.STRING, "r'''string'''"); | 838 _assertToken(TokenType.STRING, "r'''string'''"); |
858 } | 839 } |
859 | 840 |
860 void test_string_raw_multi_unterminated() { | 841 void test_string_raw_multi_unterminated() { |
861 String source = "r'''string"; | 842 String source = "r'''string"; |
862 _assertErrorAndTokens( | 843 _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, |
863 ScannerErrorCode.UNTERMINATED_STRING_LITERAL, | 844 9, source, [new StringToken(TokenType.STRING, source, 0)]); |
864 9, | |
865 source, | |
866 [new StringToken(TokenType.STRING, source, 0)]); | |
867 } | 845 } |
868 | 846 |
869 void test_string_raw_simple_double() { | 847 void test_string_raw_simple_double() { |
870 _assertToken(TokenType.STRING, "r\"string\""); | 848 _assertToken(TokenType.STRING, "r\"string\""); |
871 } | 849 } |
872 | 850 |
873 void test_string_raw_simple_single() { | 851 void test_string_raw_simple_single() { |
874 _assertToken(TokenType.STRING, "r'string'"); | 852 _assertToken(TokenType.STRING, "r'string'"); |
875 } | 853 } |
876 | 854 |
877 void test_string_raw_simple_unterminated_eof() { | 855 void test_string_raw_simple_unterminated_eof() { |
878 String source = "r'string"; | 856 String source = "r'string"; |
879 _assertErrorAndTokens( | 857 _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, |
880 ScannerErrorCode.UNTERMINATED_STRING_LITERAL, | 858 7, source, [new StringToken(TokenType.STRING, source, 0)]); |
881 7, | |
882 source, | |
883 [new StringToken(TokenType.STRING, source, 0)]); | |
884 } | 859 } |
885 | 860 |
886 void test_string_raw_simple_unterminated_eol() { | 861 void test_string_raw_simple_unterminated_eol() { |
887 String source = "r'string"; | 862 String source = "r'string"; |
888 _assertErrorAndTokens( | 863 _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, |
889 ScannerErrorCode.UNTERMINATED_STRING_LITERAL, | 864 8, "$source\n", [new StringToken(TokenType.STRING, source, 0)]); |
890 8, | |
891 "$source\n", | |
892 [new StringToken(TokenType.STRING, source, 0)]); | |
893 } | 865 } |
894 | 866 |
895 void test_string_simple_double() { | 867 void test_string_simple_double() { |
896 _assertToken(TokenType.STRING, "\"string\""); | 868 _assertToken(TokenType.STRING, "\"string\""); |
897 } | 869 } |
898 | 870 |
899 void test_string_simple_escapedDollar() { | 871 void test_string_simple_escapedDollar() { |
900 _assertToken(TokenType.STRING, "'a\\\$b'"); | 872 _assertToken(TokenType.STRING, "'a\\\$b'"); |
901 } | 873 } |
902 | 874 |
903 void test_string_simple_interpolation_adjacentIdentifiers() { | 875 void test_string_simple_interpolation_adjacentIdentifiers() { |
904 _assertTokens( | 876 _assertTokens("'\$a\$b'", [ |
905 "'\$a\$b'", | 877 new StringToken(TokenType.STRING, "'", 0), |
906 [ | 878 new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 1), |
907 new StringToken(TokenType.STRING, "'", 0), | 879 new StringToken(TokenType.IDENTIFIER, "a", 2), |
908 new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 1), | 880 new StringToken(TokenType.STRING, "", 3), |
909 new StringToken(TokenType.IDENTIFIER, "a", 2), | 881 new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 3), |
910 new StringToken(TokenType.STRING, "", 3), | 882 new StringToken(TokenType.IDENTIFIER, "b", 4), |
911 new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 3), | 883 new StringToken(TokenType.STRING, "'", 5) |
912 new StringToken(TokenType.IDENTIFIER, "b", 4), | 884 ]); |
913 new StringToken(TokenType.STRING, "'", 5)]); | |
914 } | 885 } |
915 | 886 |
916 void test_string_simple_interpolation_block() { | 887 void test_string_simple_interpolation_block() { |
917 _assertTokens( | 888 _assertTokens("'Hello \${name}!'", [ |
918 "'Hello \${name}!'", | 889 new StringToken(TokenType.STRING, "'Hello ", 0), |
919 [ | 890 new StringToken(TokenType.STRING_INTERPOLATION_EXPRESSION, "\${", 7), |
920 new StringToken(TokenType.STRING, "'Hello ", 0), | 891 new StringToken(TokenType.IDENTIFIER, "name", 9), |
921 new StringToken(TokenType.STRING_INTERPOLATION_EXPRESSION, "\${", 7)
, | 892 new Token(TokenType.CLOSE_CURLY_BRACKET, 13), |
922 new StringToken(TokenType.IDENTIFIER, "name", 9), | 893 new StringToken(TokenType.STRING, "!'", 14) |
923 new Token(TokenType.CLOSE_CURLY_BRACKET, 13), | 894 ]); |
924 new StringToken(TokenType.STRING, "!'", 14)]); | |
925 } | 895 } |
926 | 896 |
927 void test_string_simple_interpolation_blockWithNestedMap() { | 897 void test_string_simple_interpolation_blockWithNestedMap() { |
928 _assertTokens( | 898 _assertTokens("'a \${f({'b' : 'c'})} d'", [ |
929 "'a \${f({'b' : 'c'})} d'", | 899 new StringToken(TokenType.STRING, "'a ", 0), |
930 [ | 900 new StringToken(TokenType.STRING_INTERPOLATION_EXPRESSION, "\${", 3), |
931 new StringToken(TokenType.STRING, "'a ", 0), | 901 new StringToken(TokenType.IDENTIFIER, "f", 5), |
932 new StringToken(TokenType.STRING_INTERPOLATION_EXPRESSION, "\${", 3)
, | 902 new Token(TokenType.OPEN_PAREN, 6), |
933 new StringToken(TokenType.IDENTIFIER, "f", 5), | 903 new Token(TokenType.OPEN_CURLY_BRACKET, 7), |
934 new Token(TokenType.OPEN_PAREN, 6), | 904 new StringToken(TokenType.STRING, "'b'", 8), |
935 new Token(TokenType.OPEN_CURLY_BRACKET, 7), | 905 new Token(TokenType.COLON, 12), |
936 new StringToken(TokenType.STRING, "'b'", 8), | 906 new StringToken(TokenType.STRING, "'c'", 14), |
937 new Token(TokenType.COLON, 12), | 907 new Token(TokenType.CLOSE_CURLY_BRACKET, 17), |
938 new StringToken(TokenType.STRING, "'c'", 14), | 908 new Token(TokenType.CLOSE_PAREN, 18), |
939 new Token(TokenType.CLOSE_CURLY_BRACKET, 17), | 909 new Token(TokenType.CLOSE_CURLY_BRACKET, 19), |
940 new Token(TokenType.CLOSE_PAREN, 18), | 910 new StringToken(TokenType.STRING, " d'", 20) |
941 new Token(TokenType.CLOSE_CURLY_BRACKET, 19), | 911 ]); |
942 new StringToken(TokenType.STRING, " d'", 20)]); | |
943 } | 912 } |
944 | 913 |
945 void test_string_simple_interpolation_firstAndLast() { | 914 void test_string_simple_interpolation_firstAndLast() { |
946 _assertTokens( | 915 _assertTokens("'\$greeting \$name'", [ |
947 "'\$greeting \$name'", | 916 new StringToken(TokenType.STRING, "'", 0), |
948 [ | 917 new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 1), |
949 new StringToken(TokenType.STRING, "'", 0), | 918 new StringToken(TokenType.IDENTIFIER, "greeting", 2), |
950 new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 1), | 919 new StringToken(TokenType.STRING, " ", 10), |
951 new StringToken(TokenType.IDENTIFIER, "greeting", 2), | 920 new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 11), |
952 new StringToken(TokenType.STRING, " ", 10), | 921 new StringToken(TokenType.IDENTIFIER, "name", 12), |
953 new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 11)
, | 922 new StringToken(TokenType.STRING, "'", 16) |
954 new StringToken(TokenType.IDENTIFIER, "name", 12), | 923 ]); |
955 new StringToken(TokenType.STRING, "'", 16)]); | |
956 } | 924 } |
957 | 925 |
958 void test_string_simple_interpolation_identifier() { | 926 void test_string_simple_interpolation_identifier() { |
959 _assertTokens( | 927 _assertTokens("'Hello \$name!'", [ |
960 "'Hello \$name!'", | 928 new StringToken(TokenType.STRING, "'Hello ", 0), |
961 [ | 929 new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 7), |
962 new StringToken(TokenType.STRING, "'Hello ", 0), | 930 new StringToken(TokenType.IDENTIFIER, "name", 8), |
963 new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 7), | 931 new StringToken(TokenType.STRING, "!'", 12) |
964 new StringToken(TokenType.IDENTIFIER, "name", 8), | 932 ]); |
965 new StringToken(TokenType.STRING, "!'", 12)]); | |
966 } | 933 } |
967 | 934 |
968 void test_string_simple_interpolation_missingIdentifier() { | 935 void test_string_simple_interpolation_missingIdentifier() { |
969 _assertTokens( | 936 _assertTokens("'\$x\$'", [ |
970 "'\$x\$'", | 937 new StringToken(TokenType.STRING, "'", 0), |
971 [ | 938 new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 1), |
972 new StringToken(TokenType.STRING, "'", 0), | 939 new StringToken(TokenType.IDENTIFIER, "x", 2), |
973 new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 1), | 940 new StringToken(TokenType.STRING, "", 3), |
974 new StringToken(TokenType.IDENTIFIER, "x", 2), | 941 new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 3), |
975 new StringToken(TokenType.STRING, "", 3), | 942 new StringToken(TokenType.STRING, "'", 4) |
976 new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 3), | 943 ]); |
977 new StringToken(TokenType.STRING, "'", 4)]); | |
978 } | 944 } |
979 | 945 |
980 void test_string_simple_interpolation_nonIdentifier() { | 946 void test_string_simple_interpolation_nonIdentifier() { |
981 _assertTokens( | 947 _assertTokens("'\$1'", [ |
982 "'\$1'", | 948 new StringToken(TokenType.STRING, "'", 0), |
983 [ | 949 new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 1), |
984 new StringToken(TokenType.STRING, "'", 0), | 950 new StringToken(TokenType.STRING, "1'", 2) |
985 new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 1), | 951 ]); |
986 new StringToken(TokenType.STRING, "1'", 2)]); | |
987 } | 952 } |
988 | 953 |
989 void test_string_simple_single() { | 954 void test_string_simple_single() { |
990 _assertToken(TokenType.STRING, "'string'"); | 955 _assertToken(TokenType.STRING, "'string'"); |
991 } | 956 } |
992 | 957 |
993 void test_string_simple_unterminated_eof() { | 958 void test_string_simple_unterminated_eof() { |
994 String source = "'string"; | 959 String source = "'string"; |
995 _assertErrorAndTokens( | 960 _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, |
996 ScannerErrorCode.UNTERMINATED_STRING_LITERAL, | 961 6, source, [new StringToken(TokenType.STRING, source, 0)]); |
997 6, | |
998 source, | |
999 [new StringToken(TokenType.STRING, source, 0)]); | |
1000 } | 962 } |
1001 | 963 |
1002 void test_string_simple_unterminated_eol() { | 964 void test_string_simple_unterminated_eol() { |
1003 String source = "'string"; | 965 String source = "'string"; |
1004 _assertErrorAndTokens( | 966 _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, |
1005 ScannerErrorCode.UNTERMINATED_STRING_LITERAL, | 967 7, "$source\r", [new StringToken(TokenType.STRING, source, 0)]); |
1006 7, | |
1007 "$source\r", | |
1008 [new StringToken(TokenType.STRING, source, 0)]); | |
1009 } | 968 } |
1010 | 969 |
1011 void test_string_simple_unterminated_interpolation_block() { | 970 void test_string_simple_unterminated_interpolation_block() { |
1012 _assertErrorAndTokens( | 971 _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, 6, |
1013 ScannerErrorCode.UNTERMINATED_STRING_LITERAL, | 972 "'\${name", [ |
1014 6, | 973 new StringToken(TokenType.STRING, "'", 0), |
1015 "'\${name", | 974 new StringToken(TokenType.STRING_INTERPOLATION_EXPRESSION, "\${", 1), |
1016 [ | 975 new StringToken(TokenType.IDENTIFIER, "name", 3), |
1017 new StringToken(TokenType.STRING, "'", 0), | 976 new StringToken(TokenType.STRING, "", 7) |
1018 new StringToken(TokenType.STRING_INTERPOLATION_EXPRESSION, "\${", 1)
, | 977 ]); |
1019 new StringToken(TokenType.IDENTIFIER, "name", 3), | |
1020 new StringToken(TokenType.STRING, "", 7)]); | |
1021 } | 978 } |
1022 | 979 |
1023 void test_string_simple_unterminated_interpolation_identifier() { | 980 void test_string_simple_unterminated_interpolation_identifier() { |
1024 _assertErrorAndTokens( | 981 _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, 5, |
1025 ScannerErrorCode.UNTERMINATED_STRING_LITERAL, | 982 "'\$name", [ |
1026 5, | 983 new StringToken(TokenType.STRING, "'", 0), |
1027 "'\$name", | 984 new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 1), |
1028 [ | 985 new StringToken(TokenType.IDENTIFIER, "name", 2), |
1029 new StringToken(TokenType.STRING, "'", 0), | 986 new StringToken(TokenType.STRING, "", 6) |
1030 new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 1), | 987 ]); |
1031 new StringToken(TokenType.IDENTIFIER, "name", 2), | |
1032 new StringToken(TokenType.STRING, "", 6)]); | |
1033 } | 988 } |
1034 | 989 |
1035 void test_tilde() { | 990 void test_tilde() { |
1036 _assertToken(TokenType.TILDE, "~"); | 991 _assertToken(TokenType.TILDE, "~"); |
1037 } | 992 } |
1038 | 993 |
1039 void test_tilde_slash() { | 994 void test_tilde_slash() { |
1040 _assertToken(TokenType.TILDE_SLASH, "~/"); | 995 _assertToken(TokenType.TILDE_SLASH, "~/"); |
1041 } | 996 } |
1042 | 997 |
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1077 } | 1032 } |
1078 | 1033 |
1079 /** | 1034 /** |
1080 * Assert that scanning the given [source] produces an error with the given | 1035 * Assert that scanning the given [source] produces an error with the given |
1081 * code. | 1036 * code. |
1082 * | 1037 * |
1083 * [expectedError] the error that should be produced | 1038 * [expectedError] the error that should be produced |
1084 * [expectedOffset] the string offset that should be associated with the error | 1039 * [expectedOffset] the string offset that should be associated with the error |
1085 * [source] the source to be scanned to produce the error | 1040 * [source] the source to be scanned to produce the error |
1086 */ | 1041 */ |
1087 void _assertError(ScannerErrorCode expectedError, int expectedOffset, | 1042 void _assertError( |
1088 String source) { | 1043 ScannerErrorCode expectedError, int expectedOffset, String source) { |
1089 GatheringErrorListener listener = new GatheringErrorListener(); | 1044 GatheringErrorListener listener = new GatheringErrorListener(); |
1090 _scanWithListener(source, listener); | 1045 _scanWithListener(source, listener); |
1091 listener.assertErrors( | 1046 listener.assertErrors([ |
1092 [ | 1047 new AnalysisError.con2(null, expectedOffset, 1, expectedError, [ |
1093 new AnalysisError.con2( | 1048 source.codeUnitAt(expectedOffset) |
1094 null, | 1049 ]) |
1095 expectedOffset, | 1050 ]); |
1096 1, | |
1097 expectedError, | |
1098 [source.codeUnitAt(expectedOffset)])]); | |
1099 } | 1051 } |
1100 | 1052 |
1101 /** | 1053 /** |
1102 * Assert that scanning the given [source] produces an error with the given | 1054 * Assert that scanning the given [source] produces an error with the given |
1103 * code, and also produces the given tokens. | 1055 * code, and also produces the given tokens. |
1104 * | 1056 * |
1105 * [expectedError] the error that should be produced | 1057 * [expectedError] the error that should be produced |
1106 * [expectedOffset] the string offset that should be associated with the error | 1058 * [expectedOffset] the string offset that should be associated with the error |
1107 * [source] the source to be scanned to produce the error | 1059 * [source] the source to be scanned to produce the error |
1108 * [expectedTokens] the tokens that are expected to be in the source | 1060 * [expectedTokens] the tokens that are expected to be in the source |
1109 */ | 1061 */ |
1110 void _assertErrorAndTokens(ScannerErrorCode expectedError, int expectedOffset, | 1062 void _assertErrorAndTokens(ScannerErrorCode expectedError, int expectedOffset, |
1111 String source, List<Token> expectedTokens) { | 1063 String source, List<Token> expectedTokens) { |
1112 GatheringErrorListener listener = new GatheringErrorListener(); | 1064 GatheringErrorListener listener = new GatheringErrorListener(); |
1113 Token token = _scanWithListener(source, listener); | 1065 Token token = _scanWithListener(source, listener); |
1114 listener.assertErrors( | 1066 listener.assertErrors([ |
1115 [ | 1067 new AnalysisError.con2(null, expectedOffset, 1, expectedError, [ |
1116 new AnalysisError.con2( | 1068 source.codeUnitAt(expectedOffset) |
1117 null, | 1069 ]) |
1118 expectedOffset, | 1070 ]); |
1119 1, | |
1120 expectedError, | |
1121 [source.codeUnitAt(expectedOffset)])]); | |
1122 _checkTokens(token, expectedTokens); | 1071 _checkTokens(token, expectedTokens); |
1123 } | 1072 } |
1124 | 1073 |
1125 /** | 1074 /** |
1126 * Assert that when scanned the given [source] contains a single keyword token | 1075 * Assert that when scanned the given [source] contains a single keyword token |
1127 * with the same lexeme as the original source. | 1076 * with the same lexeme as the original source. |
1128 */ | 1077 */ |
1129 void _assertKeywordToken(String source) { | 1078 void _assertKeywordToken(String source) { |
1130 Token token = _scan(source); | 1079 Token token = _scan(source); |
1131 expect(token, isNotNull); | 1080 expect(token, isNotNull); |
1132 expect(token.type, TokenType.KEYWORD); | 1081 expect(token.type, TokenType.KEYWORD); |
1133 expect(token.offset, 0); | 1082 expect(token.offset, 0); |
1134 expect(token.length, source.length); | 1083 expect(token.length, source.length); |
1135 expect(token.lexeme, source); | 1084 expect(token.lexeme, source); |
1136 Object value = token.value(); | 1085 Object value = token.value(); |
1137 expect(value is Keyword, isTrue); | 1086 expect(value is Keyword, isTrue); |
1138 expect((value as Keyword).syntax, source); | 1087 expect((value as Keyword).syntax, source); |
1139 token = _scan(" $source "); | 1088 token = _scan(" $source "); |
1140 expect(token, isNotNull); | 1089 expect(token, isNotNull); |
1141 expect(token.type, TokenType.KEYWORD); | 1090 expect(token.type, TokenType.KEYWORD); |
1142 expect(token.offset, 1); | 1091 expect(token.offset, 1); |
1143 expect(token.length, source.length); | 1092 expect(token.length, source.length); |
1144 expect(token.lexeme, source); | 1093 expect(token.lexeme, source); |
1145 value = token.value(); | 1094 value = token.value(); |
1146 expect(value is Keyword, isTrue); | 1095 expect(value is Keyword, isTrue); |
1147 expect((value as Keyword).syntax, source); | 1096 expect((value as Keyword).syntax, source); |
1148 expect(token.next.type, TokenType.EOF); | 1097 expect(token.next.type, TokenType.EOF); |
1149 } | 1098 } |
1150 | 1099 |
1151 void _assertLineInfo(String source, | 1100 void _assertLineInfo( |
1152 List<ScannerTest_ExpectedLocation> expectedLocations) { | 1101 String source, List<ScannerTest_ExpectedLocation> expectedLocations) { |
1153 GatheringErrorListener listener = new GatheringErrorListener(); | 1102 GatheringErrorListener listener = new GatheringErrorListener(); |
1154 _scanWithListener(source, listener); | 1103 _scanWithListener(source, listener); |
1155 listener.assertNoErrors(); | 1104 listener.assertNoErrors(); |
1156 LineInfo info = listener.getLineInfo(new TestSource()); | 1105 LineInfo info = listener.getLineInfo(new TestSource()); |
1157 expect(info, isNotNull); | 1106 expect(info, isNotNull); |
1158 for (ScannerTest_ExpectedLocation expectedLocation in expectedLocations) { | 1107 for (ScannerTest_ExpectedLocation expectedLocation in expectedLocations) { |
1159 LineInfo_Location location = info.getLocation(expectedLocation._offset); | 1108 LineInfo_Location location = info.getLocation(expectedLocation._offset); |
1160 expect(location.lineNumber, expectedLocation._lineNumber); | 1109 expect(location.lineNumber, expectedLocation._lineNumber); |
1161 expect(location.columnNumber, expectedLocation._columnNumber); | 1110 expect(location.columnNumber, expectedLocation._columnNumber); |
1162 } | 1111 } |
(...skipping 56 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1219 Token token = _scan(source); | 1168 Token token = _scan(source); |
1220 _checkTokens(token, expectedTokens); | 1169 _checkTokens(token, expectedTokens); |
1221 } | 1170 } |
1222 | 1171 |
1223 void _checkTokens(Token firstToken, List<Token> expectedTokens) { | 1172 void _checkTokens(Token firstToken, List<Token> expectedTokens) { |
1224 expect(firstToken, isNotNull); | 1173 expect(firstToken, isNotNull); |
1225 Token token = firstToken; | 1174 Token token = firstToken; |
1226 for (int i = 0; i < expectedTokens.length; i++) { | 1175 for (int i = 0; i < expectedTokens.length; i++) { |
1227 Token expectedToken = expectedTokens[i]; | 1176 Token expectedToken = expectedTokens[i]; |
1228 expect(token.type, expectedToken.type, reason: "Wrong type for token $i"); | 1177 expect(token.type, expectedToken.type, reason: "Wrong type for token $i"); |
1229 expect( | 1178 expect(token.offset, expectedToken.offset, |
1230 token.offset, | |
1231 expectedToken.offset, | |
1232 reason: "Wrong offset for token $i"); | 1179 reason: "Wrong offset for token $i"); |
1233 expect( | 1180 expect(token.length, expectedToken.length, |
1234 token.length, | |
1235 expectedToken.length, | |
1236 reason: "Wrong length for token $i"); | 1181 reason: "Wrong length for token $i"); |
1237 expect( | 1182 expect(token.lexeme, expectedToken.lexeme, |
1238 token.lexeme, | |
1239 expectedToken.lexeme, | |
1240 reason: "Wrong lexeme for token $i"); | 1183 reason: "Wrong lexeme for token $i"); |
1241 token = token.next; | 1184 token = token.next; |
1242 expect(token, isNotNull); | 1185 expect(token, isNotNull); |
1243 } | 1186 } |
1244 expect(token.type, TokenType.EOF); | 1187 expect(token.type, TokenType.EOF); |
1245 } | 1188 } |
1246 | 1189 |
1247 Token _scan(String source) { | 1190 Token _scan(String source) { |
1248 GatheringErrorListener listener = new GatheringErrorListener(); | 1191 GatheringErrorListener listener = new GatheringErrorListener(); |
1249 Token token = _scanWithListener(source, listener); | 1192 Token token = _scanWithListener(source, listener); |
(...skipping 14 matching lines...) Expand all Loading... |
1264 * An `ExpectedLocation` encodes information about the expected location of a | 1207 * An `ExpectedLocation` encodes information about the expected location of a |
1265 * given offset in source code. | 1208 * given offset in source code. |
1266 */ | 1209 */ |
1267 class ScannerTest_ExpectedLocation { | 1210 class ScannerTest_ExpectedLocation { |
1268 final int _offset; | 1211 final int _offset; |
1269 | 1212 |
1270 final int _lineNumber; | 1213 final int _lineNumber; |
1271 | 1214 |
1272 final int _columnNumber; | 1215 final int _columnNumber; |
1273 | 1216 |
1274 ScannerTest_ExpectedLocation(this._offset, this._lineNumber, | 1217 ScannerTest_ExpectedLocation( |
1275 this._columnNumber); | 1218 this._offset, this._lineNumber, this._columnNumber); |
1276 } | 1219 } |
1277 | 1220 |
1278 /** | 1221 /** |
1279 * A `TokenStreamValidator` is used to validate the correct construction of a | 1222 * A `TokenStreamValidator` is used to validate the correct construction of a |
1280 * stream of tokens. | 1223 * stream of tokens. |
1281 */ | 1224 */ |
1282 class TokenStreamValidator { | 1225 class TokenStreamValidator { |
1283 /** | 1226 /** |
1284 * Validate that the stream of tokens that starts with the given [token] is | 1227 * Validate that the stream of tokens that starts with the given [token] is |
1285 * correct. | 1228 * correct. |
(...skipping 115 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1401 expect(TokenType.LT_LT.isUserDefinableOperator, isTrue); | 1344 expect(TokenType.LT_LT.isUserDefinableOperator, isTrue); |
1402 expect(TokenType.MINUS.isUserDefinableOperator, isTrue); | 1345 expect(TokenType.MINUS.isUserDefinableOperator, isTrue); |
1403 expect(TokenType.PERCENT.isUserDefinableOperator, isTrue); | 1346 expect(TokenType.PERCENT.isUserDefinableOperator, isTrue); |
1404 expect(TokenType.PLUS.isUserDefinableOperator, isTrue); | 1347 expect(TokenType.PLUS.isUserDefinableOperator, isTrue); |
1405 expect(TokenType.SLASH.isUserDefinableOperator, isTrue); | 1348 expect(TokenType.SLASH.isUserDefinableOperator, isTrue); |
1406 expect(TokenType.STAR.isUserDefinableOperator, isTrue); | 1349 expect(TokenType.STAR.isUserDefinableOperator, isTrue); |
1407 expect(TokenType.TILDE.isUserDefinableOperator, isTrue); | 1350 expect(TokenType.TILDE.isUserDefinableOperator, isTrue); |
1408 expect(TokenType.TILDE_SLASH.isUserDefinableOperator, isTrue); | 1351 expect(TokenType.TILDE_SLASH.isUserDefinableOperator, isTrue); |
1409 } | 1352 } |
1410 } | 1353 } |
OLD | NEW |