OLD | NEW |
1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 library fasta.analyzer.token_utils; | 5 library fasta.analyzer.token_utils; |
6 | 6 |
7 import 'package:front_end/src/fasta/parser/error_kind.dart' show | 7 import 'package:front_end/src/fasta/parser/error_kind.dart' show ErrorKind; |
8 ErrorKind; | |
9 | 8 |
10 import 'package:front_end/src/fasta/scanner/error_token.dart' show | 9 import 'package:front_end/src/fasta/scanner/error_token.dart' show ErrorToken; |
11 ErrorToken; | |
12 | 10 |
13 import 'package:front_end/src/fasta/scanner/keyword.dart' show | 11 import 'package:front_end/src/fasta/scanner/keyword.dart' show Keyword; |
14 Keyword; | |
15 | 12 |
16 import 'package:front_end/src/fasta/scanner/precedence.dart'; | 13 import 'package:front_end/src/fasta/scanner/precedence.dart'; |
17 | 14 |
18 import 'package:front_end/src/fasta/scanner/token.dart' show | 15 import 'package:front_end/src/fasta/scanner/token.dart' |
19 BeginGroupToken, | 16 show BeginGroupToken, KeywordToken, StringToken, SymbolToken, Token; |
20 KeywordToken, | |
21 StringToken, | |
22 SymbolToken, | |
23 Token; | |
24 | 17 |
25 import 'package:front_end/src/fasta/scanner/token_constants.dart'; | 18 import 'package:front_end/src/fasta/scanner/token_constants.dart'; |
26 | 19 |
27 import 'package:front_end/src/scanner/token.dart' as analyzer show | 20 import 'package:front_end/src/scanner/token.dart' as analyzer |
28 BeginToken, | 21 show |
29 BeginTokenWithComment, | 22 BeginToken, |
30 CommentToken, | 23 BeginTokenWithComment, |
31 Keyword, | 24 CommentToken, |
32 KeywordToken, | 25 Keyword, |
33 KeywordTokenWithComment, | 26 KeywordToken, |
34 StringToken, | 27 KeywordTokenWithComment, |
35 StringTokenWithComment, | 28 StringToken, |
36 Token, | 29 StringTokenWithComment, |
37 TokenWithComment; | 30 Token, |
| 31 TokenWithComment; |
38 | 32 |
39 import 'package:front_end/src/scanner/errors.dart' as analyzer show | 33 import 'package:front_end/src/scanner/errors.dart' as analyzer |
40 ScannerErrorCode; | 34 show ScannerErrorCode; |
41 | 35 |
42 import 'package:analyzer/dart/ast/token.dart' show | 36 import 'package:analyzer/dart/ast/token.dart' show TokenType; |
43 TokenType; | |
44 | 37 |
45 import '../errors.dart' show | 38 import '../errors.dart' show internalError; |
46 internalError; | |
47 | 39 |
48 /// Class capable of converting a stream of Fasta tokens to a stream of analyzer | 40 /// Class capable of converting a stream of Fasta tokens to a stream of analyzer |
49 /// tokens. | 41 /// tokens. |
50 /// | 42 /// |
51 /// This is a class rather than an ordinary method so that it can be subclassed | 43 /// This is a class rather than an ordinary method so that it can be subclassed |
52 /// in tests. | 44 /// in tests. |
53 /// | 45 /// |
54 /// TODO(paulberry,ahe): Fasta includes comments directly in the token | 46 /// TODO(paulberry,ahe): Fasta includes comments directly in the token |
55 /// stream, rather than pointing to them via a "precedingComment" pointer, as | 47 /// stream, rather than pointing to them via a "precedingComment" pointer, as |
56 /// analyzer does. This seems like it will complicate parsing and other | 48 /// analyzer does. This seems like it will complicate parsing and other |
(...skipping 567 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
624 "operator": analyzer.Keyword.OPERATOR, | 616 "operator": analyzer.Keyword.OPERATOR, |
625 "part": analyzer.Keyword.PART, | 617 "part": analyzer.Keyword.PART, |
626 "set": analyzer.Keyword.SET, | 618 "set": analyzer.Keyword.SET, |
627 "static": analyzer.Keyword.STATIC, | 619 "static": analyzer.Keyword.STATIC, |
628 "typedef": analyzer.Keyword.TYPEDEF, | 620 "typedef": analyzer.Keyword.TYPEDEF, |
629 "deferred": analyzer.Keyword.DEFERRED, | 621 "deferred": analyzer.Keyword.DEFERRED, |
630 }; | 622 }; |
631 | 623 |
632 TokenType getTokenType(Token token) { | 624 TokenType getTokenType(Token token) { |
633 switch (token.kind) { | 625 switch (token.kind) { |
634 case EOF_TOKEN: return TokenType.EOF; | 626 case EOF_TOKEN: |
635 case DOUBLE_TOKEN: return TokenType.DOUBLE; | 627 return TokenType.EOF; |
636 case HEXADECIMAL_TOKEN: return TokenType.HEXADECIMAL; | 628 case DOUBLE_TOKEN: |
637 case IDENTIFIER_TOKEN: return TokenType.IDENTIFIER; | 629 return TokenType.DOUBLE; |
638 case INT_TOKEN: return TokenType.INT; | 630 case HEXADECIMAL_TOKEN: |
639 case KEYWORD_TOKEN: return TokenType.KEYWORD; | 631 return TokenType.HEXADECIMAL; |
| 632 case IDENTIFIER_TOKEN: |
| 633 return TokenType.IDENTIFIER; |
| 634 case INT_TOKEN: |
| 635 return TokenType.INT; |
| 636 case KEYWORD_TOKEN: |
| 637 return TokenType.KEYWORD; |
640 // case MULTI_LINE_COMMENT_TOKEN: return TokenType.MULTI_LINE_COMMENT; | 638 // case MULTI_LINE_COMMENT_TOKEN: return TokenType.MULTI_LINE_COMMENT; |
641 // case SCRIPT_TAG_TOKEN: return TokenType.SCRIPT_TAG; | 639 // case SCRIPT_TAG_TOKEN: return TokenType.SCRIPT_TAG; |
642 // case SINGLE_LINE_COMMENT_TOKEN: return TokenType.SINGLE_LINE_COMMENT; | 640 // case SINGLE_LINE_COMMENT_TOKEN: return TokenType.SINGLE_LINE_COMMENT; |
643 case STRING_TOKEN: return TokenType.STRING; | 641 case STRING_TOKEN: |
644 case AMPERSAND_TOKEN: return TokenType.AMPERSAND; | 642 return TokenType.STRING; |
645 case AMPERSAND_AMPERSAND_TOKEN: return TokenType.AMPERSAND_AMPERSAND; | 643 case AMPERSAND_TOKEN: |
| 644 return TokenType.AMPERSAND; |
| 645 case AMPERSAND_AMPERSAND_TOKEN: |
| 646 return TokenType.AMPERSAND_AMPERSAND; |
646 // case AMPERSAND_AMPERSAND_EQ_TOKEN: | 647 // case AMPERSAND_AMPERSAND_EQ_TOKEN: |
647 // return TokenType.AMPERSAND_AMPERSAND_EQ; | 648 // return TokenType.AMPERSAND_AMPERSAND_EQ; |
648 case AMPERSAND_EQ_TOKEN: return TokenType.AMPERSAND_EQ; | 649 case AMPERSAND_EQ_TOKEN: |
649 case AT_TOKEN: return TokenType.AT; | 650 return TokenType.AMPERSAND_EQ; |
650 case BANG_TOKEN: return TokenType.BANG; | 651 case AT_TOKEN: |
651 case BANG_EQ_TOKEN: return TokenType.BANG_EQ; | 652 return TokenType.AT; |
652 case BAR_TOKEN: return TokenType.BAR; | 653 case BANG_TOKEN: |
653 case BAR_BAR_TOKEN: return TokenType.BAR_BAR; | 654 return TokenType.BANG; |
| 655 case BANG_EQ_TOKEN: |
| 656 return TokenType.BANG_EQ; |
| 657 case BAR_TOKEN: |
| 658 return TokenType.BAR; |
| 659 case BAR_BAR_TOKEN: |
| 660 return TokenType.BAR_BAR; |
654 // case BAR_BAR_EQ_TOKEN: return TokenType.BAR_BAR_EQ; | 661 // case BAR_BAR_EQ_TOKEN: return TokenType.BAR_BAR_EQ; |
655 case BAR_EQ_TOKEN: return TokenType.BAR_EQ; | 662 case BAR_EQ_TOKEN: |
656 case COLON_TOKEN: return TokenType.COLON; | 663 return TokenType.BAR_EQ; |
657 case COMMA_TOKEN: return TokenType.COMMA; | 664 case COLON_TOKEN: |
658 case CARET_TOKEN: return TokenType.CARET; | 665 return TokenType.COLON; |
659 case CARET_EQ_TOKEN: return TokenType.CARET_EQ; | 666 case COMMA_TOKEN: |
660 case CLOSE_CURLY_BRACKET_TOKEN: return TokenType.CLOSE_CURLY_BRACKET; | 667 return TokenType.COMMA; |
661 case CLOSE_PAREN_TOKEN: return TokenType.CLOSE_PAREN; | 668 case CARET_TOKEN: |
662 case CLOSE_SQUARE_BRACKET_TOKEN: return TokenType.CLOSE_SQUARE_BRACKET; | 669 return TokenType.CARET; |
663 case EQ_TOKEN: return TokenType.EQ; | 670 case CARET_EQ_TOKEN: |
664 case EQ_EQ_TOKEN: return TokenType.EQ_EQ; | 671 return TokenType.CARET_EQ; |
665 case FUNCTION_TOKEN: return TokenType.FUNCTION; | 672 case CLOSE_CURLY_BRACKET_TOKEN: |
666 case GT_TOKEN: return TokenType.GT; | 673 return TokenType.CLOSE_CURLY_BRACKET; |
667 case GT_EQ_TOKEN: return TokenType.GT_EQ; | 674 case CLOSE_PAREN_TOKEN: |
668 case GT_GT_TOKEN: return TokenType.GT_GT; | 675 return TokenType.CLOSE_PAREN; |
669 case GT_GT_EQ_TOKEN: return TokenType.GT_GT_EQ; | 676 case CLOSE_SQUARE_BRACKET_TOKEN: |
670 case HASH_TOKEN: return TokenType.HASH; | 677 return TokenType.CLOSE_SQUARE_BRACKET; |
671 case INDEX_TOKEN: return TokenType.INDEX; | 678 case EQ_TOKEN: |
672 case INDEX_EQ_TOKEN: return TokenType.INDEX_EQ; | 679 return TokenType.EQ; |
| 680 case EQ_EQ_TOKEN: |
| 681 return TokenType.EQ_EQ; |
| 682 case FUNCTION_TOKEN: |
| 683 return TokenType.FUNCTION; |
| 684 case GT_TOKEN: |
| 685 return TokenType.GT; |
| 686 case GT_EQ_TOKEN: |
| 687 return TokenType.GT_EQ; |
| 688 case GT_GT_TOKEN: |
| 689 return TokenType.GT_GT; |
| 690 case GT_GT_EQ_TOKEN: |
| 691 return TokenType.GT_GT_EQ; |
| 692 case HASH_TOKEN: |
| 693 return TokenType.HASH; |
| 694 case INDEX_TOKEN: |
| 695 return TokenType.INDEX; |
| 696 case INDEX_EQ_TOKEN: |
| 697 return TokenType.INDEX_EQ; |
673 // case IS_TOKEN: return TokenType.IS; | 698 // case IS_TOKEN: return TokenType.IS; |
674 case LT_TOKEN: return TokenType.LT; | 699 case LT_TOKEN: |
675 case LT_EQ_TOKEN: return TokenType.LT_EQ; | 700 return TokenType.LT; |
676 case LT_LT_TOKEN: return TokenType.LT_LT; | 701 case LT_EQ_TOKEN: |
677 case LT_LT_EQ_TOKEN: return TokenType.LT_LT_EQ; | 702 return TokenType.LT_EQ; |
678 case MINUS_TOKEN: return TokenType.MINUS; | 703 case LT_LT_TOKEN: |
679 case MINUS_EQ_TOKEN: return TokenType.MINUS_EQ; | 704 return TokenType.LT_LT; |
680 case MINUS_MINUS_TOKEN: return TokenType.MINUS_MINUS; | 705 case LT_LT_EQ_TOKEN: |
681 case OPEN_CURLY_BRACKET_TOKEN: return TokenType.OPEN_CURLY_BRACKET; | 706 return TokenType.LT_LT_EQ; |
682 case OPEN_PAREN_TOKEN: return TokenType.OPEN_PAREN; | 707 case MINUS_TOKEN: |
683 case OPEN_SQUARE_BRACKET_TOKEN: return TokenType.OPEN_SQUARE_BRACKET; | 708 return TokenType.MINUS; |
684 case PERCENT_TOKEN: return TokenType.PERCENT; | 709 case MINUS_EQ_TOKEN: |
685 case PERCENT_EQ_TOKEN: return TokenType.PERCENT_EQ; | 710 return TokenType.MINUS_EQ; |
686 case PERIOD_TOKEN: return TokenType.PERIOD; | 711 case MINUS_MINUS_TOKEN: |
687 case PERIOD_PERIOD_TOKEN: return TokenType.PERIOD_PERIOD; | 712 return TokenType.MINUS_MINUS; |
688 case PLUS_TOKEN: return TokenType.PLUS; | 713 case OPEN_CURLY_BRACKET_TOKEN: |
689 case PLUS_EQ_TOKEN: return TokenType.PLUS_EQ; | 714 return TokenType.OPEN_CURLY_BRACKET; |
690 case PLUS_PLUS_TOKEN: return TokenType.PLUS_PLUS; | 715 case OPEN_PAREN_TOKEN: |
691 case QUESTION_TOKEN: return TokenType.QUESTION; | 716 return TokenType.OPEN_PAREN; |
692 case QUESTION_PERIOD_TOKEN: return TokenType.QUESTION_PERIOD; | 717 case OPEN_SQUARE_BRACKET_TOKEN: |
693 case QUESTION_QUESTION_TOKEN: return TokenType.QUESTION_QUESTION; | 718 return TokenType.OPEN_SQUARE_BRACKET; |
694 case QUESTION_QUESTION_EQ_TOKEN: return TokenType.QUESTION_QUESTION_EQ; | 719 case PERCENT_TOKEN: |
695 case SEMICOLON_TOKEN: return TokenType.SEMICOLON; | 720 return TokenType.PERCENT; |
696 case SLASH_TOKEN: return TokenType.SLASH; | 721 case PERCENT_EQ_TOKEN: |
697 case SLASH_EQ_TOKEN: return TokenType.SLASH_EQ; | 722 return TokenType.PERCENT_EQ; |
698 case STAR_TOKEN: return TokenType.STAR; | 723 case PERIOD_TOKEN: |
699 case STAR_EQ_TOKEN: return TokenType.STAR_EQ; | 724 return TokenType.PERIOD; |
| 725 case PERIOD_PERIOD_TOKEN: |
| 726 return TokenType.PERIOD_PERIOD; |
| 727 case PLUS_TOKEN: |
| 728 return TokenType.PLUS; |
| 729 case PLUS_EQ_TOKEN: |
| 730 return TokenType.PLUS_EQ; |
| 731 case PLUS_PLUS_TOKEN: |
| 732 return TokenType.PLUS_PLUS; |
| 733 case QUESTION_TOKEN: |
| 734 return TokenType.QUESTION; |
| 735 case QUESTION_PERIOD_TOKEN: |
| 736 return TokenType.QUESTION_PERIOD; |
| 737 case QUESTION_QUESTION_TOKEN: |
| 738 return TokenType.QUESTION_QUESTION; |
| 739 case QUESTION_QUESTION_EQ_TOKEN: |
| 740 return TokenType.QUESTION_QUESTION_EQ; |
| 741 case SEMICOLON_TOKEN: |
| 742 return TokenType.SEMICOLON; |
| 743 case SLASH_TOKEN: |
| 744 return TokenType.SLASH; |
| 745 case SLASH_EQ_TOKEN: |
| 746 return TokenType.SLASH_EQ; |
| 747 case STAR_TOKEN: |
| 748 return TokenType.STAR; |
| 749 case STAR_EQ_TOKEN: |
| 750 return TokenType.STAR_EQ; |
700 case STRING_INTERPOLATION_TOKEN: | 751 case STRING_INTERPOLATION_TOKEN: |
701 return TokenType.STRING_INTERPOLATION_EXPRESSION; | 752 return TokenType.STRING_INTERPOLATION_EXPRESSION; |
702 case STRING_INTERPOLATION_IDENTIFIER_TOKEN: | 753 case STRING_INTERPOLATION_IDENTIFIER_TOKEN: |
703 return TokenType.STRING_INTERPOLATION_IDENTIFIER; | 754 return TokenType.STRING_INTERPOLATION_IDENTIFIER; |
704 case TILDE_TOKEN: return TokenType.TILDE; | 755 case TILDE_TOKEN: |
705 case TILDE_SLASH_TOKEN: return TokenType.TILDE_SLASH; | 756 return TokenType.TILDE; |
706 case TILDE_SLASH_EQ_TOKEN: return TokenType.TILDE_SLASH_EQ; | 757 case TILDE_SLASH_TOKEN: |
707 case BACKPING_TOKEN: return TokenType.BACKPING; | 758 return TokenType.TILDE_SLASH; |
708 case BACKSLASH_TOKEN: return TokenType.BACKSLASH; | 759 case TILDE_SLASH_EQ_TOKEN: |
709 case PERIOD_PERIOD_PERIOD_TOKEN: return TokenType.PERIOD_PERIOD_PERIOD; | 760 return TokenType.TILDE_SLASH_EQ; |
| 761 case BACKPING_TOKEN: |
| 762 return TokenType.BACKPING; |
| 763 case BACKSLASH_TOKEN: |
| 764 return TokenType.BACKSLASH; |
| 765 case PERIOD_PERIOD_PERIOD_TOKEN: |
| 766 return TokenType.PERIOD_PERIOD_PERIOD; |
710 // case GENERIC_METHOD_TYPE_LIST_TOKEN: | 767 // case GENERIC_METHOD_TYPE_LIST_TOKEN: |
711 // return TokenType.GENERIC_METHOD_TYPE_LIST; | 768 // return TokenType.GENERIC_METHOD_TYPE_LIST; |
712 // case GENERIC_METHOD_TYPE_ASSIGN_TOKEN: | 769 // case GENERIC_METHOD_TYPE_ASSIGN_TOKEN: |
713 // return TokenType.GENERIC_METHOD_TYPE_ASSIGN; | 770 // return TokenType.GENERIC_METHOD_TYPE_ASSIGN; |
714 default: | 771 default: |
715 return internalError("Unhandled token ${token.info}"); | 772 return internalError("Unhandled token ${token.info}"); |
716 } | 773 } |
717 } | 774 } |
OLD | NEW |