| OLD | NEW |
| 1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library fasta.analyzer.token_utils; | 5 library fasta.analyzer.token_utils; |
| 6 | 6 |
| 7 import 'package:front_end/src/fasta/parser/error_kind.dart' show | 7 import 'package:front_end/src/fasta/parser/error_kind.dart' show ErrorKind; |
| 8 ErrorKind; | |
| 9 | 8 |
| 10 import 'package:front_end/src/fasta/scanner/error_token.dart' show | 9 import 'package:front_end/src/fasta/scanner/error_token.dart' show ErrorToken; |
| 11 ErrorToken; | |
| 12 | 10 |
| 13 import 'package:front_end/src/fasta/scanner/keyword.dart' show | 11 import 'package:front_end/src/fasta/scanner/keyword.dart' show Keyword; |
| 14 Keyword; | |
| 15 | 12 |
| 16 import 'package:front_end/src/fasta/scanner/precedence.dart'; | 13 import 'package:front_end/src/fasta/scanner/precedence.dart'; |
| 17 | 14 |
| 18 import 'package:front_end/src/fasta/scanner/token.dart' show | 15 import 'package:front_end/src/fasta/scanner/token.dart' |
| 19 BeginGroupToken, | 16 show BeginGroupToken, KeywordToken, StringToken, SymbolToken, Token; |
| 20 KeywordToken, | |
| 21 StringToken, | |
| 22 SymbolToken, | |
| 23 Token; | |
| 24 | 17 |
| 25 import 'package:front_end/src/fasta/scanner/token_constants.dart'; | 18 import 'package:front_end/src/fasta/scanner/token_constants.dart'; |
| 26 | 19 |
| 27 import 'package:front_end/src/scanner/token.dart' as analyzer show | 20 import 'package:front_end/src/scanner/token.dart' as analyzer |
| 28 BeginToken, | 21 show |
| 29 BeginTokenWithComment, | 22 BeginToken, |
| 30 CommentToken, | 23 BeginTokenWithComment, |
| 31 Keyword, | 24 CommentToken, |
| 32 KeywordToken, | 25 Keyword, |
| 33 KeywordTokenWithComment, | 26 KeywordToken, |
| 34 StringToken, | 27 KeywordTokenWithComment, |
| 35 StringTokenWithComment, | 28 StringToken, |
| 36 Token, | 29 StringTokenWithComment, |
| 37 TokenWithComment; | 30 Token, |
| 31 TokenWithComment; |
| 38 | 32 |
| 39 import 'package:front_end/src/scanner/errors.dart' as analyzer show | 33 import 'package:front_end/src/scanner/errors.dart' as analyzer |
| 40 ScannerErrorCode; | 34 show ScannerErrorCode; |
| 41 | 35 |
| 42 import 'package:analyzer/dart/ast/token.dart' show | 36 import 'package:analyzer/dart/ast/token.dart' show TokenType; |
| 43 TokenType; | |
| 44 | 37 |
| 45 import '../errors.dart' show | 38 import '../errors.dart' show internalError; |
| 46 internalError; | |
| 47 | 39 |
| 48 /// Class capable of converting a stream of Fasta tokens to a stream of analyzer | 40 /// Class capable of converting a stream of Fasta tokens to a stream of analyzer |
| 49 /// tokens. | 41 /// tokens. |
| 50 /// | 42 /// |
| 51 /// This is a class rather than an ordinary method so that it can be subclassed | 43 /// This is a class rather than an ordinary method so that it can be subclassed |
| 52 /// in tests. | 44 /// in tests. |
| 53 /// | 45 /// |
| 54 /// TODO(paulberry,ahe): Fasta includes comments directly in the token | 46 /// TODO(paulberry,ahe): Fasta includes comments directly in the token |
| 55 /// stream, rather than pointing to them via a "precedingComment" pointer, as | 47 /// stream, rather than pointing to them via a "precedingComment" pointer, as |
| 56 /// analyzer does. This seems like it will complicate parsing and other | 48 /// analyzer does. This seems like it will complicate parsing and other |
| (...skipping 567 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 624 "operator": analyzer.Keyword.OPERATOR, | 616 "operator": analyzer.Keyword.OPERATOR, |
| 625 "part": analyzer.Keyword.PART, | 617 "part": analyzer.Keyword.PART, |
| 626 "set": analyzer.Keyword.SET, | 618 "set": analyzer.Keyword.SET, |
| 627 "static": analyzer.Keyword.STATIC, | 619 "static": analyzer.Keyword.STATIC, |
| 628 "typedef": analyzer.Keyword.TYPEDEF, | 620 "typedef": analyzer.Keyword.TYPEDEF, |
| 629 "deferred": analyzer.Keyword.DEFERRED, | 621 "deferred": analyzer.Keyword.DEFERRED, |
| 630 }; | 622 }; |
| 631 | 623 |
| 632 TokenType getTokenType(Token token) { | 624 TokenType getTokenType(Token token) { |
| 633 switch (token.kind) { | 625 switch (token.kind) { |
| 634 case EOF_TOKEN: return TokenType.EOF; | 626 case EOF_TOKEN: |
| 635 case DOUBLE_TOKEN: return TokenType.DOUBLE; | 627 return TokenType.EOF; |
| 636 case HEXADECIMAL_TOKEN: return TokenType.HEXADECIMAL; | 628 case DOUBLE_TOKEN: |
| 637 case IDENTIFIER_TOKEN: return TokenType.IDENTIFIER; | 629 return TokenType.DOUBLE; |
| 638 case INT_TOKEN: return TokenType.INT; | 630 case HEXADECIMAL_TOKEN: |
| 639 case KEYWORD_TOKEN: return TokenType.KEYWORD; | 631 return TokenType.HEXADECIMAL; |
| 632 case IDENTIFIER_TOKEN: |
| 633 return TokenType.IDENTIFIER; |
| 634 case INT_TOKEN: |
| 635 return TokenType.INT; |
| 636 case KEYWORD_TOKEN: |
| 637 return TokenType.KEYWORD; |
| 640 // case MULTI_LINE_COMMENT_TOKEN: return TokenType.MULTI_LINE_COMMENT; | 638 // case MULTI_LINE_COMMENT_TOKEN: return TokenType.MULTI_LINE_COMMENT; |
| 641 // case SCRIPT_TAG_TOKEN: return TokenType.SCRIPT_TAG; | 639 // case SCRIPT_TAG_TOKEN: return TokenType.SCRIPT_TAG; |
| 642 // case SINGLE_LINE_COMMENT_TOKEN: return TokenType.SINGLE_LINE_COMMENT; | 640 // case SINGLE_LINE_COMMENT_TOKEN: return TokenType.SINGLE_LINE_COMMENT; |
| 643 case STRING_TOKEN: return TokenType.STRING; | 641 case STRING_TOKEN: |
| 644 case AMPERSAND_TOKEN: return TokenType.AMPERSAND; | 642 return TokenType.STRING; |
| 645 case AMPERSAND_AMPERSAND_TOKEN: return TokenType.AMPERSAND_AMPERSAND; | 643 case AMPERSAND_TOKEN: |
| 644 return TokenType.AMPERSAND; |
| 645 case AMPERSAND_AMPERSAND_TOKEN: |
| 646 return TokenType.AMPERSAND_AMPERSAND; |
| 646 // case AMPERSAND_AMPERSAND_EQ_TOKEN: | 647 // case AMPERSAND_AMPERSAND_EQ_TOKEN: |
| 647 // return TokenType.AMPERSAND_AMPERSAND_EQ; | 648 // return TokenType.AMPERSAND_AMPERSAND_EQ; |
| 648 case AMPERSAND_EQ_TOKEN: return TokenType.AMPERSAND_EQ; | 649 case AMPERSAND_EQ_TOKEN: |
| 649 case AT_TOKEN: return TokenType.AT; | 650 return TokenType.AMPERSAND_EQ; |
| 650 case BANG_TOKEN: return TokenType.BANG; | 651 case AT_TOKEN: |
| 651 case BANG_EQ_TOKEN: return TokenType.BANG_EQ; | 652 return TokenType.AT; |
| 652 case BAR_TOKEN: return TokenType.BAR; | 653 case BANG_TOKEN: |
| 653 case BAR_BAR_TOKEN: return TokenType.BAR_BAR; | 654 return TokenType.BANG; |
| 655 case BANG_EQ_TOKEN: |
| 656 return TokenType.BANG_EQ; |
| 657 case BAR_TOKEN: |
| 658 return TokenType.BAR; |
| 659 case BAR_BAR_TOKEN: |
| 660 return TokenType.BAR_BAR; |
| 654 // case BAR_BAR_EQ_TOKEN: return TokenType.BAR_BAR_EQ; | 661 // case BAR_BAR_EQ_TOKEN: return TokenType.BAR_BAR_EQ; |
| 655 case BAR_EQ_TOKEN: return TokenType.BAR_EQ; | 662 case BAR_EQ_TOKEN: |
| 656 case COLON_TOKEN: return TokenType.COLON; | 663 return TokenType.BAR_EQ; |
| 657 case COMMA_TOKEN: return TokenType.COMMA; | 664 case COLON_TOKEN: |
| 658 case CARET_TOKEN: return TokenType.CARET; | 665 return TokenType.COLON; |
| 659 case CARET_EQ_TOKEN: return TokenType.CARET_EQ; | 666 case COMMA_TOKEN: |
| 660 case CLOSE_CURLY_BRACKET_TOKEN: return TokenType.CLOSE_CURLY_BRACKET; | 667 return TokenType.COMMA; |
| 661 case CLOSE_PAREN_TOKEN: return TokenType.CLOSE_PAREN; | 668 case CARET_TOKEN: |
| 662 case CLOSE_SQUARE_BRACKET_TOKEN: return TokenType.CLOSE_SQUARE_BRACKET; | 669 return TokenType.CARET; |
| 663 case EQ_TOKEN: return TokenType.EQ; | 670 case CARET_EQ_TOKEN: |
| 664 case EQ_EQ_TOKEN: return TokenType.EQ_EQ; | 671 return TokenType.CARET_EQ; |
| 665 case FUNCTION_TOKEN: return TokenType.FUNCTION; | 672 case CLOSE_CURLY_BRACKET_TOKEN: |
| 666 case GT_TOKEN: return TokenType.GT; | 673 return TokenType.CLOSE_CURLY_BRACKET; |
| 667 case GT_EQ_TOKEN: return TokenType.GT_EQ; | 674 case CLOSE_PAREN_TOKEN: |
| 668 case GT_GT_TOKEN: return TokenType.GT_GT; | 675 return TokenType.CLOSE_PAREN; |
| 669 case GT_GT_EQ_TOKEN: return TokenType.GT_GT_EQ; | 676 case CLOSE_SQUARE_BRACKET_TOKEN: |
| 670 case HASH_TOKEN: return TokenType.HASH; | 677 return TokenType.CLOSE_SQUARE_BRACKET; |
| 671 case INDEX_TOKEN: return TokenType.INDEX; | 678 case EQ_TOKEN: |
| 672 case INDEX_EQ_TOKEN: return TokenType.INDEX_EQ; | 679 return TokenType.EQ; |
| 680 case EQ_EQ_TOKEN: |
| 681 return TokenType.EQ_EQ; |
| 682 case FUNCTION_TOKEN: |
| 683 return TokenType.FUNCTION; |
| 684 case GT_TOKEN: |
| 685 return TokenType.GT; |
| 686 case GT_EQ_TOKEN: |
| 687 return TokenType.GT_EQ; |
| 688 case GT_GT_TOKEN: |
| 689 return TokenType.GT_GT; |
| 690 case GT_GT_EQ_TOKEN: |
| 691 return TokenType.GT_GT_EQ; |
| 692 case HASH_TOKEN: |
| 693 return TokenType.HASH; |
| 694 case INDEX_TOKEN: |
| 695 return TokenType.INDEX; |
| 696 case INDEX_EQ_TOKEN: |
| 697 return TokenType.INDEX_EQ; |
| 673 // case IS_TOKEN: return TokenType.IS; | 698 // case IS_TOKEN: return TokenType.IS; |
| 674 case LT_TOKEN: return TokenType.LT; | 699 case LT_TOKEN: |
| 675 case LT_EQ_TOKEN: return TokenType.LT_EQ; | 700 return TokenType.LT; |
| 676 case LT_LT_TOKEN: return TokenType.LT_LT; | 701 case LT_EQ_TOKEN: |
| 677 case LT_LT_EQ_TOKEN: return TokenType.LT_LT_EQ; | 702 return TokenType.LT_EQ; |
| 678 case MINUS_TOKEN: return TokenType.MINUS; | 703 case LT_LT_TOKEN: |
| 679 case MINUS_EQ_TOKEN: return TokenType.MINUS_EQ; | 704 return TokenType.LT_LT; |
| 680 case MINUS_MINUS_TOKEN: return TokenType.MINUS_MINUS; | 705 case LT_LT_EQ_TOKEN: |
| 681 case OPEN_CURLY_BRACKET_TOKEN: return TokenType.OPEN_CURLY_BRACKET; | 706 return TokenType.LT_LT_EQ; |
| 682 case OPEN_PAREN_TOKEN: return TokenType.OPEN_PAREN; | 707 case MINUS_TOKEN: |
| 683 case OPEN_SQUARE_BRACKET_TOKEN: return TokenType.OPEN_SQUARE_BRACKET; | 708 return TokenType.MINUS; |
| 684 case PERCENT_TOKEN: return TokenType.PERCENT; | 709 case MINUS_EQ_TOKEN: |
| 685 case PERCENT_EQ_TOKEN: return TokenType.PERCENT_EQ; | 710 return TokenType.MINUS_EQ; |
| 686 case PERIOD_TOKEN: return TokenType.PERIOD; | 711 case MINUS_MINUS_TOKEN: |
| 687 case PERIOD_PERIOD_TOKEN: return TokenType.PERIOD_PERIOD; | 712 return TokenType.MINUS_MINUS; |
| 688 case PLUS_TOKEN: return TokenType.PLUS; | 713 case OPEN_CURLY_BRACKET_TOKEN: |
| 689 case PLUS_EQ_TOKEN: return TokenType.PLUS_EQ; | 714 return TokenType.OPEN_CURLY_BRACKET; |
| 690 case PLUS_PLUS_TOKEN: return TokenType.PLUS_PLUS; | 715 case OPEN_PAREN_TOKEN: |
| 691 case QUESTION_TOKEN: return TokenType.QUESTION; | 716 return TokenType.OPEN_PAREN; |
| 692 case QUESTION_PERIOD_TOKEN: return TokenType.QUESTION_PERIOD; | 717 case OPEN_SQUARE_BRACKET_TOKEN: |
| 693 case QUESTION_QUESTION_TOKEN: return TokenType.QUESTION_QUESTION; | 718 return TokenType.OPEN_SQUARE_BRACKET; |
| 694 case QUESTION_QUESTION_EQ_TOKEN: return TokenType.QUESTION_QUESTION_EQ; | 719 case PERCENT_TOKEN: |
| 695 case SEMICOLON_TOKEN: return TokenType.SEMICOLON; | 720 return TokenType.PERCENT; |
| 696 case SLASH_TOKEN: return TokenType.SLASH; | 721 case PERCENT_EQ_TOKEN: |
| 697 case SLASH_EQ_TOKEN: return TokenType.SLASH_EQ; | 722 return TokenType.PERCENT_EQ; |
| 698 case STAR_TOKEN: return TokenType.STAR; | 723 case PERIOD_TOKEN: |
| 699 case STAR_EQ_TOKEN: return TokenType.STAR_EQ; | 724 return TokenType.PERIOD; |
| 725 case PERIOD_PERIOD_TOKEN: |
| 726 return TokenType.PERIOD_PERIOD; |
| 727 case PLUS_TOKEN: |
| 728 return TokenType.PLUS; |
| 729 case PLUS_EQ_TOKEN: |
| 730 return TokenType.PLUS_EQ; |
| 731 case PLUS_PLUS_TOKEN: |
| 732 return TokenType.PLUS_PLUS; |
| 733 case QUESTION_TOKEN: |
| 734 return TokenType.QUESTION; |
| 735 case QUESTION_PERIOD_TOKEN: |
| 736 return TokenType.QUESTION_PERIOD; |
| 737 case QUESTION_QUESTION_TOKEN: |
| 738 return TokenType.QUESTION_QUESTION; |
| 739 case QUESTION_QUESTION_EQ_TOKEN: |
| 740 return TokenType.QUESTION_QUESTION_EQ; |
| 741 case SEMICOLON_TOKEN: |
| 742 return TokenType.SEMICOLON; |
| 743 case SLASH_TOKEN: |
| 744 return TokenType.SLASH; |
| 745 case SLASH_EQ_TOKEN: |
| 746 return TokenType.SLASH_EQ; |
| 747 case STAR_TOKEN: |
| 748 return TokenType.STAR; |
| 749 case STAR_EQ_TOKEN: |
| 750 return TokenType.STAR_EQ; |
| 700 case STRING_INTERPOLATION_TOKEN: | 751 case STRING_INTERPOLATION_TOKEN: |
| 701 return TokenType.STRING_INTERPOLATION_EXPRESSION; | 752 return TokenType.STRING_INTERPOLATION_EXPRESSION; |
| 702 case STRING_INTERPOLATION_IDENTIFIER_TOKEN: | 753 case STRING_INTERPOLATION_IDENTIFIER_TOKEN: |
| 703 return TokenType.STRING_INTERPOLATION_IDENTIFIER; | 754 return TokenType.STRING_INTERPOLATION_IDENTIFIER; |
| 704 case TILDE_TOKEN: return TokenType.TILDE; | 755 case TILDE_TOKEN: |
| 705 case TILDE_SLASH_TOKEN: return TokenType.TILDE_SLASH; | 756 return TokenType.TILDE; |
| 706 case TILDE_SLASH_EQ_TOKEN: return TokenType.TILDE_SLASH_EQ; | 757 case TILDE_SLASH_TOKEN: |
| 707 case BACKPING_TOKEN: return TokenType.BACKPING; | 758 return TokenType.TILDE_SLASH; |
| 708 case BACKSLASH_TOKEN: return TokenType.BACKSLASH; | 759 case TILDE_SLASH_EQ_TOKEN: |
| 709 case PERIOD_PERIOD_PERIOD_TOKEN: return TokenType.PERIOD_PERIOD_PERIOD; | 760 return TokenType.TILDE_SLASH_EQ; |
| 761 case BACKPING_TOKEN: |
| 762 return TokenType.BACKPING; |
| 763 case BACKSLASH_TOKEN: |
| 764 return TokenType.BACKSLASH; |
| 765 case PERIOD_PERIOD_PERIOD_TOKEN: |
| 766 return TokenType.PERIOD_PERIOD_PERIOD; |
| 710 // case GENERIC_METHOD_TYPE_LIST_TOKEN: | 767 // case GENERIC_METHOD_TYPE_LIST_TOKEN: |
| 711 // return TokenType.GENERIC_METHOD_TYPE_LIST; | 768 // return TokenType.GENERIC_METHOD_TYPE_LIST; |
| 712 // case GENERIC_METHOD_TYPE_ASSIGN_TOKEN: | 769 // case GENERIC_METHOD_TYPE_ASSIGN_TOKEN: |
| 713 // return TokenType.GENERIC_METHOD_TYPE_ASSIGN; | 770 // return TokenType.GENERIC_METHOD_TYPE_ASSIGN; |
| 714 default: | 771 default: |
| 715 return internalError("Unhandled token ${token.info}"); | 772 return internalError("Unhandled token ${token.info}"); |
| 716 } | 773 } |
| 717 } | 774 } |
| OLD | NEW |