| OLD | NEW |
| 1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /** | 5 /** |
| 6 * Defines the tokens that are produced by the scanner, used by the parser, and | 6 * Defines the tokens that are produced by the scanner, used by the parser, and |
| 7 * referenced from the [AST structure](ast.dart). | 7 * referenced from the [AST structure](ast.dart). |
| 8 */ | 8 */ |
| 9 import 'dart:collection'; | 9 import 'dart:collection'; |
| 10 | 10 |
| 11 import 'package:front_end/src/base/syntactic_entity.dart'; | 11 import 'package:front_end/src/base/syntactic_entity.dart'; |
| 12 import 'package:front_end/src/fasta/scanner/precedence.dart'; | 12 import 'package:front_end/src/fasta/scanner/precedence.dart'; |
| 13 import 'package:front_end/src/scanner/string_utilities.dart'; | 13 import 'package:front_end/src/scanner/string_utilities.dart'; |
| 14 import 'package:front_end/src/fasta/scanner/precedence.dart' as fasta; | 14 import 'package:front_end/src/fasta/scanner/precedence.dart' as fasta; |
| 15 | 15 |
| 16 const int NO_PRECEDENCE = 0; |
| 17 const int ASSIGNMENT_PRECEDENCE = 1; |
| 18 const int CASCADE_PRECEDENCE = 2; |
| 19 const int CONDITIONAL_PRECEDENCE = 3; |
| 20 const int IF_NULL_PRECEDENCE = 4; |
| 21 const int LOGICAL_OR_PRECEDENCE = 5; |
| 22 const int LOGICAL_AND_PRECEDENCE = 6; |
| 23 const int EQUALITY_PRECEDENCE = 7; |
| 24 const int RELATIONAL_PRECEDENCE = 8; |
| 25 const int BITWISE_OR_PRECEDENCE = 9; |
| 26 const int BITWISE_XOR_PRECEDENCE = 10; |
| 27 const int BITWISE_AND_PRECEDENCE = 11; |
| 28 const int SHIFT_PRECEDENCE = 12; |
| 29 const int ADDITIVE_PRECEDENCE = 13; |
| 30 const int MULTIPLICATIVE_PRECEDENCE = 14; |
| 31 const int PREFIX_PRECEDENCE = 15; |
| 32 const int POSTFIX_PRECEDENCE = 16; |
| 33 |
| 16 /** | 34 /** |
| 17 * The opening half of a grouping pair of tokens. This is used for curly | 35 * The opening half of a grouping pair of tokens. This is used for curly |
| 18 * brackets ('{'), parentheses ('('), and square brackets ('['). | 36 * brackets ('{'), parentheses ('('), and square brackets ('['). |
| 19 */ | 37 */ |
| 20 class BeginToken extends SimpleToken { | 38 class BeginToken extends SimpleToken { |
| 21 /** | 39 /** |
| 22 * The token that corresponds to this token. | 40 * The token that corresponds to this token. |
| 23 */ | 41 */ |
| 24 Token endToken; | 42 Token endToken; |
| 25 | 43 |
| (...skipping 313 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 339 WHILE, | 357 WHILE, |
| 340 WITH, | 358 WITH, |
| 341 YIELD, | 359 YIELD, |
| 342 ]; | 360 ]; |
| 343 | 361 |
| 344 /** | 362 /** |
| 345 * A table mapping the lexemes of keywords to the corresponding keyword. | 363 * A table mapping the lexemes of keywords to the corresponding keyword. |
| 346 */ | 364 */ |
| 347 static final Map<String, Keyword> keywords = _createKeywordMap(); | 365 static final Map<String, Keyword> keywords = _createKeywordMap(); |
| 348 | 366 |
| 349 final fasta.PrecedenceInfo info; | 367 final TokenType info; |
| 350 | 368 |
| 351 /** | 369 /** |
| 352 * A flag indicating whether the keyword is "built-in" identifier. | 370 * A flag indicating whether the keyword is "built-in" identifier. |
| 353 */ | 371 */ |
| 354 final bool isBuiltIn; | 372 final bool isBuiltIn; |
| 355 | 373 |
| 356 /** | 374 /** |
| 357 * A flag indicating whether the keyword can be used as an identifier | 375 * A flag indicating whether the keyword can be used as an identifier |
| 358 * in some situations. | 376 * in some situations. |
| 359 */ | 377 */ |
| (...skipping 645 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1005 | 1023 |
| 1006 static const TokenType SINGLE_LINE_COMMENT = fasta.SINGLE_LINE_COMMENT_INFO; | 1024 static const TokenType SINGLE_LINE_COMMENT = fasta.SINGLE_LINE_COMMENT_INFO; |
| 1007 | 1025 |
| 1008 static const TokenType STRING = fasta.STRING_INFO; | 1026 static const TokenType STRING = fasta.STRING_INFO; |
| 1009 | 1027 |
| 1010 static const TokenType AMPERSAND = fasta.AMPERSAND_INFO; | 1028 static const TokenType AMPERSAND = fasta.AMPERSAND_INFO; |
| 1011 | 1029 |
| 1012 static const TokenType AMPERSAND_AMPERSAND = fasta.AMPERSAND_AMPERSAND_INFO; | 1030 static const TokenType AMPERSAND_AMPERSAND = fasta.AMPERSAND_AMPERSAND_INFO; |
| 1013 | 1031 |
| 1014 static const TokenType AMPERSAND_AMPERSAND_EQ = | 1032 static const TokenType AMPERSAND_AMPERSAND_EQ = |
| 1015 const fasta.PrecedenceInfo('&&=', 'AMPERSAND_AMPERSAND_EQ', 1, -1); | 1033 const TokenType('&&=', 'AMPERSAND_AMPERSAND_EQ', 1, -1); |
| 1016 | 1034 |
| 1017 static const TokenType AMPERSAND_EQ = fasta.AMPERSAND_EQ_INFO; | 1035 static const TokenType AMPERSAND_EQ = fasta.AMPERSAND_EQ_INFO; |
| 1018 | 1036 |
| 1019 static const TokenType AT = fasta.AT_INFO; | 1037 static const TokenType AT = fasta.AT_INFO; |
| 1020 | 1038 |
| 1021 static const TokenType BANG = fasta.BANG_INFO; | 1039 static const TokenType BANG = fasta.BANG_INFO; |
| 1022 | 1040 |
| 1023 static const TokenType BANG_EQ = fasta.BANG_EQ_INFO; | 1041 static const TokenType BANG_EQ = fasta.BANG_EQ_INFO; |
| 1024 | 1042 |
| 1025 static const TokenType BAR = fasta.BAR_INFO; | 1043 static const TokenType BAR = fasta.BAR_INFO; |
| 1026 | 1044 |
| 1027 static const TokenType BAR_BAR = fasta.BAR_BAR_INFO; | 1045 static const TokenType BAR_BAR = fasta.BAR_BAR_INFO; |
| 1028 | 1046 |
| 1029 static const TokenType BAR_BAR_EQ = | 1047 static const TokenType BAR_BAR_EQ = |
| 1030 const fasta.PrecedenceInfo('||=', 'BAR_BAR_EQ', 1, -1); | 1048 const TokenType('||=', 'BAR_BAR_EQ', 1, -1); |
| 1031 | 1049 |
| 1032 static const TokenType BAR_EQ = fasta.BAR_EQ_INFO; | 1050 static const TokenType BAR_EQ = fasta.BAR_EQ_INFO; |
| 1033 | 1051 |
| 1034 static const TokenType COLON = fasta.COLON_INFO; | 1052 static const TokenType COLON = fasta.COLON_INFO; |
| 1035 | 1053 |
| 1036 static const TokenType COMMA = fasta.COMMA_INFO; | 1054 static const TokenType COMMA = fasta.COMMA_INFO; |
| 1037 | 1055 |
| 1038 static const TokenType CARET = fasta.CARET_INFO; | 1056 static const TokenType CARET = fasta.CARET_INFO; |
| 1039 | 1057 |
| 1040 static const TokenType CARET_EQ = fasta.CARET_EQ_INFO; | 1058 static const TokenType CARET_EQ = fasta.CARET_EQ_INFO; |
| (...skipping 93 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1134 static const TokenType BACKSLASH = fasta.BACKSLASH_INFO; | 1152 static const TokenType BACKSLASH = fasta.BACKSLASH_INFO; |
| 1135 | 1153 |
| 1136 static const TokenType PERIOD_PERIOD_PERIOD = fasta.PERIOD_PERIOD_PERIOD_INFO; | 1154 static const TokenType PERIOD_PERIOD_PERIOD = fasta.PERIOD_PERIOD_PERIOD_INFO; |
| 1137 | 1155 |
| 1138 static const TokenType GENERIC_METHOD_TYPE_LIST = | 1156 static const TokenType GENERIC_METHOD_TYPE_LIST = |
| 1139 fasta.GENERIC_METHOD_TYPE_LIST; | 1157 fasta.GENERIC_METHOD_TYPE_LIST; |
| 1140 | 1158 |
| 1141 static const TokenType GENERIC_METHOD_TYPE_ASSIGN = | 1159 static const TokenType GENERIC_METHOD_TYPE_ASSIGN = |
| 1142 fasta.GENERIC_METHOD_TYPE_ASSIGN; | 1160 fasta.GENERIC_METHOD_TYPE_ASSIGN; |
| 1143 | 1161 |
| 1162 static const List<TokenType> all = const <TokenType>[ |
| 1163 TokenType.EOF, |
| 1164 TokenType.DOUBLE, |
| 1165 TokenType.HEXADECIMAL, |
| 1166 TokenType.IDENTIFIER, |
| 1167 TokenType.INT, |
| 1168 TokenType.KEYWORD, |
| 1169 TokenType.MULTI_LINE_COMMENT, |
| 1170 TokenType.SCRIPT_TAG, |
| 1171 TokenType.SINGLE_LINE_COMMENT, |
| 1172 TokenType.STRING, |
| 1173 TokenType.AMPERSAND, |
| 1174 TokenType.AMPERSAND_AMPERSAND, |
| 1175 TokenType.AMPERSAND_EQ, |
| 1176 TokenType.AT, |
| 1177 TokenType.BANG, |
| 1178 TokenType.BANG_EQ, |
| 1179 TokenType.BAR, |
| 1180 TokenType.BAR_BAR, |
| 1181 TokenType.BAR_EQ, |
| 1182 TokenType.COLON, |
| 1183 TokenType.COMMA, |
| 1184 TokenType.CARET, |
| 1185 TokenType.CARET_EQ, |
| 1186 TokenType.CLOSE_CURLY_BRACKET, |
| 1187 TokenType.CLOSE_PAREN, |
| 1188 TokenType.CLOSE_SQUARE_BRACKET, |
| 1189 TokenType.EQ, |
| 1190 TokenType.EQ_EQ, |
| 1191 TokenType.FUNCTION, |
| 1192 TokenType.GT, |
| 1193 TokenType.GT_EQ, |
| 1194 TokenType.GT_GT, |
| 1195 TokenType.GT_GT_EQ, |
| 1196 TokenType.HASH, |
| 1197 TokenType.INDEX, |
| 1198 TokenType.INDEX_EQ, |
| 1199 TokenType.LT, |
| 1200 TokenType.LT_EQ, |
| 1201 TokenType.LT_LT, |
| 1202 TokenType.LT_LT_EQ, |
| 1203 TokenType.MINUS, |
| 1204 TokenType.MINUS_EQ, |
| 1205 TokenType.MINUS_MINUS, |
| 1206 TokenType.OPEN_CURLY_BRACKET, |
| 1207 TokenType.OPEN_PAREN, |
| 1208 TokenType.OPEN_SQUARE_BRACKET, |
| 1209 TokenType.PERCENT, |
| 1210 TokenType.PERCENT_EQ, |
| 1211 TokenType.PERIOD, |
| 1212 TokenType.PERIOD_PERIOD, |
| 1213 TokenType.PLUS, |
| 1214 TokenType.PLUS_EQ, |
| 1215 TokenType.PLUS_PLUS, |
| 1216 TokenType.QUESTION, |
| 1217 TokenType.QUESTION_PERIOD, |
| 1218 TokenType.QUESTION_QUESTION, |
| 1219 TokenType.QUESTION_QUESTION_EQ, |
| 1220 TokenType.SEMICOLON, |
| 1221 TokenType.SLASH, |
| 1222 TokenType.SLASH_EQ, |
| 1223 TokenType.STAR, |
| 1224 TokenType.STAR_EQ, |
| 1225 TokenType.STRING_INTERPOLATION_EXPRESSION, |
| 1226 TokenType.STRING_INTERPOLATION_IDENTIFIER, |
| 1227 TokenType.TILDE, |
| 1228 TokenType.TILDE_SLASH, |
| 1229 TokenType.TILDE_SLASH_EQ, |
| 1230 TokenType.BACKPING, |
| 1231 TokenType.BACKSLASH, |
| 1232 TokenType.PERIOD_PERIOD_PERIOD, |
| 1233 TokenType.GENERIC_METHOD_TYPE_LIST, |
| 1234 TokenType.GENERIC_METHOD_TYPE_ASSIGN, |
| 1235 |
| 1236 // These are not yet part of the language and not supported by fasta |
| 1237 //TokenType.AMPERSAND_AMPERSAND_EQ, |
| 1238 //TokenType.BAR_BAR_EQ, |
| 1239 ]; |
| 1240 |
| 1144 final int kind; | 1241 final int kind; |
| 1145 | 1242 |
| 1146 /** | 1243 /** |
| 1147 * `true` if this token type represents an operator. | 1244 * `true` if this token type represents an operator. |
| 1148 */ | 1245 */ |
| 1149 final bool isOperator; | 1246 final bool isOperator; |
| 1150 | 1247 |
| 1151 /** | 1248 /** |
| 1152 * `true` if this token type represents an operator | 1249 * `true` if this token type represents an operator |
| 1153 * that can be defined by users. | 1250 * that can be defined by users. |
| (...skipping 128 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1282 | 1379 |
| 1283 void set precedingComments(CommentToken comment) { | 1380 void set precedingComments(CommentToken comment) { |
| 1284 _precedingComment = comment; | 1381 _precedingComment = comment; |
| 1285 _setCommentParent(_precedingComment); | 1382 _setCommentParent(_precedingComment); |
| 1286 } | 1383 } |
| 1287 | 1384 |
| 1288 @override | 1385 @override |
| 1289 Token copy() => | 1386 Token copy() => |
| 1290 new TokenWithComment(type, offset, copyComments(precedingComments)); | 1387 new TokenWithComment(type, offset, copyComments(precedingComments)); |
| 1291 } | 1388 } |
| OLD | NEW |