| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file | |
| 2 // for details. All rights reserved. Use of this source code is governed by a | |
| 3 // BSD-style license that can be found in the LICENSE file. | |
| 4 | |
| 5 library fasta.analyzer.token_utils; | |
| 6 | |
| 7 import '../parser/error_kind.dart' show ErrorKind; | |
| 8 | |
| 9 import '../scanner/error_token.dart' show ErrorToken; | |
| 10 | |
| 11 import '../scanner/keyword.dart' show Keyword; | |
| 12 | |
| 13 import '../scanner/precedence.dart'; | |
| 14 | |
| 15 import '../scanner/token.dart' | |
| 16 show BeginGroupToken, KeywordToken, StringToken, SymbolToken, Token; | |
| 17 | |
| 18 import '../scanner/token_constants.dart'; | |
| 19 | |
| 20 import '../../scanner/token.dart' as analyzer | |
| 21 show | |
| 22 BeginToken, | |
| 23 BeginTokenWithComment, | |
| 24 CommentToken, | |
| 25 Keyword, | |
| 26 KeywordToken, | |
| 27 KeywordTokenWithComment, | |
| 28 StringToken, | |
| 29 StringTokenWithComment, | |
| 30 Token, | |
| 31 TokenWithComment; | |
| 32 | |
| 33 import '../../scanner/errors.dart' as analyzer show ScannerErrorCode; | |
| 34 | |
| 35 import 'package:analyzer/dart/ast/token.dart' show TokenType; | |
| 36 | |
| 37 import '../errors.dart' show internalError; | |
| 38 | |
| 39 /// Class capable of converting a stream of Fasta tokens to a stream of analyzer | |
| 40 /// tokens. | |
| 41 /// | |
| 42 /// This is a class rather than an ordinary method so that it can be subclassed | |
| 43 /// in tests. | |
| 44 /// | |
| 45 /// TODO(paulberry,ahe): Fasta includes comments directly in the token | |
| 46 /// stream, rather than pointing to them via a "precedingComment" pointer, as | |
| 47 /// analyzer does. This seems like it will complicate parsing and other | |
| 48 /// operations. | |
| 49 class ToAnalyzerTokenStreamConverter { | |
| 50 /// Synthetic token pointing to the first token in the analyzer token stream. | |
| 51 analyzer.Token _analyzerTokenHead; | |
| 52 | |
| 53 /// The most recently generated analyzer token, or [_analyzerTokenHead] if no | |
| 54 /// tokens have been generated yet. | |
| 55 analyzer.Token _analyzerTokenTail; | |
| 56 | |
| 57 /// Stack of analyzer "begin" tokens which need to be linked up to | |
| 58 /// corresponding "end" tokens once those tokens are translated. | |
| 59 /// | |
| 60 /// The first element of this list is always a sentinel `null` value so that | |
| 61 /// we don't have to check if it is empty. | |
| 62 /// | |
| 63 /// See additional documentation in [_matchGroups]. | |
| 64 List<analyzer.BeginToken> _beginTokenStack; | |
| 65 | |
| 66 /// Stack of fasta "end" tokens corresponding to the tokens in | |
| 67 /// [_endTokenStack]. | |
| 68 /// | |
| 69 /// The first element of this list is always a sentinel `null` value so that | |
| 70 /// we don't have to check if it is empty. | |
| 71 /// | |
| 72 /// See additional documentation in [_matchGroups]. | |
| 73 List<Token> _endTokenStack; | |
| 74 | |
| 75 /// Converts a stream of Fasta tokens (starting with [token] and continuing to | |
| 76 /// EOF) to a stream of analyzer tokens. | |
| 77 analyzer.Token convertTokens(Token token) { | |
| 78 _analyzerTokenHead = new analyzer.Token(TokenType.EOF, -1); | |
| 79 _analyzerTokenHead.previous = _analyzerTokenHead; | |
| 80 _analyzerTokenTail = _analyzerTokenHead; | |
| 81 _beginTokenStack = [null]; | |
| 82 _endTokenStack = <Token>[null]; | |
| 83 | |
| 84 while (true) { | |
| 85 if (token.info.kind == BAD_INPUT_TOKEN) { | |
| 86 ErrorToken errorToken = token; | |
| 87 _translateErrorToken(errorToken); | |
| 88 } else { | |
| 89 var translatedToken = translateToken( | |
| 90 token, translateCommentTokens(token.precedingComments)); | |
| 91 _matchGroups(token, translatedToken); | |
| 92 translatedToken.setNext(translatedToken); | |
| 93 _analyzerTokenTail.setNext(translatedToken); | |
| 94 translatedToken.previous = _analyzerTokenTail; | |
| 95 _analyzerTokenTail = translatedToken; | |
| 96 } | |
| 97 if (token.isEof) { | |
| 98 return _analyzerTokenHead.next; | |
| 99 } | |
| 100 token = token.next; | |
| 101 } | |
| 102 } | |
| 103 | |
| 104 /// Handles an error found during [convertTokens]. | |
| 105 /// | |
| 106 /// Intended to be overridden by derived classes; by default, does nothing. | |
| 107 void reportError(analyzer.ScannerErrorCode errorCode, int offset, | |
| 108 List<Object> arguments) {} | |
| 109 | |
| 110 /// Translates a sequence of fasta comment tokens to the corresponding | |
| 111 /// analyzer tokens. | |
| 112 analyzer.CommentToken translateCommentTokens(Token token) { | |
| 113 analyzer.CommentToken head; | |
| 114 if (token != null) { | |
| 115 head = toAnalyzerCommentToken(token); | |
| 116 analyzer.CommentToken tail = head; | |
| 117 token = token.next; | |
| 118 while (token != null) { | |
| 119 tail = tail.setNext(toAnalyzerCommentToken(token)); | |
| 120 token = token.next; | |
| 121 } | |
| 122 } | |
| 123 return head; | |
| 124 } | |
| 125 | |
| 126 /// Translates a single fasta non-comment token to the corresponding analyzer | |
| 127 /// token. | |
| 128 /// | |
| 129 /// [precedingComments] is not `null`, the translated token is pointed to it. | |
| 130 analyzer.Token translateToken( | |
| 131 Token token, analyzer.CommentToken precedingComments) => | |
| 132 toAnalyzerToken(token, precedingComments); | |
| 133 | |
| 134 /// Creates appropriate begin/end token links based on the fact that [token] | |
| 135 /// was translated to [translatedToken]. | |
| 136 /// | |
| 137 /// Background: both fasta and analyzer have links from a "BeginToken" to its | |
| 138 /// matching "EndToken" in a group (like parentheses and braces). However, | |
| 139 /// fasta may contain synthetic tokens from error recovery that are not mapped | |
| 140 /// to the analyzer token stream. We use [_beginTokenStack] and | |
| 141 /// [_endTokenStack] to create the appropriate links for non-synthetic tokens | |
| 142 /// in the way analyzer expects. | |
| 143 void _matchGroups(Token token, analyzer.Token translatedToken) { | |
| 144 if (identical(_endTokenStack.last, token)) { | |
| 145 _beginTokenStack.last.endToken = translatedToken; | |
| 146 _beginTokenStack.removeLast(); | |
| 147 _endTokenStack.removeLast(); | |
| 148 } | |
| 149 // Synthetic end tokens use the same offset as the begin token. | |
| 150 if (translatedToken is analyzer.BeginToken && | |
| 151 token is BeginGroupToken && | |
| 152 token.endGroup != null && | |
| 153 token.endGroup.charOffset != token.charOffset) { | |
| 154 _beginTokenStack.add(translatedToken); | |
| 155 _endTokenStack.add(token.endGroup); | |
| 156 } | |
| 157 } | |
| 158 | |
| 159 /// Translates the given error [token] into an analyzer error and reports it | |
| 160 /// using [reportError]. | |
| 161 void _translateErrorToken(ErrorToken token) { | |
| 162 int charOffset = token.charOffset; | |
| 163 // TODO(paulberry,ahe): why is endOffset sometimes null? | |
| 164 int endOffset = token.endOffset ?? charOffset; | |
| 165 void _makeError( | |
| 166 analyzer.ScannerErrorCode errorCode, List<Object> arguments) { | |
| 167 if (_isAtEnd(token, charOffset)) { | |
| 168 // Analyzer never generates an error message past the end of the input, | |
| 169 // since such an error would not be visible in an editor. | |
| 170 // TODO(paulberry,ahe): would it make sense to replicate this behavior | |
| 171 // in fasta, or move it elsewhere in analyzer? | |
| 172 charOffset--; | |
| 173 } | |
| 174 reportError(errorCode, charOffset, arguments); | |
| 175 } | |
| 176 | |
| 177 var errorCode = token.errorCode; | |
| 178 switch (errorCode) { | |
| 179 case ErrorKind.UnterminatedString: | |
| 180 // TODO(paulberry,ahe): Fasta reports the error location as the entire | |
| 181 // string; analyzer expects the end of the string. | |
| 182 charOffset = endOffset; | |
| 183 return _makeError( | |
| 184 analyzer.ScannerErrorCode.UNTERMINATED_STRING_LITERAL, null); | |
| 185 case ErrorKind.UnmatchedToken: | |
| 186 return null; | |
| 187 case ErrorKind.UnterminatedComment: | |
| 188 // TODO(paulberry,ahe): Fasta reports the error location as the entire | |
| 189 // comment; analyzer expects the end of the comment. | |
| 190 charOffset = endOffset; | |
| 191 return _makeError( | |
| 192 analyzer.ScannerErrorCode.UNTERMINATED_MULTI_LINE_COMMENT, null); | |
| 193 case ErrorKind.MissingExponent: | |
| 194 // TODO(paulberry,ahe): Fasta reports the error location as the entire | |
| 195 // number; analyzer expects the end of the number. | |
| 196 charOffset = endOffset; | |
| 197 return _makeError(analyzer.ScannerErrorCode.MISSING_DIGIT, null); | |
| 198 case ErrorKind.ExpectedHexDigit: | |
| 199 // TODO(paulberry,ahe): Fasta reports the error location as the entire | |
| 200 // number; analyzer expects the end of the number. | |
| 201 charOffset = endOffset; | |
| 202 return _makeError(analyzer.ScannerErrorCode.MISSING_HEX_DIGIT, null); | |
| 203 case ErrorKind.NonAsciiIdentifier: | |
| 204 case ErrorKind.NonAsciiWhitespace: | |
| 205 return _makeError( | |
| 206 analyzer.ScannerErrorCode.ILLEGAL_CHARACTER, [token.character]); | |
| 207 case ErrorKind.UnexpectedDollarInString: | |
| 208 return null; | |
| 209 default: | |
| 210 throw new UnimplementedError('$errorCode'); | |
| 211 } | |
| 212 } | |
| 213 } | |
| 214 | |
| 215 /// Converts a single Fasta comment token to an analyzer comment token. | |
| 216 analyzer.CommentToken toAnalyzerCommentToken(Token token) { | |
| 217 // TODO(paulberry,ahe): It would be nice if the scanner gave us an | |
| 218 // easier way to distinguish between the two types of comment. | |
| 219 var type = token.lexeme.startsWith('/*') | |
| 220 ? TokenType.MULTI_LINE_COMMENT | |
| 221 : TokenType.SINGLE_LINE_COMMENT; | |
| 222 return new analyzer.CommentToken(type, token.lexeme, token.charOffset); | |
| 223 } | |
| 224 | |
| 225 /// Converts a stream of Analyzer tokens (starting with [token] and continuing | |
| 226 /// to EOF) to a stream of Fasta tokens. | |
| 227 /// | |
| 228 /// TODO(paulberry): Analyzer tokens do not record error conditions, so a round | |
| 229 /// trip through this function and [toAnalyzerTokenStream] will lose error | |
| 230 /// information. | |
| 231 Token fromAnalyzerTokenStream(analyzer.Token analyzerToken) { | |
| 232 Token tokenHead = new SymbolToken(EOF_INFO, -1); | |
| 233 Token tokenTail = tokenHead; | |
| 234 | |
| 235 // Both fasta and analyzer have links from a "BeginToken" to its matching | |
| 236 // "EndToken" in a group (like parentheses and braces). However, only fasta | |
| 237 // makes these links for angle brackets. We use these stacks to map the | |
| 238 // links from the analyzer token stream into equivalent links in the fasta | |
| 239 // token stream, and to create the links that fasta expects for angle | |
| 240 // brackets. | |
| 241 | |
| 242 // Note: beginTokenStack and endTokenStack are seeded with a sentinel value | |
| 243 // so that we don't have to check if they're empty. | |
| 244 var beginTokenStack = <BeginGroupToken>[null]; | |
| 245 var endTokenStack = <analyzer.Token>[null]; | |
| 246 var angleBracketStack = <BeginGroupToken>[]; | |
| 247 void matchGroups(analyzer.Token analyzerToken, Token translatedToken) { | |
| 248 if (identical(endTokenStack.last, analyzerToken)) { | |
| 249 angleBracketStack.clear(); | |
| 250 beginTokenStack.last.endGroup = translatedToken; | |
| 251 beginTokenStack.removeLast(); | |
| 252 endTokenStack.removeLast(); | |
| 253 } else if (translatedToken.info.kind == LT_TOKEN) { | |
| 254 BeginGroupToken beginGroupToken = translatedToken; | |
| 255 angleBracketStack.add(beginGroupToken); | |
| 256 } else if (translatedToken.info.kind == GT_TOKEN && | |
| 257 angleBracketStack.isNotEmpty) { | |
| 258 angleBracketStack.removeLast().endGroup = translatedToken; | |
| 259 } else if (translatedToken.info.kind == GT_GT_TOKEN && | |
| 260 angleBracketStack.isNotEmpty) { | |
| 261 angleBracketStack.removeLast(); | |
| 262 if (angleBracketStack.isNotEmpty) { | |
| 263 angleBracketStack.removeLast().endGroup = translatedToken; | |
| 264 } | |
| 265 } | |
| 266 // TODO(paulberry): generate synthetic closer tokens and "UnmatchedToken" | |
| 267 // tokens as appropriate. | |
| 268 if (translatedToken is BeginGroupToken && | |
| 269 analyzerToken is analyzer.BeginToken && | |
| 270 analyzerToken.endToken != null) { | |
| 271 angleBracketStack.clear(); | |
| 272 beginTokenStack.add(translatedToken); | |
| 273 endTokenStack.add(analyzerToken.endToken); | |
| 274 } | |
| 275 } | |
| 276 | |
| 277 Token translateComments(analyzer.Token token) { | |
| 278 if (token == null) { | |
| 279 return null; | |
| 280 } | |
| 281 Token head = fromAnalyzerToken(token); | |
| 282 Token tail = head; | |
| 283 token = token.next; | |
| 284 while (token != null) { | |
| 285 tail.next = fromAnalyzerToken(token); | |
| 286 tail.next.previousToken = tail; | |
| 287 tail = tail.next; | |
| 288 token = token.next; | |
| 289 } | |
| 290 return head; | |
| 291 } | |
| 292 | |
| 293 analyzer.Token translateAndAppend(analyzer.Token analyzerToken) { | |
| 294 var token = fromAnalyzerToken(analyzerToken); | |
| 295 token.precedingComments = | |
| 296 translateComments(analyzerToken.precedingComments); | |
| 297 tokenTail.next = token; | |
| 298 tokenTail.next.previousToken = tokenTail; | |
| 299 tokenTail = token; | |
| 300 matchGroups(analyzerToken, token); | |
| 301 return analyzerToken.next; | |
| 302 } | |
| 303 | |
| 304 while (true) { | |
| 305 // TODO(paulberry): join up begingroup/endgroup. | |
| 306 if (analyzerToken.type == TokenType.EOF) { | |
| 307 tokenTail.next = new SymbolToken(EOF_INFO, analyzerToken.offset); | |
| 308 tokenTail.next.previousToken = tokenTail; | |
| 309 tokenTail.next.precedingComments = | |
| 310 translateComments(analyzerToken.precedingComments); | |
| 311 return tokenHead.next; | |
| 312 } | |
| 313 analyzerToken = translateAndAppend(analyzerToken); | |
| 314 } | |
| 315 } | |
| 316 | |
| 317 /// Converts a single analyzer token into a Fasta token. | |
| 318 Token fromAnalyzerToken(analyzer.Token token) { | |
| 319 Token beginGroup(PrecedenceInfo info) => | |
| 320 new BeginGroupToken(info, token.offset); | |
| 321 Token string(PrecedenceInfo info) => | |
| 322 new StringToken.fromString(info, token.lexeme, token.offset); | |
| 323 Token symbol(PrecedenceInfo info) => new SymbolToken(info, token.offset); | |
| 324 switch (token.type) { | |
| 325 case TokenType.DOUBLE: | |
| 326 return string(DOUBLE_INFO); | |
| 327 case TokenType.HEXADECIMAL: | |
| 328 return string(HEXADECIMAL_INFO); | |
| 329 case TokenType.IDENTIFIER: | |
| 330 // Certain identifiers have special grammatical meanings even though they | |
| 331 // are neither keywords nor built-in identifiers (e.g. "async"). Analyzer | |
| 332 // represents these as identifiers. Fasta represents them as keywords | |
| 333 // with the "isPseudo" property. | |
| 334 var keyword = Keyword.keywords[token.lexeme]; | |
| 335 if (keyword != null) { | |
| 336 assert(keyword.isPseudo); | |
| 337 return new KeywordToken(keyword, token.offset); | |
| 338 } else { | |
| 339 return string(IDENTIFIER_INFO); | |
| 340 } | |
| 341 break; | |
| 342 case TokenType.INT: | |
| 343 return string(INT_INFO); | |
| 344 case TokenType.KEYWORD: | |
| 345 var keyword = Keyword.keywords[token.lexeme]; | |
| 346 if (keyword != null) { | |
| 347 return new KeywordToken(keyword, token.offset); | |
| 348 } else { | |
| 349 return internalError("Unrecognized keyword: '${token.lexeme}'."); | |
| 350 } | |
| 351 break; | |
| 352 case TokenType.MULTI_LINE_COMMENT: | |
| 353 return string(COMMENT_INFO); | |
| 354 // case TokenType.SCRIPT_TAG | |
| 355 case TokenType.SINGLE_LINE_COMMENT: | |
| 356 return string(COMMENT_INFO); | |
| 357 case TokenType.STRING: | |
| 358 return string(STRING_INFO); | |
| 359 case TokenType.AMPERSAND: | |
| 360 return symbol(AMPERSAND_INFO); | |
| 361 case TokenType.AMPERSAND_AMPERSAND: | |
| 362 return symbol(AMPERSAND_AMPERSAND_INFO); | |
| 363 // case TokenType.AMPERSAND_AMPERSAND_EQ | |
| 364 case TokenType.AMPERSAND_EQ: | |
| 365 return symbol(AMPERSAND_EQ_INFO); | |
| 366 case TokenType.AT: | |
| 367 return symbol(AT_INFO); | |
| 368 case TokenType.BANG: | |
| 369 return symbol(BANG_INFO); | |
| 370 case TokenType.BANG_EQ: | |
| 371 return symbol(BANG_EQ_INFO); | |
| 372 case TokenType.BAR: | |
| 373 return symbol(BAR_INFO); | |
| 374 case TokenType.BAR_BAR: | |
| 375 return symbol(BAR_BAR_INFO); | |
| 376 // case TokenType.BAR_BAR_EQ | |
| 377 case TokenType.BAR_EQ: | |
| 378 return symbol(BAR_EQ_INFO); | |
| 379 case TokenType.COLON: | |
| 380 return symbol(COLON_INFO); | |
| 381 case TokenType.COMMA: | |
| 382 return symbol(COMMA_INFO); | |
| 383 case TokenType.CARET: | |
| 384 return symbol(CARET_INFO); | |
| 385 case TokenType.CARET_EQ: | |
| 386 return symbol(CARET_EQ_INFO); | |
| 387 case TokenType.CLOSE_CURLY_BRACKET: | |
| 388 return symbol(CLOSE_CURLY_BRACKET_INFO); | |
| 389 case TokenType.CLOSE_PAREN: | |
| 390 return symbol(CLOSE_PAREN_INFO); | |
| 391 case TokenType.CLOSE_SQUARE_BRACKET: | |
| 392 return symbol(CLOSE_SQUARE_BRACKET_INFO); | |
| 393 case TokenType.EQ: | |
| 394 return symbol(EQ_INFO); | |
| 395 case TokenType.EQ_EQ: | |
| 396 return symbol(EQ_EQ_INFO); | |
| 397 case TokenType.FUNCTION: | |
| 398 return symbol(FUNCTION_INFO); | |
| 399 case TokenType.GT: | |
| 400 return symbol(GT_INFO); | |
| 401 case TokenType.GT_EQ: | |
| 402 return symbol(GT_EQ_INFO); | |
| 403 case TokenType.GT_GT: | |
| 404 return symbol(GT_GT_INFO); | |
| 405 case TokenType.GT_GT_EQ: | |
| 406 return symbol(GT_GT_EQ_INFO); | |
| 407 case TokenType.HASH: | |
| 408 return symbol(HASH_INFO); | |
| 409 case TokenType.INDEX: | |
| 410 return symbol(INDEX_INFO); | |
| 411 case TokenType.INDEX_EQ: | |
| 412 return symbol(INDEX_EQ_INFO); | |
| 413 case TokenType.LT: | |
| 414 return beginGroup(LT_INFO); | |
| 415 case TokenType.LT_EQ: | |
| 416 return symbol(LT_EQ_INFO); | |
| 417 case TokenType.LT_LT: | |
| 418 return symbol(LT_LT_INFO); | |
| 419 case TokenType.LT_LT_EQ: | |
| 420 return symbol(LT_LT_EQ_INFO); | |
| 421 case TokenType.MINUS: | |
| 422 return symbol(MINUS_INFO); | |
| 423 case TokenType.MINUS_EQ: | |
| 424 return symbol(MINUS_EQ_INFO); | |
| 425 case TokenType.MINUS_MINUS: | |
| 426 return symbol(MINUS_MINUS_INFO); | |
| 427 case TokenType.OPEN_CURLY_BRACKET: | |
| 428 return beginGroup(OPEN_CURLY_BRACKET_INFO); | |
| 429 case TokenType.OPEN_PAREN: | |
| 430 return beginGroup(OPEN_PAREN_INFO); | |
| 431 case TokenType.OPEN_SQUARE_BRACKET: | |
| 432 return beginGroup(OPEN_SQUARE_BRACKET_INFO); | |
| 433 case TokenType.PERCENT: | |
| 434 return symbol(PERCENT_INFO); | |
| 435 case TokenType.PERCENT_EQ: | |
| 436 return symbol(PERCENT_EQ_INFO); | |
| 437 case TokenType.PERIOD: | |
| 438 return symbol(PERIOD_INFO); | |
| 439 case TokenType.PERIOD_PERIOD: | |
| 440 return symbol(PERIOD_PERIOD_INFO); | |
| 441 case TokenType.PLUS: | |
| 442 return symbol(PLUS_INFO); | |
| 443 case TokenType.PLUS_EQ: | |
| 444 return symbol(PLUS_EQ_INFO); | |
| 445 case TokenType.PLUS_PLUS: | |
| 446 return symbol(PLUS_PLUS_INFO); | |
| 447 case TokenType.QUESTION: | |
| 448 return symbol(QUESTION_INFO); | |
| 449 case TokenType.QUESTION_PERIOD: | |
| 450 return symbol(QUESTION_PERIOD_INFO); | |
| 451 case TokenType.QUESTION_QUESTION: | |
| 452 return symbol(QUESTION_QUESTION_INFO); | |
| 453 case TokenType.QUESTION_QUESTION_EQ: | |
| 454 return symbol(QUESTION_QUESTION_EQ_INFO); | |
| 455 case TokenType.SEMICOLON: | |
| 456 return symbol(SEMICOLON_INFO); | |
| 457 case TokenType.SLASH: | |
| 458 return symbol(SLASH_INFO); | |
| 459 case TokenType.SLASH_EQ: | |
| 460 return symbol(SLASH_EQ_INFO); | |
| 461 case TokenType.STAR: | |
| 462 return symbol(STAR_INFO); | |
| 463 case TokenType.STAR_EQ: | |
| 464 return symbol(STAR_EQ_INFO); | |
| 465 case TokenType.STRING_INTERPOLATION_EXPRESSION: | |
| 466 return beginGroup(STRING_INTERPOLATION_INFO); | |
| 467 case TokenType.STRING_INTERPOLATION_IDENTIFIER: | |
| 468 return symbol(STRING_INTERPOLATION_IDENTIFIER_INFO); | |
| 469 case TokenType.TILDE: | |
| 470 return symbol(TILDE_INFO); | |
| 471 case TokenType.TILDE_SLASH: | |
| 472 return symbol(TILDE_SLASH_INFO); | |
| 473 case TokenType.TILDE_SLASH_EQ: | |
| 474 return symbol(TILDE_SLASH_EQ_INFO); | |
| 475 case TokenType.BACKPING: | |
| 476 return symbol(BACKPING_INFO); | |
| 477 case TokenType.BACKSLASH: | |
| 478 return symbol(BACKSLASH_INFO); | |
| 479 case TokenType.PERIOD_PERIOD_PERIOD: | |
| 480 return symbol(PERIOD_PERIOD_PERIOD_INFO); | |
| 481 // case TokenType.GENERIC_METHOD_TYPE_ASSIGN | |
| 482 // case TokenType.GENERIC_METHOD_TYPE_LIST | |
| 483 default: | |
| 484 return internalError('Unhandled token type ${token.type}'); | |
| 485 } | |
| 486 } | |
| 487 | |
| 488 /// Determines whether the given [charOffset], which came from the non-EOF token | |
| 489 /// [token], represents the end of the input. | |
| 490 bool _isAtEnd(Token token, int charOffset) { | |
| 491 while (true) { | |
| 492 // Skip to the next token. | |
| 493 token = token.next; | |
| 494 // If we've found an EOF token, its charOffset indicates where the end of | |
| 495 // the input is. | |
| 496 if (token.isEof) return token.charOffset == charOffset; | |
| 497 // If we've found a non-error token, then we know there is additional input | |
| 498 // text after [charOffset]. | |
| 499 if (token.info.kind != BAD_INPUT_TOKEN) return false; | |
| 500 // Otherwise keep looking. | |
| 501 } | |
| 502 } | |
| 503 | |
| 504 analyzer.Token toAnalyzerToken(Token token, | |
| 505 [analyzer.CommentToken commentToken]) { | |
| 506 if (token == null) return null; | |
| 507 analyzer.Token makeStringToken(TokenType tokenType) { | |
| 508 if (commentToken == null) { | |
| 509 return new analyzer.StringToken(tokenType, token.lexeme, token.charOffset)
; | |
| 510 } else { | |
| 511 return new analyzer.StringTokenWithComment( | |
| 512 tokenType, token.lexeme, token.charOffset, commentToken); | |
| 513 } | |
| 514 } | |
| 515 | |
| 516 analyzer.Token makeBeginToken(TokenType tokenType) { | |
| 517 if (commentToken == null) { | |
| 518 return new analyzer.BeginToken(tokenType, token.charOffset); | |
| 519 } else { | |
| 520 return new analyzer.BeginTokenWithComment( | |
| 521 tokenType, token.charOffset, commentToken); | |
| 522 } | |
| 523 } | |
| 524 | |
| 525 switch (token.kind) { | |
| 526 case DOUBLE_TOKEN: | |
| 527 return makeStringToken(TokenType.DOUBLE); | |
| 528 | |
| 529 case HEXADECIMAL_TOKEN: | |
| 530 return makeStringToken(TokenType.HEXADECIMAL); | |
| 531 | |
| 532 case IDENTIFIER_TOKEN: | |
| 533 return makeStringToken(TokenType.IDENTIFIER); | |
| 534 | |
| 535 case INT_TOKEN: | |
| 536 return makeStringToken(TokenType.INT); | |
| 537 | |
| 538 case KEYWORD_TOKEN: | |
| 539 KeywordToken keywordToken = token; | |
| 540 var syntax = keywordToken.keyword.syntax; | |
| 541 if (keywordToken.keyword.isPseudo) { | |
| 542 // TODO(paulberry,ahe): Fasta considers "deferred" be a "pseudo-keyword" | |
| 543 // (ordinary identifier which has special meaning under circumstances), | |
| 544 // but analyzer and the spec consider it to be a built-in identifier | |
| 545 // (identifier which can't be used in type names). | |
| 546 if (!identical(syntax, 'deferred')) { | |
| 547 return makeStringToken(TokenType.IDENTIFIER); | |
| 548 } | |
| 549 } | |
| 550 // TODO(paulberry): if the map lookup proves to be too slow, consider | |
| 551 // using a switch statement, or perhaps a string of | |
| 552 // "if (identical(syntax, "foo"))" checks. (Note that identical checks | |
| 553 // should be safe because the Fasta scanner uses string literals for | |
| 554 // the values of keyword.syntax.) | |
| 555 var keyword = | |
| 556 _keywordMap[syntax] ?? internalError('Unknown keyword: $syntax'); | |
| 557 if (commentToken == null) { | |
| 558 return new analyzer.KeywordToken(keyword, token.charOffset); | |
| 559 } else { | |
| 560 return new analyzer.KeywordTokenWithComment( | |
| 561 keyword, token.charOffset, commentToken); | |
| 562 } | |
| 563 break; | |
| 564 | |
| 565 case SCRIPT_TOKEN: | |
| 566 return makeStringToken(TokenType.SCRIPT_TAG); | |
| 567 | |
| 568 case STRING_TOKEN: | |
| 569 return makeStringToken(TokenType.STRING); | |
| 570 | |
| 571 case OPEN_CURLY_BRACKET_TOKEN: | |
| 572 case OPEN_SQUARE_BRACKET_TOKEN: | |
| 573 case OPEN_PAREN_TOKEN: | |
| 574 case STRING_INTERPOLATION_TOKEN: | |
| 575 return makeBeginToken(getTokenType(token)); | |
| 576 | |
| 577 default: | |
| 578 if (commentToken == null) { | |
| 579 return new analyzer.Token(getTokenType(token), token.charOffset); | |
| 580 } else { | |
| 581 return new analyzer.TokenWithComment( | |
| 582 getTokenType(token), token.charOffset, commentToken); | |
| 583 } | |
| 584 break; | |
| 585 } | |
| 586 } | |
| 587 | |
| 588 final _keywordMap = { | |
| 589 "assert": analyzer.Keyword.ASSERT, | |
| 590 "break": analyzer.Keyword.BREAK, | |
| 591 "case": analyzer.Keyword.CASE, | |
| 592 "catch": analyzer.Keyword.CATCH, | |
| 593 "class": analyzer.Keyword.CLASS, | |
| 594 "const": analyzer.Keyword.CONST, | |
| 595 "continue": analyzer.Keyword.CONTINUE, | |
| 596 "default": analyzer.Keyword.DEFAULT, | |
| 597 "do": analyzer.Keyword.DO, | |
| 598 "else": analyzer.Keyword.ELSE, | |
| 599 "enum": analyzer.Keyword.ENUM, | |
| 600 "extends": analyzer.Keyword.EXTENDS, | |
| 601 "false": analyzer.Keyword.FALSE, | |
| 602 "final": analyzer.Keyword.FINAL, | |
| 603 "finally": analyzer.Keyword.FINALLY, | |
| 604 "for": analyzer.Keyword.FOR, | |
| 605 "if": analyzer.Keyword.IF, | |
| 606 "in": analyzer.Keyword.IN, | |
| 607 "new": analyzer.Keyword.NEW, | |
| 608 "null": analyzer.Keyword.NULL, | |
| 609 "rethrow": analyzer.Keyword.RETHROW, | |
| 610 "return": analyzer.Keyword.RETURN, | |
| 611 "super": analyzer.Keyword.SUPER, | |
| 612 "switch": analyzer.Keyword.SWITCH, | |
| 613 "this": analyzer.Keyword.THIS, | |
| 614 "throw": analyzer.Keyword.THROW, | |
| 615 "true": analyzer.Keyword.TRUE, | |
| 616 "try": analyzer.Keyword.TRY, | |
| 617 "var": analyzer.Keyword.VAR, | |
| 618 "void": analyzer.Keyword.VOID, | |
| 619 "while": analyzer.Keyword.WHILE, | |
| 620 "with": analyzer.Keyword.WITH, | |
| 621 "is": analyzer.Keyword.IS, | |
| 622 "abstract": analyzer.Keyword.ABSTRACT, | |
| 623 "as": analyzer.Keyword.AS, | |
| 624 "covariant": analyzer.Keyword.COVARIANT, | |
| 625 "dynamic": analyzer.Keyword.DYNAMIC, | |
| 626 "export": analyzer.Keyword.EXPORT, | |
| 627 "external": analyzer.Keyword.EXTERNAL, | |
| 628 "factory": analyzer.Keyword.FACTORY, | |
| 629 "get": analyzer.Keyword.GET, | |
| 630 "implements": analyzer.Keyword.IMPLEMENTS, | |
| 631 "import": analyzer.Keyword.IMPORT, | |
| 632 "library": analyzer.Keyword.LIBRARY, | |
| 633 "operator": analyzer.Keyword.OPERATOR, | |
| 634 "part": analyzer.Keyword.PART, | |
| 635 "set": analyzer.Keyword.SET, | |
| 636 "static": analyzer.Keyword.STATIC, | |
| 637 "typedef": analyzer.Keyword.TYPEDEF, | |
| 638 "deferred": analyzer.Keyword.DEFERRED, | |
| 639 }; | |
| 640 | |
| 641 TokenType getTokenType(Token token) { | |
| 642 switch (token.kind) { | |
| 643 case EOF_TOKEN: | |
| 644 return TokenType.EOF; | |
| 645 case DOUBLE_TOKEN: | |
| 646 return TokenType.DOUBLE; | |
| 647 case HEXADECIMAL_TOKEN: | |
| 648 return TokenType.HEXADECIMAL; | |
| 649 case IDENTIFIER_TOKEN: | |
| 650 return TokenType.IDENTIFIER; | |
| 651 case INT_TOKEN: | |
| 652 return TokenType.INT; | |
| 653 case KEYWORD_TOKEN: | |
| 654 return TokenType.KEYWORD; | |
| 655 // case MULTI_LINE_COMMENT_TOKEN: return TokenType.MULTI_LINE_COMMENT; | |
| 656 // case SCRIPT_TAG_TOKEN: return TokenType.SCRIPT_TAG; | |
| 657 // case SINGLE_LINE_COMMENT_TOKEN: return TokenType.SINGLE_LINE_COMMENT; | |
| 658 case STRING_TOKEN: | |
| 659 return TokenType.STRING; | |
| 660 case AMPERSAND_TOKEN: | |
| 661 return TokenType.AMPERSAND; | |
| 662 case AMPERSAND_AMPERSAND_TOKEN: | |
| 663 return TokenType.AMPERSAND_AMPERSAND; | |
| 664 // case AMPERSAND_AMPERSAND_EQ_TOKEN: | |
| 665 // return TokenType.AMPERSAND_AMPERSAND_EQ; | |
| 666 case AMPERSAND_EQ_TOKEN: | |
| 667 return TokenType.AMPERSAND_EQ; | |
| 668 case AT_TOKEN: | |
| 669 return TokenType.AT; | |
| 670 case BANG_TOKEN: | |
| 671 return TokenType.BANG; | |
| 672 case BANG_EQ_TOKEN: | |
| 673 return TokenType.BANG_EQ; | |
| 674 case BAR_TOKEN: | |
| 675 return TokenType.BAR; | |
| 676 case BAR_BAR_TOKEN: | |
| 677 return TokenType.BAR_BAR; | |
| 678 // case BAR_BAR_EQ_TOKEN: return TokenType.BAR_BAR_EQ; | |
| 679 case BAR_EQ_TOKEN: | |
| 680 return TokenType.BAR_EQ; | |
| 681 case COLON_TOKEN: | |
| 682 return TokenType.COLON; | |
| 683 case COMMA_TOKEN: | |
| 684 return TokenType.COMMA; | |
| 685 case CARET_TOKEN: | |
| 686 return TokenType.CARET; | |
| 687 case CARET_EQ_TOKEN: | |
| 688 return TokenType.CARET_EQ; | |
| 689 case CLOSE_CURLY_BRACKET_TOKEN: | |
| 690 return TokenType.CLOSE_CURLY_BRACKET; | |
| 691 case CLOSE_PAREN_TOKEN: | |
| 692 return TokenType.CLOSE_PAREN; | |
| 693 case CLOSE_SQUARE_BRACKET_TOKEN: | |
| 694 return TokenType.CLOSE_SQUARE_BRACKET; | |
| 695 case EQ_TOKEN: | |
| 696 return TokenType.EQ; | |
| 697 case EQ_EQ_TOKEN: | |
| 698 return TokenType.EQ_EQ; | |
| 699 case FUNCTION_TOKEN: | |
| 700 return TokenType.FUNCTION; | |
| 701 case GT_TOKEN: | |
| 702 return TokenType.GT; | |
| 703 case GT_EQ_TOKEN: | |
| 704 return TokenType.GT_EQ; | |
| 705 case GT_GT_TOKEN: | |
| 706 return TokenType.GT_GT; | |
| 707 case GT_GT_EQ_TOKEN: | |
| 708 return TokenType.GT_GT_EQ; | |
| 709 case HASH_TOKEN: | |
| 710 return TokenType.HASH; | |
| 711 case INDEX_TOKEN: | |
| 712 return TokenType.INDEX; | |
| 713 case INDEX_EQ_TOKEN: | |
| 714 return TokenType.INDEX_EQ; | |
| 715 // case IS_TOKEN: return TokenType.IS; | |
| 716 case LT_TOKEN: | |
| 717 return TokenType.LT; | |
| 718 case LT_EQ_TOKEN: | |
| 719 return TokenType.LT_EQ; | |
| 720 case LT_LT_TOKEN: | |
| 721 return TokenType.LT_LT; | |
| 722 case LT_LT_EQ_TOKEN: | |
| 723 return TokenType.LT_LT_EQ; | |
| 724 case MINUS_TOKEN: | |
| 725 return TokenType.MINUS; | |
| 726 case MINUS_EQ_TOKEN: | |
| 727 return TokenType.MINUS_EQ; | |
| 728 case MINUS_MINUS_TOKEN: | |
| 729 return TokenType.MINUS_MINUS; | |
| 730 case OPEN_CURLY_BRACKET_TOKEN: | |
| 731 return TokenType.OPEN_CURLY_BRACKET; | |
| 732 case OPEN_PAREN_TOKEN: | |
| 733 return TokenType.OPEN_PAREN; | |
| 734 case OPEN_SQUARE_BRACKET_TOKEN: | |
| 735 return TokenType.OPEN_SQUARE_BRACKET; | |
| 736 case PERCENT_TOKEN: | |
| 737 return TokenType.PERCENT; | |
| 738 case PERCENT_EQ_TOKEN: | |
| 739 return TokenType.PERCENT_EQ; | |
| 740 case PERIOD_TOKEN: | |
| 741 return TokenType.PERIOD; | |
| 742 case PERIOD_PERIOD_TOKEN: | |
| 743 return TokenType.PERIOD_PERIOD; | |
| 744 case PLUS_TOKEN: | |
| 745 return TokenType.PLUS; | |
| 746 case PLUS_EQ_TOKEN: | |
| 747 return TokenType.PLUS_EQ; | |
| 748 case PLUS_PLUS_TOKEN: | |
| 749 return TokenType.PLUS_PLUS; | |
| 750 case QUESTION_TOKEN: | |
| 751 return TokenType.QUESTION; | |
| 752 case QUESTION_PERIOD_TOKEN: | |
| 753 return TokenType.QUESTION_PERIOD; | |
| 754 case QUESTION_QUESTION_TOKEN: | |
| 755 return TokenType.QUESTION_QUESTION; | |
| 756 case QUESTION_QUESTION_EQ_TOKEN: | |
| 757 return TokenType.QUESTION_QUESTION_EQ; | |
| 758 case SEMICOLON_TOKEN: | |
| 759 return TokenType.SEMICOLON; | |
| 760 case SLASH_TOKEN: | |
| 761 return TokenType.SLASH; | |
| 762 case SLASH_EQ_TOKEN: | |
| 763 return TokenType.SLASH_EQ; | |
| 764 case STAR_TOKEN: | |
| 765 return TokenType.STAR; | |
| 766 case STAR_EQ_TOKEN: | |
| 767 return TokenType.STAR_EQ; | |
| 768 case STRING_INTERPOLATION_TOKEN: | |
| 769 return TokenType.STRING_INTERPOLATION_EXPRESSION; | |
| 770 case STRING_INTERPOLATION_IDENTIFIER_TOKEN: | |
| 771 return TokenType.STRING_INTERPOLATION_IDENTIFIER; | |
| 772 case TILDE_TOKEN: | |
| 773 return TokenType.TILDE; | |
| 774 case TILDE_SLASH_TOKEN: | |
| 775 return TokenType.TILDE_SLASH; | |
| 776 case TILDE_SLASH_EQ_TOKEN: | |
| 777 return TokenType.TILDE_SLASH_EQ; | |
| 778 case BACKPING_TOKEN: | |
| 779 return TokenType.BACKPING; | |
| 780 case BACKSLASH_TOKEN: | |
| 781 return TokenType.BACKSLASH; | |
| 782 case PERIOD_PERIOD_PERIOD_TOKEN: | |
| 783 return TokenType.PERIOD_PERIOD_PERIOD; | |
| 784 // case GENERIC_METHOD_TYPE_LIST_TOKEN: | |
| 785 // return TokenType.GENERIC_METHOD_TYPE_LIST; | |
| 786 // case GENERIC_METHOD_TYPE_ASSIGN_TOKEN: | |
| 787 // return TokenType.GENERIC_METHOD_TYPE_ASSIGN; | |
| 788 default: | |
| 789 return internalError("Unhandled token ${token.info}"); | |
| 790 } | |
| 791 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2756593004:
Start unraveling circularities between analyzer and front_end/kernel. (Closed)
