| OLD | NEW |
| 1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library fasta.analyzer.token_utils; | 5 library fasta.analyzer.token_utils; |
| 6 | 6 |
| 7 import 'package:front_end/src/fasta/scanner/error_token.dart' show ErrorToken; | 7 import 'package:front_end/src/fasta/scanner/error_token.dart' show ErrorToken; |
| 8 | 8 |
| 9 import 'package:front_end/src/scanner/token.dart' show Keyword; | 9 import 'package:front_end/src/scanner/token.dart' show Keyword; |
| 10 | 10 |
| 11 import 'package:front_end/src/fasta/scanner/precedence.dart'; | |
| 12 | |
| 13 import 'package:front_end/src/fasta/scanner/token.dart' | 11 import 'package:front_end/src/fasta/scanner/token.dart' |
| 14 show | 12 show |
| 15 BeginGroupToken, | 13 BeginGroupToken, |
| 16 CommentToken, | 14 CommentToken, |
| 17 DartDocToken, | 15 DartDocToken, |
| 18 KeywordToken, | 16 KeywordToken, |
| 19 StringToken, | 17 StringToken, |
| 20 SymbolToken, | 18 SymbolToken, |
| 21 Token; | 19 Token; |
| 22 | 20 |
| (...skipping 138 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 161 token.endGroup.charOffset != token.charOffset) { | 159 token.endGroup.charOffset != token.charOffset) { |
| 162 _beginTokenStack.add(translatedToken); | 160 _beginTokenStack.add(translatedToken); |
| 163 _endTokenStack.add(token.endGroup); | 161 _endTokenStack.add(token.endGroup); |
| 164 } | 162 } |
| 165 } | 163 } |
| 166 } | 164 } |
| 167 | 165 |
| 168 /// Converts a single Fasta comment token to an analyzer comment token. | 166 /// Converts a single Fasta comment token to an analyzer comment token. |
| 169 analyzer.CommentToken toAnalyzerCommentToken(Token token) { | 167 analyzer.CommentToken toAnalyzerCommentToken(Token token) { |
| 170 TokenType type; | 168 TokenType type; |
| 171 if (token.type == GENERIC_METHOD_TYPE_ASSIGN) { | 169 if (token.type == TokenType.GENERIC_METHOD_TYPE_ASSIGN) { |
| 172 type = TokenType.GENERIC_METHOD_TYPE_ASSIGN; | 170 type = TokenType.GENERIC_METHOD_TYPE_ASSIGN; |
| 173 } else if (token.type == GENERIC_METHOD_TYPE_LIST) { | 171 } else if (token.type == TokenType.GENERIC_METHOD_TYPE_LIST) { |
| 174 type = TokenType.GENERIC_METHOD_TYPE_LIST; | 172 type = TokenType.GENERIC_METHOD_TYPE_LIST; |
| 175 } else { | 173 } else { |
| 176 // TODO(paulberry,ahe): It would be nice if the scanner gave us an | 174 // TODO(paulberry,ahe): It would be nice if the scanner gave us an |
| 177 // easier way to distinguish between the two types of comment. | 175 // easier way to distinguish between the two types of comment. |
| 178 type = token.lexeme.startsWith('/*') | 176 type = token.lexeme.startsWith('/*') |
| 179 ? TokenType.MULTI_LINE_COMMENT | 177 ? TokenType.MULTI_LINE_COMMENT |
| 180 : TokenType.SINGLE_LINE_COMMENT; | 178 : TokenType.SINGLE_LINE_COMMENT; |
| 181 } | 179 } |
| 182 return new analyzer.CommentToken(type, token.lexeme, token.charOffset); | 180 return new analyzer.CommentToken(type, token.lexeme, token.charOffset); |
| 183 } | 181 } |
| (...skipping 92 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 276 } | 274 } |
| 277 | 275 |
| 278 /// Converts a single analyzer token into a Fasta token. | 276 /// Converts a single analyzer token into a Fasta token. |
| 279 Token fromAnalyzerToken(analyzer.Token token) { | 277 Token fromAnalyzerToken(analyzer.Token token) { |
| 280 Token beginGroup(TokenType info) => new BeginGroupToken(info, token.offset); | 278 Token beginGroup(TokenType info) => new BeginGroupToken(info, token.offset); |
| 281 Token string(TokenType info) => | 279 Token string(TokenType info) => |
| 282 new StringToken.fromString(info, token.lexeme, token.offset); | 280 new StringToken.fromString(info, token.lexeme, token.offset); |
| 283 Token symbol(TokenType info) => new SymbolToken(info, token.offset); | 281 Token symbol(TokenType info) => new SymbolToken(info, token.offset); |
| 284 switch (token.type) { | 282 switch (token.type) { |
| 285 case TokenType.DOUBLE: | 283 case TokenType.DOUBLE: |
| 286 return string(DOUBLE_INFO); | 284 return string(TokenType.DOUBLE); |
| 287 case TokenType.HEXADECIMAL: | 285 case TokenType.HEXADECIMAL: |
| 288 return string(HEXADECIMAL_INFO); | 286 return string(TokenType.HEXADECIMAL); |
| 289 case TokenType.IDENTIFIER: | 287 case TokenType.IDENTIFIER: |
| 290 // Certain identifiers have special grammatical meanings even though they | 288 // Certain identifiers have special grammatical meanings even though they |
| 291 // are neither keywords nor built-in identifiers (e.g. "async"). Analyzer | 289 // are neither keywords nor built-in identifiers (e.g. "async"). Analyzer |
| 292 // represents these as identifiers. Fasta represents them as keywords | 290 // represents these as identifiers. Fasta represents them as keywords |
| 293 // with the "isPseudo" property. | 291 // with the "isPseudo" property. |
| 294 var keyword = Keyword.keywords[token.lexeme]; | 292 var keyword = Keyword.keywords[token.lexeme]; |
| 295 if (keyword != null) { | 293 if (keyword != null) { |
| 296 assert(keyword.isPseudo); | 294 assert(keyword.isPseudo); |
| 297 return new KeywordToken(keyword, token.offset); | 295 return new KeywordToken(keyword, token.offset); |
| 298 } else { | 296 } else { |
| 299 return string(IDENTIFIER_INFO); | 297 return string(TokenType.IDENTIFIER); |
| 300 } | 298 } |
| 301 break; | 299 break; |
| 302 case TokenType.INT: | 300 case TokenType.INT: |
| 303 return string(INT_INFO); | 301 return string(TokenType.INT); |
| 304 case TokenType.KEYWORD: | 302 case TokenType.KEYWORD: |
| 305 var keyword = Keyword.keywords[token.lexeme]; | 303 var keyword = Keyword.keywords[token.lexeme]; |
| 306 if (keyword != null) { | 304 if (keyword != null) { |
| 307 return new KeywordToken(keyword, token.offset); | 305 return new KeywordToken(keyword, token.offset); |
| 308 } else { | 306 } else { |
| 309 return internalError("Unrecognized keyword: '${token.lexeme}'."); | 307 return internalError("Unrecognized keyword: '${token.lexeme}'."); |
| 310 } | 308 } |
| 311 break; | 309 break; |
| 312 case TokenType.MULTI_LINE_COMMENT: | 310 case TokenType.MULTI_LINE_COMMENT: |
| 313 if (token.lexeme.startsWith('/**')) { | 311 if (token.lexeme.startsWith('/**')) { |
| 314 return new DartDocToken.fromSubstring( | 312 return new DartDocToken.fromSubstring(TokenType.MULTI_LINE_COMMENT, |
| 315 MULTI_LINE_COMMENT_INFO, token.lexeme, 0, token.lexeme.length, 0); | 313 token.lexeme, 0, token.lexeme.length, 0); |
| 316 } | 314 } |
| 317 return new CommentToken.fromSubstring( | 315 return new CommentToken.fromSubstring(TokenType.MULTI_LINE_COMMENT, |
| 318 MULTI_LINE_COMMENT_INFO, token.lexeme, 0, token.lexeme.length, 0); | 316 token.lexeme, 0, token.lexeme.length, 0); |
| 319 case TokenType.SCRIPT_TAG: | 317 case TokenType.SCRIPT_TAG: |
| 320 return string(SCRIPT_INFO); | 318 return string(TokenType.SCRIPT_TAG); |
| 321 case TokenType.SINGLE_LINE_COMMENT: | 319 case TokenType.SINGLE_LINE_COMMENT: |
| 322 if (token.lexeme.startsWith('///')) { | 320 if (token.lexeme.startsWith('///')) { |
| 323 return new DartDocToken.fromSubstring( | 321 return new DartDocToken.fromSubstring(TokenType.SINGLE_LINE_COMMENT, |
| 324 SINGLE_LINE_COMMENT_INFO, token.lexeme, 0, token.lexeme.length, 0); | 322 token.lexeme, 0, token.lexeme.length, 0); |
| 325 } | 323 } |
| 326 return new CommentToken.fromSubstring( | 324 return new CommentToken.fromSubstring(TokenType.SINGLE_LINE_COMMENT, |
| 327 SINGLE_LINE_COMMENT_INFO, token.lexeme, 0, token.lexeme.length, 0); | 325 token.lexeme, 0, token.lexeme.length, 0); |
| 328 case TokenType.STRING: | 326 case TokenType.STRING: |
| 329 return string(STRING_INFO); | 327 return string(TokenType.STRING); |
| 330 case TokenType.AMPERSAND: | 328 case TokenType.AMPERSAND: |
| 331 return symbol(AMPERSAND_INFO); | 329 return symbol(TokenType.AMPERSAND); |
| 332 case TokenType.AMPERSAND_AMPERSAND: | 330 case TokenType.AMPERSAND_AMPERSAND: |
| 333 return symbol(AMPERSAND_AMPERSAND_INFO); | 331 return symbol(TokenType.AMPERSAND_AMPERSAND); |
| 334 // case TokenType.AMPERSAND_AMPERSAND_EQ | 332 // case TokenType.AMPERSAND_AMPERSAND_EQ |
| 335 case TokenType.AMPERSAND_EQ: | 333 case TokenType.AMPERSAND_EQ: |
| 336 return symbol(AMPERSAND_EQ_INFO); | 334 return symbol(TokenType.AMPERSAND_EQ); |
| 337 case TokenType.AT: | 335 case TokenType.AT: |
| 338 return symbol(AT_INFO); | 336 return symbol(TokenType.AT); |
| 339 case TokenType.BANG: | 337 case TokenType.BANG: |
| 340 return symbol(BANG_INFO); | 338 return symbol(TokenType.BANG); |
| 341 case TokenType.BANG_EQ: | 339 case TokenType.BANG_EQ: |
| 342 return symbol(BANG_EQ_INFO); | 340 return symbol(TokenType.BANG_EQ); |
| 343 case TokenType.BAR: | 341 case TokenType.BAR: |
| 344 return symbol(BAR_INFO); | 342 return symbol(TokenType.BAR); |
| 345 case TokenType.BAR_BAR: | 343 case TokenType.BAR_BAR: |
| 346 return symbol(BAR_BAR_INFO); | 344 return symbol(TokenType.BAR_BAR); |
| 347 // case TokenType.BAR_BAR_EQ | 345 // case TokenType.BAR_BAR_EQ |
| 348 case TokenType.BAR_EQ: | 346 case TokenType.BAR_EQ: |
| 349 return symbol(BAR_EQ_INFO); | 347 return symbol(TokenType.BAR_EQ); |
| 350 case TokenType.COLON: | 348 case TokenType.COLON: |
| 351 return symbol(COLON_INFO); | 349 return symbol(TokenType.COLON); |
| 352 case TokenType.COMMA: | 350 case TokenType.COMMA: |
| 353 return symbol(COMMA_INFO); | 351 return symbol(TokenType.COMMA); |
| 354 case TokenType.CARET: | 352 case TokenType.CARET: |
| 355 return symbol(CARET_INFO); | 353 return symbol(TokenType.CARET); |
| 356 case TokenType.CARET_EQ: | 354 case TokenType.CARET_EQ: |
| 357 return symbol(CARET_EQ_INFO); | 355 return symbol(TokenType.CARET_EQ); |
| 358 case TokenType.CLOSE_CURLY_BRACKET: | 356 case TokenType.CLOSE_CURLY_BRACKET: |
| 359 return symbol(CLOSE_CURLY_BRACKET_INFO); | 357 return symbol(TokenType.CLOSE_CURLY_BRACKET); |
| 360 case TokenType.CLOSE_PAREN: | 358 case TokenType.CLOSE_PAREN: |
| 361 return symbol(CLOSE_PAREN_INFO); | 359 return symbol(TokenType.CLOSE_PAREN); |
| 362 case TokenType.CLOSE_SQUARE_BRACKET: | 360 case TokenType.CLOSE_SQUARE_BRACKET: |
| 363 return symbol(CLOSE_SQUARE_BRACKET_INFO); | 361 return symbol(TokenType.CLOSE_SQUARE_BRACKET); |
| 364 case TokenType.EQ: | 362 case TokenType.EQ: |
| 365 return symbol(EQ_INFO); | 363 return symbol(TokenType.EQ); |
| 366 case TokenType.EQ_EQ: | 364 case TokenType.EQ_EQ: |
| 367 return symbol(EQ_EQ_INFO); | 365 return symbol(TokenType.EQ_EQ); |
| 368 case TokenType.FUNCTION: | 366 case TokenType.FUNCTION: |
| 369 return symbol(FUNCTION_INFO); | 367 return symbol(TokenType.FUNCTION); |
| 370 case TokenType.GT: | 368 case TokenType.GT: |
| 371 return symbol(GT_INFO); | 369 return symbol(TokenType.GT); |
| 372 case TokenType.GT_EQ: | 370 case TokenType.GT_EQ: |
| 373 return symbol(GT_EQ_INFO); | 371 return symbol(TokenType.GT_EQ); |
| 374 case TokenType.GT_GT: | 372 case TokenType.GT_GT: |
| 375 return symbol(GT_GT_INFO); | 373 return symbol(TokenType.GT_GT); |
| 376 case TokenType.GT_GT_EQ: | 374 case TokenType.GT_GT_EQ: |
| 377 return symbol(GT_GT_EQ_INFO); | 375 return symbol(TokenType.GT_GT_EQ); |
| 378 case TokenType.HASH: | 376 case TokenType.HASH: |
| 379 return symbol(HASH_INFO); | 377 return symbol(TokenType.HASH); |
| 380 case TokenType.INDEX: | 378 case TokenType.INDEX: |
| 381 return symbol(INDEX_INFO); | 379 return symbol(TokenType.INDEX); |
| 382 case TokenType.INDEX_EQ: | 380 case TokenType.INDEX_EQ: |
| 383 return symbol(INDEX_EQ_INFO); | 381 return symbol(TokenType.INDEX_EQ); |
| 384 case TokenType.LT: | 382 case TokenType.LT: |
| 385 return beginGroup(LT_INFO); | 383 return beginGroup(TokenType.LT); |
| 386 case TokenType.LT_EQ: | 384 case TokenType.LT_EQ: |
| 387 return symbol(LT_EQ_INFO); | 385 return symbol(TokenType.LT_EQ); |
| 388 case TokenType.LT_LT: | 386 case TokenType.LT_LT: |
| 389 return symbol(LT_LT_INFO); | 387 return symbol(TokenType.LT_LT); |
| 390 case TokenType.LT_LT_EQ: | 388 case TokenType.LT_LT_EQ: |
| 391 return symbol(LT_LT_EQ_INFO); | 389 return symbol(TokenType.LT_LT_EQ); |
| 392 case TokenType.MINUS: | 390 case TokenType.MINUS: |
| 393 return symbol(MINUS_INFO); | 391 return symbol(TokenType.MINUS); |
| 394 case TokenType.MINUS_EQ: | 392 case TokenType.MINUS_EQ: |
| 395 return symbol(MINUS_EQ_INFO); | 393 return symbol(TokenType.MINUS_EQ); |
| 396 case TokenType.MINUS_MINUS: | 394 case TokenType.MINUS_MINUS: |
| 397 return symbol(MINUS_MINUS_INFO); | 395 return symbol(TokenType.MINUS_MINUS); |
| 398 case TokenType.OPEN_CURLY_BRACKET: | 396 case TokenType.OPEN_CURLY_BRACKET: |
| 399 return beginGroup(OPEN_CURLY_BRACKET_INFO); | 397 return beginGroup(TokenType.OPEN_CURLY_BRACKET); |
| 400 case TokenType.OPEN_PAREN: | 398 case TokenType.OPEN_PAREN: |
| 401 return beginGroup(OPEN_PAREN_INFO); | 399 return beginGroup(TokenType.OPEN_PAREN); |
| 402 case TokenType.OPEN_SQUARE_BRACKET: | 400 case TokenType.OPEN_SQUARE_BRACKET: |
| 403 return beginGroup(OPEN_SQUARE_BRACKET_INFO); | 401 return beginGroup(TokenType.OPEN_SQUARE_BRACKET); |
| 404 case TokenType.PERCENT: | 402 case TokenType.PERCENT: |
| 405 return symbol(PERCENT_INFO); | 403 return symbol(TokenType.PERCENT); |
| 406 case TokenType.PERCENT_EQ: | 404 case TokenType.PERCENT_EQ: |
| 407 return symbol(PERCENT_EQ_INFO); | 405 return symbol(TokenType.PERCENT_EQ); |
| 408 case TokenType.PERIOD: | 406 case TokenType.PERIOD: |
| 409 return symbol(PERIOD_INFO); | 407 return symbol(TokenType.PERIOD); |
| 410 case TokenType.PERIOD_PERIOD: | 408 case TokenType.PERIOD_PERIOD: |
| 411 return symbol(PERIOD_PERIOD_INFO); | 409 return symbol(TokenType.PERIOD_PERIOD); |
| 412 case TokenType.PLUS: | 410 case TokenType.PLUS: |
| 413 return symbol(PLUS_INFO); | 411 return symbol(TokenType.PLUS); |
| 414 case TokenType.PLUS_EQ: | 412 case TokenType.PLUS_EQ: |
| 415 return symbol(PLUS_EQ_INFO); | 413 return symbol(TokenType.PLUS_EQ); |
| 416 case TokenType.PLUS_PLUS: | 414 case TokenType.PLUS_PLUS: |
| 417 return symbol(PLUS_PLUS_INFO); | 415 return symbol(TokenType.PLUS_PLUS); |
| 418 case TokenType.QUESTION: | 416 case TokenType.QUESTION: |
| 419 return symbol(QUESTION_INFO); | 417 return symbol(TokenType.QUESTION); |
| 420 case TokenType.QUESTION_PERIOD: | 418 case TokenType.QUESTION_PERIOD: |
| 421 return symbol(QUESTION_PERIOD_INFO); | 419 return symbol(TokenType.QUESTION_PERIOD); |
| 422 case TokenType.QUESTION_QUESTION: | 420 case TokenType.QUESTION_QUESTION: |
| 423 return symbol(QUESTION_QUESTION_INFO); | 421 return symbol(TokenType.QUESTION_QUESTION); |
| 424 case TokenType.QUESTION_QUESTION_EQ: | 422 case TokenType.QUESTION_QUESTION_EQ: |
| 425 return symbol(QUESTION_QUESTION_EQ_INFO); | 423 return symbol(TokenType.QUESTION_QUESTION_EQ); |
| 426 case TokenType.SEMICOLON: | 424 case TokenType.SEMICOLON: |
| 427 return symbol(SEMICOLON_INFO); | 425 return symbol(TokenType.SEMICOLON); |
| 428 case TokenType.SLASH: | 426 case TokenType.SLASH: |
| 429 return symbol(SLASH_INFO); | 427 return symbol(TokenType.SLASH); |
| 430 case TokenType.SLASH_EQ: | 428 case TokenType.SLASH_EQ: |
| 431 return symbol(SLASH_EQ_INFO); | 429 return symbol(TokenType.SLASH_EQ); |
| 432 case TokenType.STAR: | 430 case TokenType.STAR: |
| 433 return symbol(STAR_INFO); | 431 return symbol(TokenType.STAR); |
| 434 case TokenType.STAR_EQ: | 432 case TokenType.STAR_EQ: |
| 435 return symbol(STAR_EQ_INFO); | 433 return symbol(TokenType.STAR_EQ); |
| 436 case TokenType.STRING_INTERPOLATION_EXPRESSION: | 434 case TokenType.STRING_INTERPOLATION_EXPRESSION: |
| 437 return beginGroup(STRING_INTERPOLATION_INFO); | 435 return beginGroup(TokenType.STRING_INTERPOLATION_EXPRESSION); |
| 438 case TokenType.STRING_INTERPOLATION_IDENTIFIER: | 436 case TokenType.STRING_INTERPOLATION_IDENTIFIER: |
| 439 return symbol(STRING_INTERPOLATION_IDENTIFIER_INFO); | 437 return symbol(TokenType.STRING_INTERPOLATION_IDENTIFIER); |
| 440 case TokenType.TILDE: | 438 case TokenType.TILDE: |
| 441 return symbol(TILDE_INFO); | 439 return symbol(TokenType.TILDE); |
| 442 case TokenType.TILDE_SLASH: | 440 case TokenType.TILDE_SLASH: |
| 443 return symbol(TILDE_SLASH_INFO); | 441 return symbol(TokenType.TILDE_SLASH); |
| 444 case TokenType.TILDE_SLASH_EQ: | 442 case TokenType.TILDE_SLASH_EQ: |
| 445 return symbol(TILDE_SLASH_EQ_INFO); | 443 return symbol(TokenType.TILDE_SLASH_EQ); |
| 446 case TokenType.BACKPING: | 444 case TokenType.BACKPING: |
| 447 return symbol(BACKPING_INFO); | 445 return symbol(TokenType.BACKPING); |
| 448 case TokenType.BACKSLASH: | 446 case TokenType.BACKSLASH: |
| 449 return symbol(BACKSLASH_INFO); | 447 return symbol(TokenType.BACKSLASH); |
| 450 case TokenType.PERIOD_PERIOD_PERIOD: | 448 case TokenType.PERIOD_PERIOD_PERIOD: |
| 451 return symbol(PERIOD_PERIOD_PERIOD_INFO); | 449 return symbol(TokenType.PERIOD_PERIOD_PERIOD); |
| 452 // case TokenType.GENERIC_METHOD_TYPE_ASSIGN | 450 // case TokenType.GENERIC_METHOD_TYPE_ASSIGN |
| 453 // case TokenType.GENERIC_METHOD_TYPE_LIST | 451 // case TokenType.GENERIC_METHOD_TYPE_LIST |
| 454 default: | 452 default: |
| 455 return internalError('Unhandled token type ${token.type}'); | 453 return internalError('Unhandled token type ${token.type}'); |
| 456 } | 454 } |
| 457 } | 455 } |
| 458 | 456 |
| 459 analyzer.Token toAnalyzerToken(Token token, | 457 analyzer.Token toAnalyzerToken(Token token, |
| 460 [analyzer.CommentToken commentToken]) { | 458 [analyzer.CommentToken commentToken]) { |
| 461 if (token == null) return null; | 459 if (token == null) return null; |
| (...skipping 50 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 512 case SCRIPT_TOKEN: | 510 case SCRIPT_TOKEN: |
| 513 return makeStringToken(TokenType.SCRIPT_TAG); | 511 return makeStringToken(TokenType.SCRIPT_TAG); |
| 514 | 512 |
| 515 case STRING_TOKEN: | 513 case STRING_TOKEN: |
| 516 return makeStringToken(TokenType.STRING); | 514 return makeStringToken(TokenType.STRING); |
| 517 | 515 |
| 518 case OPEN_CURLY_BRACKET_TOKEN: | 516 case OPEN_CURLY_BRACKET_TOKEN: |
| 519 case OPEN_SQUARE_BRACKET_TOKEN: | 517 case OPEN_SQUARE_BRACKET_TOKEN: |
| 520 case OPEN_PAREN_TOKEN: | 518 case OPEN_PAREN_TOKEN: |
| 521 case STRING_INTERPOLATION_TOKEN: | 519 case STRING_INTERPOLATION_TOKEN: |
| 522 return makeBeginToken(getTokenType(token)); | 520 return makeBeginToken(token.type); |
| 523 | 521 |
| 524 default: | 522 default: |
| 525 if (commentToken == null) { | 523 if (commentToken == null) { |
| 526 return new analyzer.Token(getTokenType(token), token.charOffset); | 524 return new analyzer.Token(token.type, token.charOffset); |
| 527 } else { | 525 } else { |
| 528 return new analyzer.TokenWithComment( | 526 return new analyzer.TokenWithComment( |
| 529 getTokenType(token), token.charOffset, commentToken); | 527 token.type, token.charOffset, commentToken); |
| 530 } | 528 } |
| 531 break; | 529 break; |
| 532 } | 530 } |
| 533 } | 531 } |
| 534 | 532 |
| 535 final _keywordMap = { | 533 final _keywordMap = { |
| 536 "assert": analyzer.Keyword.ASSERT, | 534 "assert": analyzer.Keyword.ASSERT, |
| 537 "break": analyzer.Keyword.BREAK, | 535 "break": analyzer.Keyword.BREAK, |
| 538 "case": analyzer.Keyword.CASE, | 536 "case": analyzer.Keyword.CASE, |
| 539 "catch": analyzer.Keyword.CATCH, | 537 "catch": analyzer.Keyword.CATCH, |
| (...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 592 "hide": analyzer.Keyword.HIDE, | 590 "hide": analyzer.Keyword.HIDE, |
| 593 "native": analyzer.Keyword.NATIVE, | 591 "native": analyzer.Keyword.NATIVE, |
| 594 "of": analyzer.Keyword.OF, | 592 "of": analyzer.Keyword.OF, |
| 595 "on": analyzer.Keyword.ON, | 593 "on": analyzer.Keyword.ON, |
| 596 "patch": analyzer.Keyword.PATCH, | 594 "patch": analyzer.Keyword.PATCH, |
| 597 "show": analyzer.Keyword.SHOW, | 595 "show": analyzer.Keyword.SHOW, |
| 598 "source": analyzer.Keyword.SOURCE, | 596 "source": analyzer.Keyword.SOURCE, |
| 599 "sync": analyzer.Keyword.SYNC, | 597 "sync": analyzer.Keyword.SYNC, |
| 600 "yield": analyzer.Keyword.YIELD, | 598 "yield": analyzer.Keyword.YIELD, |
| 601 }; | 599 }; |
| 602 | |
| 603 TokenType getTokenType(Token token) { | |
| 604 switch (token.kind) { | |
| 605 case EOF_TOKEN: | |
| 606 return TokenType.EOF; | |
| 607 case DOUBLE_TOKEN: | |
| 608 return TokenType.DOUBLE; | |
| 609 case HEXADECIMAL_TOKEN: | |
| 610 return TokenType.HEXADECIMAL; | |
| 611 case IDENTIFIER_TOKEN: | |
| 612 return TokenType.IDENTIFIER; | |
| 613 case INT_TOKEN: | |
| 614 return TokenType.INT; | |
| 615 case KEYWORD_TOKEN: | |
| 616 return TokenType.KEYWORD; | |
| 617 // case MULTI_LINE_COMMENT_TOKEN: return TokenType.MULTI_LINE_COMMENT; | |
| 618 // case SCRIPT_TAG_TOKEN: return TokenType.SCRIPT_TAG; | |
| 619 // case SINGLE_LINE_COMMENT_TOKEN: return TokenType.SINGLE_LINE_COMMENT; | |
| 620 case STRING_TOKEN: | |
| 621 return TokenType.STRING; | |
| 622 case AMPERSAND_TOKEN: | |
| 623 return TokenType.AMPERSAND; | |
| 624 case AMPERSAND_AMPERSAND_TOKEN: | |
| 625 return TokenType.AMPERSAND_AMPERSAND; | |
| 626 // case AMPERSAND_AMPERSAND_EQ_TOKEN: | |
| 627 // return TokenType.AMPERSAND_AMPERSAND_EQ; | |
| 628 case AMPERSAND_EQ_TOKEN: | |
| 629 return TokenType.AMPERSAND_EQ; | |
| 630 case AT_TOKEN: | |
| 631 return TokenType.AT; | |
| 632 case BANG_TOKEN: | |
| 633 return TokenType.BANG; | |
| 634 case BANG_EQ_TOKEN: | |
| 635 return TokenType.BANG_EQ; | |
| 636 case BAR_TOKEN: | |
| 637 return TokenType.BAR; | |
| 638 case BAR_BAR_TOKEN: | |
| 639 return TokenType.BAR_BAR; | |
| 640 // case BAR_BAR_EQ_TOKEN: return TokenType.BAR_BAR_EQ; | |
| 641 case BAR_EQ_TOKEN: | |
| 642 return TokenType.BAR_EQ; | |
| 643 case COLON_TOKEN: | |
| 644 return TokenType.COLON; | |
| 645 case COMMA_TOKEN: | |
| 646 return TokenType.COMMA; | |
| 647 case CARET_TOKEN: | |
| 648 return TokenType.CARET; | |
| 649 case CARET_EQ_TOKEN: | |
| 650 return TokenType.CARET_EQ; | |
| 651 case CLOSE_CURLY_BRACKET_TOKEN: | |
| 652 return TokenType.CLOSE_CURLY_BRACKET; | |
| 653 case CLOSE_PAREN_TOKEN: | |
| 654 return TokenType.CLOSE_PAREN; | |
| 655 case CLOSE_SQUARE_BRACKET_TOKEN: | |
| 656 return TokenType.CLOSE_SQUARE_BRACKET; | |
| 657 case EQ_TOKEN: | |
| 658 return TokenType.EQ; | |
| 659 case EQ_EQ_TOKEN: | |
| 660 return TokenType.EQ_EQ; | |
| 661 case FUNCTION_TOKEN: | |
| 662 return TokenType.FUNCTION; | |
| 663 case GT_TOKEN: | |
| 664 return TokenType.GT; | |
| 665 case GT_EQ_TOKEN: | |
| 666 return TokenType.GT_EQ; | |
| 667 case GT_GT_TOKEN: | |
| 668 return TokenType.GT_GT; | |
| 669 case GT_GT_EQ_TOKEN: | |
| 670 return TokenType.GT_GT_EQ; | |
| 671 case HASH_TOKEN: | |
| 672 return TokenType.HASH; | |
| 673 case INDEX_TOKEN: | |
| 674 return TokenType.INDEX; | |
| 675 case INDEX_EQ_TOKEN: | |
| 676 return TokenType.INDEX_EQ; | |
| 677 // case IS_TOKEN: return TokenType.IS; | |
| 678 case LT_TOKEN: | |
| 679 return TokenType.LT; | |
| 680 case LT_EQ_TOKEN: | |
| 681 return TokenType.LT_EQ; | |
| 682 case LT_LT_TOKEN: | |
| 683 return TokenType.LT_LT; | |
| 684 case LT_LT_EQ_TOKEN: | |
| 685 return TokenType.LT_LT_EQ; | |
| 686 case MINUS_TOKEN: | |
| 687 return TokenType.MINUS; | |
| 688 case MINUS_EQ_TOKEN: | |
| 689 return TokenType.MINUS_EQ; | |
| 690 case MINUS_MINUS_TOKEN: | |
| 691 return TokenType.MINUS_MINUS; | |
| 692 case OPEN_CURLY_BRACKET_TOKEN: | |
| 693 return TokenType.OPEN_CURLY_BRACKET; | |
| 694 case OPEN_PAREN_TOKEN: | |
| 695 return TokenType.OPEN_PAREN; | |
| 696 case OPEN_SQUARE_BRACKET_TOKEN: | |
| 697 return TokenType.OPEN_SQUARE_BRACKET; | |
| 698 case PERCENT_TOKEN: | |
| 699 return TokenType.PERCENT; | |
| 700 case PERCENT_EQ_TOKEN: | |
| 701 return TokenType.PERCENT_EQ; | |
| 702 case PERIOD_TOKEN: | |
| 703 return TokenType.PERIOD; | |
| 704 case PERIOD_PERIOD_TOKEN: | |
| 705 return TokenType.PERIOD_PERIOD; | |
| 706 case PLUS_TOKEN: | |
| 707 return TokenType.PLUS; | |
| 708 case PLUS_EQ_TOKEN: | |
| 709 return TokenType.PLUS_EQ; | |
| 710 case PLUS_PLUS_TOKEN: | |
| 711 return TokenType.PLUS_PLUS; | |
| 712 case QUESTION_TOKEN: | |
| 713 return TokenType.QUESTION; | |
| 714 case QUESTION_PERIOD_TOKEN: | |
| 715 return TokenType.QUESTION_PERIOD; | |
| 716 case QUESTION_QUESTION_TOKEN: | |
| 717 return TokenType.QUESTION_QUESTION; | |
| 718 case QUESTION_QUESTION_EQ_TOKEN: | |
| 719 return TokenType.QUESTION_QUESTION_EQ; | |
| 720 case SEMICOLON_TOKEN: | |
| 721 return TokenType.SEMICOLON; | |
| 722 case SLASH_TOKEN: | |
| 723 return TokenType.SLASH; | |
| 724 case SLASH_EQ_TOKEN: | |
| 725 return TokenType.SLASH_EQ; | |
| 726 case STAR_TOKEN: | |
| 727 return TokenType.STAR; | |
| 728 case STAR_EQ_TOKEN: | |
| 729 return TokenType.STAR_EQ; | |
| 730 case STRING_INTERPOLATION_TOKEN: | |
| 731 return TokenType.STRING_INTERPOLATION_EXPRESSION; | |
| 732 case STRING_INTERPOLATION_IDENTIFIER_TOKEN: | |
| 733 return TokenType.STRING_INTERPOLATION_IDENTIFIER; | |
| 734 case TILDE_TOKEN: | |
| 735 return TokenType.TILDE; | |
| 736 case TILDE_SLASH_TOKEN: | |
| 737 return TokenType.TILDE_SLASH; | |
| 738 case TILDE_SLASH_EQ_TOKEN: | |
| 739 return TokenType.TILDE_SLASH_EQ; | |
| 740 case BACKPING_TOKEN: | |
| 741 return TokenType.BACKPING; | |
| 742 case BACKSLASH_TOKEN: | |
| 743 return TokenType.BACKSLASH; | |
| 744 case PERIOD_PERIOD_PERIOD_TOKEN: | |
| 745 return TokenType.PERIOD_PERIOD_PERIOD; | |
| 746 case GENERIC_METHOD_TYPE_LIST_TOKEN: | |
| 747 return TokenType.GENERIC_METHOD_TYPE_LIST; | |
| 748 case GENERIC_METHOD_TYPE_ASSIGN_TOKEN: | |
| 749 return TokenType.GENERIC_METHOD_TYPE_ASSIGN; | |
| 750 default: | |
| 751 return internalError("Unhandled token ${token.info}"); | |
| 752 } | |
| 753 } | |
| OLD | NEW |