OLD | NEW |
1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. |
6 | 6 |
7 #ifndef V8_SCANNER_H_ | 7 #ifndef V8_SCANNER_H_ |
8 #define V8_SCANNER_H_ | 8 #define V8_SCANNER_H_ |
9 | 9 |
10 #include "src/allocation.h" | 10 #include "src/allocation.h" |
(...skipping 373 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
384 bool is_literal_contextual_keyword(Vector<const char> keyword) { | 384 bool is_literal_contextual_keyword(Vector<const char> keyword) { |
385 DCHECK_NOT_NULL(current_.literal_chars); | 385 DCHECK_NOT_NULL(current_.literal_chars); |
386 return current_.literal_chars->is_contextual_keyword(keyword); | 386 return current_.literal_chars->is_contextual_keyword(keyword); |
387 } | 387 } |
388 bool is_next_contextual_keyword(Vector<const char> keyword) { | 388 bool is_next_contextual_keyword(Vector<const char> keyword) { |
389 DCHECK_NOT_NULL(next_.literal_chars); | 389 DCHECK_NOT_NULL(next_.literal_chars); |
390 return next_.literal_chars->is_contextual_keyword(keyword); | 390 return next_.literal_chars->is_contextual_keyword(keyword); |
391 } | 391 } |
392 | 392 |
393 const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory); | 393 const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory); |
| 394 const AstRawString* CurrentRawSymbol(AstValueFactory* ast_value_factory); |
394 const AstRawString* NextSymbol(AstValueFactory* ast_value_factory); | 395 const AstRawString* NextSymbol(AstValueFactory* ast_value_factory); |
| 396 const AstRawString* NextRawSymbol(AstValueFactory* ast_value_factory); |
395 | 397 |
396 double DoubleValue(); | 398 double DoubleValue(); |
397 bool LiteralMatches(const char* data, int length, bool allow_escapes = true) { | 399 bool LiteralMatches(const char* data, int length, bool allow_escapes = true) { |
398 if (is_literal_one_byte() && | 400 if (is_literal_one_byte() && |
399 literal_length() == length && | 401 literal_length() == length && |
400 (allow_escapes || !literal_contains_escapes())) { | 402 (allow_escapes || !literal_contains_escapes())) { |
401 const char* token = | 403 const char* token = |
402 reinterpret_cast<const char*>(literal_one_byte_string().start()); | 404 reinterpret_cast<const char*>(literal_one_byte_string().start()); |
403 return !strncmp(token, data, length); | 405 return !strncmp(token, data, length); |
404 } | 406 } |
(...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
451 } | 453 } |
452 void SetHarmonyNumericLiterals(bool numeric_literals) { | 454 void SetHarmonyNumericLiterals(bool numeric_literals) { |
453 harmony_numeric_literals_ = numeric_literals; | 455 harmony_numeric_literals_ = numeric_literals; |
454 } | 456 } |
455 bool HarmonyClasses() const { | 457 bool HarmonyClasses() const { |
456 return harmony_classes_; | 458 return harmony_classes_; |
457 } | 459 } |
458 void SetHarmonyClasses(bool classes) { | 460 void SetHarmonyClasses(bool classes) { |
459 harmony_classes_ = classes; | 461 harmony_classes_ = classes; |
460 } | 462 } |
| 463 bool HarmonyTemplates() const { return harmony_templates_; } |
| 464 void SetHarmonyTemplates(bool templates) { harmony_templates_ = templates; } |
461 | 465 |
462 // Returns true if there was a line terminator before the peek'ed token, | 466 // Returns true if there was a line terminator before the peek'ed token, |
463 // possibly inside a multi-line comment. | 467 // possibly inside a multi-line comment. |
464 bool HasAnyLineTerminatorBeforeNext() const { | 468 bool HasAnyLineTerminatorBeforeNext() const { |
465 return has_line_terminator_before_next_ || | 469 return has_line_terminator_before_next_ || |
466 has_multiline_comment_before_next_; | 470 has_multiline_comment_before_next_; |
467 } | 471 } |
468 | 472 |
469 // Scans the input as a regular expression pattern, previous | 473 // Scans the input as a regular expression pattern, previous |
470 // character(s) must be /(=). Returns true if a pattern is scanned. | 474 // character(s) must be /(=). Returns true if a pattern is scanned. |
471 bool ScanRegExpPattern(bool seen_equal); | 475 bool ScanRegExpPattern(bool seen_equal); |
472 // Returns true if regexp flags are scanned (always since flags can | 476 // Returns true if regexp flags are scanned (always since flags can |
473 // be empty). | 477 // be empty). |
474 bool ScanRegExpFlags(); | 478 bool ScanRegExpFlags(); |
475 | 479 |
| 480 // Scans the input as a template literal |
| 481 Token::Value ScanTemplateSpan(); |
| 482 |
476 const LiteralBuffer* source_url() const { return &source_url_; } | 483 const LiteralBuffer* source_url() const { return &source_url_; } |
477 const LiteralBuffer* source_mapping_url() const { | 484 const LiteralBuffer* source_mapping_url() const { |
478 return &source_mapping_url_; | 485 return &source_mapping_url_; |
479 } | 486 } |
480 | 487 |
481 bool IdentifierIsFutureStrictReserved(const AstRawString* string) const; | 488 bool IdentifierIsFutureStrictReserved(const AstRawString* string) const; |
482 | 489 |
483 private: | 490 private: |
484 // The current and look-ahead token. | 491 // The current and look-ahead token. |
485 struct TokenDesc { | 492 struct TokenDesc { |
486 Token::Value token; | 493 Token::Value token; |
487 Location location; | 494 Location location; |
488 LiteralBuffer* literal_chars; | 495 LiteralBuffer* literal_chars; |
| 496 LiteralBuffer* raw_literal_chars; |
489 }; | 497 }; |
490 | 498 |
491 static const int kCharacterLookaheadBufferSize = 1; | 499 static const int kCharacterLookaheadBufferSize = 1; |
492 | 500 |
493 // Scans octal escape sequence. Also accepts "\0" decimal escape sequence. | 501 // Scans octal escape sequence. Also accepts "\0" decimal escape sequence. |
494 uc32 ScanOctalEscape(uc32 c, int length); | 502 uc32 ScanOctalEscape(uc32 c, int length, bool recordRaw = false); |
495 | 503 |
496 // Call this after setting source_ to the input. | 504 // Call this after setting source_ to the input. |
497 void Init() { | 505 void Init() { |
498 // Set c0_ (one character ahead) | 506 // Set c0_ (one character ahead) |
499 STATIC_ASSERT(kCharacterLookaheadBufferSize == 1); | 507 STATIC_ASSERT(kCharacterLookaheadBufferSize == 1); |
500 Advance(); | 508 Advance(); |
501 // Initialize current_ to not refer to a literal. | 509 // Initialize current_ to not refer to a literal. |
502 current_.literal_chars = NULL; | 510 current_.literal_chars = NULL; |
| 511 current_.raw_literal_chars = NULL; |
503 } | 512 } |
504 | 513 |
505 // Literal buffer support | 514 // Literal buffer support |
506 inline void StartLiteral() { | 515 inline void StartLiteral() { |
507 LiteralBuffer* free_buffer = (current_.literal_chars == &literal_buffer1_) ? | 516 LiteralBuffer* free_buffer = (current_.literal_chars == &literal_buffer1_) ? |
508 &literal_buffer2_ : &literal_buffer1_; | 517 &literal_buffer2_ : &literal_buffer1_; |
| 518 LiteralBuffer* raw_buffer = (current_.raw_literal_chars == &raw_buffer1_) |
| 519 ? &raw_buffer2_ |
| 520 : &raw_buffer1_; |
509 free_buffer->Reset(); | 521 free_buffer->Reset(); |
| 522 raw_buffer->Reset(); |
510 next_.literal_chars = free_buffer; | 523 next_.literal_chars = free_buffer; |
| 524 next_.raw_literal_chars = raw_buffer; |
511 } | 525 } |
512 | 526 |
513 INLINE(void AddLiteralChar(uc32 c)) { | 527 INLINE(void AddLiteralChar(uc32 c)) { |
514 DCHECK_NOT_NULL(next_.literal_chars); | 528 DCHECK_NOT_NULL(next_.literal_chars); |
515 next_.literal_chars->AddChar(c); | 529 next_.literal_chars->AddChar(c); |
516 } | 530 } |
517 | 531 |
| 532 INLINE(void AddRawLiteralChar(uc32 c)) { |
| 533 DCHECK_NOT_NULL(next_.raw_literal_chars); |
| 534 next_.raw_literal_chars->AddChar(c); |
| 535 } |
| 536 |
518 // Complete scanning of a literal. | 537 // Complete scanning of a literal. |
519 inline void TerminateLiteral() { | 538 inline void TerminateLiteral() { |
520 // Does nothing in the current implementation. | 539 // Does nothing in the current implementation. |
521 } | 540 } |
522 | 541 |
523 // Stops scanning of a literal and drop the collected characters, | 542 // Stops scanning of a literal and drop the collected characters, |
524 // e.g., due to an encountered error. | 543 // e.g., due to an encountered error. |
525 inline void DropLiteral() { | 544 inline void DropLiteral() { |
526 next_.literal_chars = NULL; | 545 next_.literal_chars = NULL; |
527 } | 546 } |
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
576 // These functions only give the correct result if the literal | 595 // These functions only give the correct result if the literal |
577 // was scanned between calls to StartLiteral() and TerminateLiteral(). | 596 // was scanned between calls to StartLiteral() and TerminateLiteral(). |
578 Vector<const uint8_t> literal_one_byte_string() { | 597 Vector<const uint8_t> literal_one_byte_string() { |
579 DCHECK_NOT_NULL(current_.literal_chars); | 598 DCHECK_NOT_NULL(current_.literal_chars); |
580 return current_.literal_chars->one_byte_literal(); | 599 return current_.literal_chars->one_byte_literal(); |
581 } | 600 } |
582 Vector<const uint16_t> literal_two_byte_string() { | 601 Vector<const uint16_t> literal_two_byte_string() { |
583 DCHECK_NOT_NULL(current_.literal_chars); | 602 DCHECK_NOT_NULL(current_.literal_chars); |
584 return current_.literal_chars->two_byte_literal(); | 603 return current_.literal_chars->two_byte_literal(); |
585 } | 604 } |
| 605 Vector<const uint8_t> raw_one_byte_string() { |
| 606 DCHECK_NOT_NULL(current_.raw_literal_chars); |
| 607 return current_.raw_literal_chars->one_byte_literal(); |
| 608 } |
| 609 Vector<const uint16_t> raw_two_byte_string() { |
| 610 DCHECK_NOT_NULL(current_.raw_literal_chars); |
| 611 return current_.raw_literal_chars->two_byte_literal(); |
| 612 } |
586 bool is_literal_one_byte() { | 613 bool is_literal_one_byte() { |
587 DCHECK_NOT_NULL(current_.literal_chars); | 614 DCHECK_NOT_NULL(current_.literal_chars); |
588 return current_.literal_chars->is_one_byte(); | 615 return current_.literal_chars->is_one_byte(); |
589 } | 616 } |
| 617 bool is_raw_one_byte() { |
| 618 DCHECK_NOT_NULL(current_.raw_literal_chars); |
| 619 return current_.raw_literal_chars->is_one_byte(); |
| 620 } |
590 int literal_length() const { | 621 int literal_length() const { |
591 DCHECK_NOT_NULL(current_.literal_chars); | 622 DCHECK_NOT_NULL(current_.literal_chars); |
592 return current_.literal_chars->length(); | 623 return current_.literal_chars->length(); |
593 } | 624 } |
594 // Returns the literal string for the next token (the token that | 625 // Returns the literal string for the next token (the token that |
595 // would be returned if Next() were called). | 626 // would be returned if Next() were called). |
596 Vector<const uint8_t> next_literal_one_byte_string() { | 627 Vector<const uint8_t> next_literal_one_byte_string() { |
597 DCHECK_NOT_NULL(next_.literal_chars); | 628 DCHECK_NOT_NULL(next_.literal_chars); |
598 return next_.literal_chars->one_byte_literal(); | 629 return next_.literal_chars->one_byte_literal(); |
599 } | 630 } |
600 Vector<const uint16_t> next_literal_two_byte_string() { | 631 Vector<const uint16_t> next_literal_two_byte_string() { |
601 DCHECK_NOT_NULL(next_.literal_chars); | 632 DCHECK_NOT_NULL(next_.literal_chars); |
602 return next_.literal_chars->two_byte_literal(); | 633 return next_.literal_chars->two_byte_literal(); |
603 } | 634 } |
| 635 Vector<const uint8_t> next_raw_one_byte_string() { |
| 636 DCHECK_NOT_NULL(next_.raw_literal_chars); |
| 637 return next_.raw_literal_chars->one_byte_literal(); |
| 638 } |
| 639 Vector<const uint16_t> next_raw_two_byte_string() { |
| 640 DCHECK_NOT_NULL(next_.raw_literal_chars); |
| 641 return next_.raw_literal_chars->two_byte_literal(); |
| 642 } |
604 bool is_next_literal_one_byte() { | 643 bool is_next_literal_one_byte() { |
605 DCHECK_NOT_NULL(next_.literal_chars); | 644 DCHECK_NOT_NULL(next_.literal_chars); |
606 return next_.literal_chars->is_one_byte(); | 645 return next_.literal_chars->is_one_byte(); |
607 } | 646 } |
| 647 bool is_next_raw_one_byte() { |
| 648 DCHECK_NOT_NULL(next_.raw_literal_chars); |
| 649 return next_.raw_literal_chars->is_one_byte(); |
| 650 } |
608 int next_literal_length() const { | 651 int next_literal_length() const { |
609 DCHECK_NOT_NULL(next_.literal_chars); | 652 DCHECK_NOT_NULL(next_.literal_chars); |
610 return next_.literal_chars->length(); | 653 return next_.literal_chars->length(); |
611 } | 654 } |
612 | 655 |
613 uc32 ScanHexNumber(int expected_length); | 656 uc32 ScanHexNumber(int expected_length, bool recordRaw = false); |
614 | 657 |
615 // Scans a single JavaScript token. | 658 // Scans a single JavaScript token. |
616 void Scan(); | 659 void Scan(); |
617 | 660 |
618 bool SkipWhiteSpace(); | 661 bool SkipWhiteSpace(); |
619 Token::Value SkipSingleLineComment(); | 662 Token::Value SkipSingleLineComment(); |
620 Token::Value SkipSourceURLComment(); | 663 Token::Value SkipSourceURLComment(); |
621 void TryToParseSourceURLComment(); | 664 void TryToParseSourceURLComment(); |
622 Token::Value SkipMultiLineComment(); | 665 Token::Value SkipMultiLineComment(); |
623 // Scans a possible HTML comment -- begins with '<!'. | 666 // Scans a possible HTML comment -- begins with '<!'. |
624 Token::Value ScanHtmlComment(); | 667 Token::Value ScanHtmlComment(); |
625 | 668 |
626 void ScanDecimalDigits(); | 669 void ScanDecimalDigits(); |
627 Token::Value ScanNumber(bool seen_period); | 670 Token::Value ScanNumber(bool seen_period); |
628 Token::Value ScanIdentifierOrKeyword(); | 671 Token::Value ScanIdentifierOrKeyword(); |
629 Token::Value ScanIdentifierSuffix(LiteralScope* literal); | 672 Token::Value ScanIdentifierSuffix(LiteralScope* literal); |
630 | 673 |
631 Token::Value ScanString(); | 674 Token::Value ScanString(); |
632 | 675 |
633 // Scans an escape-sequence which is part of a string and adds the | 676 // Scans an escape-sequence which is part of a string and adds the |
634 // decoded character to the current literal. Returns true if a pattern | 677 // decoded character to the current literal. Returns true if a pattern |
635 // is scanned. | 678 // is scanned. |
636 bool ScanEscape(); | 679 bool ScanEscape(bool recordRaw = false); |
637 // Decodes a Unicode escape-sequence which is part of an identifier. | 680 // Decodes a Unicode escape-sequence which is part of an identifier. |
638 // If the escape sequence cannot be decoded the result is kBadChar. | 681 // If the escape sequence cannot be decoded the result is kBadChar. |
639 uc32 ScanIdentifierUnicodeEscape(); | 682 uc32 ScanIdentifierUnicodeEscape(); |
640 // Scans a Unicode escape-sequence and adds its characters, | 683 // Scans a Unicode escape-sequence and adds its characters, |
641 // uninterpreted, to the current literal. Used for parsing RegExp | 684 // uninterpreted, to the current literal. Used for parsing RegExp |
642 // flags. | 685 // flags. |
643 bool ScanLiteralUnicodeEscape(); | 686 bool ScanLiteralUnicodeEscape(); |
644 | 687 |
645 // Return the current source position. | 688 // Return the current source position. |
646 int source_pos() { | 689 int source_pos() { |
647 return source_->pos() - kCharacterLookaheadBufferSize; | 690 return source_->pos() - kCharacterLookaheadBufferSize; |
648 } | 691 } |
649 | 692 |
650 UnicodeCache* unicode_cache_; | 693 UnicodeCache* unicode_cache_; |
651 | 694 |
652 // Buffers collecting literal strings, numbers, etc. | 695 // Buffers collecting literal strings, numbers, etc. |
653 LiteralBuffer literal_buffer1_; | 696 LiteralBuffer literal_buffer1_; |
654 LiteralBuffer literal_buffer2_; | 697 LiteralBuffer literal_buffer2_; |
655 | 698 |
| 699 // Buffer to store raw string values |
| 700 LiteralBuffer raw_buffer1_; |
| 701 LiteralBuffer raw_buffer2_; |
| 702 |
656 // Values parsed from magic comments. | 703 // Values parsed from magic comments. |
657 LiteralBuffer source_url_; | 704 LiteralBuffer source_url_; |
658 LiteralBuffer source_mapping_url_; | 705 LiteralBuffer source_mapping_url_; |
659 | 706 |
660 TokenDesc current_; // desc for current token (as returned by Next()) | 707 TokenDesc current_; // desc for current token (as returned by Next()) |
661 TokenDesc next_; // desc for next token (one token look-ahead) | 708 TokenDesc next_; // desc for next token (one token look-ahead) |
662 | 709 |
663 // Input stream. Must be initialized to an Utf16CharacterStream. | 710 // Input stream. Must be initialized to an Utf16CharacterStream. |
664 Utf16CharacterStream* source_; | 711 Utf16CharacterStream* source_; |
665 | 712 |
(...skipping 12 matching lines...) Expand all Loading... |
678 // line-terminator after the current token, and before the next. | 725 // line-terminator after the current token, and before the next. |
679 bool has_multiline_comment_before_next_; | 726 bool has_multiline_comment_before_next_; |
680 // Whether we scan 'let' as a keyword for harmony block-scoped let bindings. | 727 // Whether we scan 'let' as a keyword for harmony block-scoped let bindings. |
681 bool harmony_scoping_; | 728 bool harmony_scoping_; |
682 // Whether we scan 'module', 'import', 'export' as keywords. | 729 // Whether we scan 'module', 'import', 'export' as keywords. |
683 bool harmony_modules_; | 730 bool harmony_modules_; |
684 // Whether we scan 0o777 and 0b111 as numbers. | 731 // Whether we scan 0o777 and 0b111 as numbers. |
685 bool harmony_numeric_literals_; | 732 bool harmony_numeric_literals_; |
686 // Whether we scan 'class', 'extends', 'static' and 'super' as keywords. | 733 // Whether we scan 'class', 'extends', 'static' and 'super' as keywords. |
687 bool harmony_classes_; | 734 bool harmony_classes_; |
| 735 // Whether we scan TEMPLATE_SPAN and TEMPLATE_TAIL |
| 736 bool harmony_templates_; |
688 }; | 737 }; |
689 | 738 |
690 } } // namespace v8::internal | 739 } } // namespace v8::internal |
691 | 740 |
692 #endif // V8_SCANNER_H_ | 741 #endif // V8_SCANNER_H_ |
OLD | NEW |