OLD | NEW |
---|---|
1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. |
6 | 6 |
7 #include <stdint.h> | 7 #include <stdint.h> |
8 | 8 |
9 #include <cmath> | 9 #include <cmath> |
10 | 10 |
(...skipping 386 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
397 } | 397 } |
398 PushBack('!'); // undo Advance() | 398 PushBack('!'); // undo Advance() |
399 DCHECK(c0_ == '!'); | 399 DCHECK(c0_ == '!'); |
400 return Token::LT; | 400 return Token::LT; |
401 } | 401 } |
402 | 402 |
403 | 403 |
404 void Scanner::Scan() { | 404 void Scanner::Scan() { |
405 next_.literal_chars = NULL; | 405 next_.literal_chars = NULL; |
406 Token::Value token; | 406 Token::Value token; |
407 | |
407 do { | 408 do { |
408 // Remember the position of the next token | 409 // Remember the position of the next token |
409 next_.location.beg_pos = source_pos(); | 410 next_.location.beg_pos = source_pos(); |
410 | 411 |
411 switch (c0_) { | 412 switch (c0_) { |
412 case ' ': | 413 case ' ': |
413 case '\t': | 414 case '\t': |
414 Advance(); | 415 Advance(); |
415 token = Token::WHITESPACE; | 416 token = Token::WHITESPACE; |
416 break; | 417 break; |
(...skipping 202 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
619 break; | 620 break; |
620 | 621 |
621 case '?': | 622 case '?': |
622 token = Select(Token::CONDITIONAL); | 623 token = Select(Token::CONDITIONAL); |
623 break; | 624 break; |
624 | 625 |
625 case '~': | 626 case '~': |
626 token = Select(Token::BIT_NOT); | 627 token = Select(Token::BIT_NOT); |
627 break; | 628 break; |
628 | 629 |
630 case '`': | |
631 if (HarmonyTemplates()) { | |
632 token = ScanTemplateSpan(); | |
633 break; | |
634 } | |
635 | |
629 default: | 636 default: |
630 if (c0_ < 0) { | 637 if (c0_ < 0) { |
631 token = Token::EOS; | 638 token = Token::EOS; |
632 } else if (unicode_cache_->IsIdentifierStart(c0_)) { | 639 } else if (unicode_cache_->IsIdentifierStart(c0_)) { |
633 token = ScanIdentifierOrKeyword(); | 640 token = ScanIdentifierOrKeyword(); |
634 } else if (IsDecimalDigit(c0_)) { | 641 } else if (IsDecimalDigit(c0_)) { |
635 token = ScanNumber(false); | 642 token = ScanNumber(false); |
636 } else if (SkipWhiteSpace()) { | 643 } else if (SkipWhiteSpace()) { |
637 token = Token::WHITESPACE; | 644 token = Token::WHITESPACE; |
638 } else { | 645 } else { |
(...skipping 27 matching lines...) Expand all Loading... | |
666 // whether there was a line terminator in the part we skip. | 673 // whether there was a line terminator in the part we skip. |
667 has_line_terminator_before_next_ = false; | 674 has_line_terminator_before_next_ = false; |
668 has_multiline_comment_before_next_ = false; | 675 has_multiline_comment_before_next_ = false; |
669 } | 676 } |
670 Scan(); | 677 Scan(); |
671 } | 678 } |
672 | 679 |
673 | 680 |
674 bool Scanner::ScanEscape() { | 681 bool Scanner::ScanEscape() { |
675 uc32 c = c0_; | 682 uc32 c = c0_; |
683 bool singleCharEscape = true; | |
arv (Not doing code reviews)
2014/11/11 17:15:15
This is no longer used
| |
676 Advance(); | 684 Advance(); |
677 | 685 |
678 // Skip escaped newlines. | 686 // Skip escaped newlines. |
679 if (c0_ >= 0 && unicode_cache_->IsLineTerminator(c)) { | 687 if (c0_ >= 0 && unicode_cache_->IsLineTerminator(c)) { |
680 // Allow CR+LF newlines in multiline string literals. | 688 // Allow CR+LF newlines in multiline string literals. |
681 if (IsCarriageReturn(c) && IsLineFeed(c0_)) Advance(); | 689 if (IsCarriageReturn(c) && IsLineFeed(c0_)) Advance(); |
682 // Allow LF+CR newlines in multiline string literals. | 690 // Allow LF+CR newlines in multiline string literals. |
683 if (IsLineFeed(c) && IsCarriageReturn(c0_)) Advance(); | 691 if (IsLineFeed(c) && IsCarriageReturn(c0_)) Advance(); |
684 return true; | 692 return true; |
685 } | 693 } |
686 | 694 |
687 switch (c) { | 695 switch (c) { |
688 case '\'': // fall through | 696 case '\'': // fall through |
689 case '"' : // fall through | 697 case '"' : // fall through |
690 case '\\': break; | 698 case '\\': break; |
691 case 'b' : c = '\b'; break; | 699 case 'b' : c = '\b'; break; |
692 case 'f' : c = '\f'; break; | 700 case 'f' : c = '\f'; break; |
693 case 'n' : c = '\n'; break; | 701 case 'n' : c = '\n'; break; |
694 case 'r' : c = '\r'; break; | 702 case 'r' : c = '\r'; break; |
695 case 't' : c = '\t'; break; | 703 case 't' : c = '\t'; break; |
696 case 'u' : { | 704 case 'u' : { |
705 singleCharEscape = false; | |
697 c = ScanHexNumber(4); | 706 c = ScanHexNumber(4); |
698 if (c < 0) return false; | 707 if (c < 0) return false; |
699 break; | 708 break; |
700 } | 709 } |
701 case 'v' : c = '\v'; break; | 710 case 'v' : c = '\v'; break; |
702 case 'x' : { | 711 case 'x' : { |
712 singleCharEscape = false; | |
703 c = ScanHexNumber(2); | 713 c = ScanHexNumber(2); |
704 if (c < 0) return false; | 714 if (c < 0) return false; |
705 break; | 715 break; |
706 } | 716 } |
707 case '0' : // fall through | 717 case '0' : // fall through |
708 case '1' : // fall through | 718 case '1' : // fall through |
709 case '2' : // fall through | 719 case '2' : // fall through |
710 case '3' : // fall through | 720 case '3' : // fall through |
711 case '4' : // fall through | 721 case '4' : // fall through |
712 case '5' : // fall through | 722 case '5' : // fall through |
713 case '6' : // fall through | 723 case '6' : // fall through |
714 case '7' : c = ScanOctalEscape(c, 2); break; | 724 case '7': |
725 singleCharEscape = false; | |
726 c = ScanOctalEscape(c, 2); | |
727 break; | |
715 } | 728 } |
716 | 729 |
717 // According to ECMA-262, section 7.8.4, characters not covered by the | 730 // According to ECMA-262, section 7.8.4, characters not covered by the |
718 // above cases should be illegal, but they are commonly handled as | 731 // above cases should be illegal, but they are commonly handled as |
719 // non-escaped characters by JS VMs. | 732 // non-escaped characters by JS VMs. |
720 AddLiteralChar(c); | 733 AddLiteralChar(c); |
721 return true; | 734 return true; |
722 } | 735 } |
723 | 736 |
724 | 737 |
(...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
763 } | 776 } |
764 } | 777 } |
765 if (c0_ != quote) return Token::ILLEGAL; | 778 if (c0_ != quote) return Token::ILLEGAL; |
766 literal.Complete(); | 779 literal.Complete(); |
767 | 780 |
768 Advance(); // consume quote | 781 Advance(); // consume quote |
769 return Token::STRING; | 782 return Token::STRING; |
770 } | 783 } |
771 | 784 |
772 | 785 |
786 Token::Value Scanner::ScanTemplateSpan() { | |
787 // When scanning a TemplateSpan, we are looking for the following construct: | |
788 // TEMPLATE_SPAN :: | |
789 // ` LiteralChars* ${ | |
790 // | } LiteralChars* ${ | |
791 // | |
792 // TEMPLATE_TAIL :: | |
793 // ` LiteralChars* ` | |
794 // | } LiteralChar* ` | |
795 // | |
796 // A TEMPLATE_SPAN should always be followed by an Expression, while a | |
797 // TEMPLATE_TAIL terminates a TemplateLiteral and does not need to be | |
798 // followed by an Expression. | |
799 // | |
800 | |
801 if (next_.token == Token::RBRACE) { | |
802 // After parsing an Expression, the source position is incorrect due to | |
803 // having scanned the brace. Push the RBRACE back into the stream. | |
804 PushBack('}'); | |
805 } | |
806 | |
807 next_.location.beg_pos = source_pos(); | |
808 Token::Value result = Token::ILLEGAL; | |
809 DCHECK(c0_ == '`' || c0_ == '}'); | |
810 Advance(); // Consume ` or } | |
811 | |
812 LiteralScope literal(this); | |
813 while (true) { | |
814 uc32 c = c0_; | |
815 Advance(); | |
816 if (c == '`') { | |
817 result = Token::TEMPLATE_TAIL; | |
818 break; | |
819 } else if (c == '$' && c0_ == '{') { | |
820 Advance(); // Consume '{' | |
821 result = Token::TEMPLATE_SPAN; | |
822 break; | |
823 } else if (c == '\\') { | |
824 if (unicode_cache_->IsLineTerminator(c0_)) { | |
825 // The TV of LineContinuation :: \ LineTerminatorSequence is the empty | |
826 // code unit sequence. | |
827 do { | |
828 uc32 lastChar = c0_; | |
829 Advance(); | |
830 if (lastChar == '\r' && c0_ == '\n') Advance(); | |
831 } while (unicode_cache_->IsLineTerminator(c0_)); | |
832 } else if (c0_ == '0') { | |
833 Advance(); | |
834 AddLiteralChar('0'); | |
835 } else { | |
836 ScanEscape(); | |
837 } | |
838 } else if (c < 0) { | |
839 // Unterminated template literal | |
840 PushBack(c); | |
841 break; | |
842 } else { | |
843 // The TRV of LineTerminatorSequence :: <CR> is the CV 0x000A. | |
844 // The TRV of LineTerminatorSequence :: <CR><LF> is the sequence | |
845 // consisting of the CV 0x000A. | |
846 if (c == '\r') { | |
847 if (c0_ == '\n') Advance(); | |
848 c = '\n'; | |
849 } | |
850 AddLiteralChar(c); | |
851 } | |
852 } | |
853 literal.Complete(); | |
854 next_.location.end_pos = source_pos(); | |
855 next_.token = result; | |
856 return result; | |
857 } | |
858 | |
859 | |
773 void Scanner::ScanDecimalDigits() { | 860 void Scanner::ScanDecimalDigits() { |
774 while (IsDecimalDigit(c0_)) | 861 while (IsDecimalDigit(c0_)) |
775 AddLiteralCharAdvance(); | 862 AddLiteralCharAdvance(); |
776 } | 863 } |
777 | 864 |
778 | 865 |
779 Token::Value Scanner::ScanNumber(bool seen_period) { | 866 Token::Value Scanner::ScanNumber(bool seen_period) { |
780 DCHECK(IsDecimalDigit(c0_)); // the first digit of the number or the fraction | 867 DCHECK(IsDecimalDigit(c0_)); // the first digit of the number or the fraction |
781 | 868 |
782 enum { DECIMAL, HEX, OCTAL, IMPLICIT_OCTAL, BINARY } kind = DECIMAL; | 869 enum { DECIMAL, HEX, OCTAL, IMPLICIT_OCTAL, BINARY } kind = DECIMAL; |
(...skipping 543 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1326 } | 1413 } |
1327 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); | 1414 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); |
1328 } | 1415 } |
1329 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); | 1416 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); |
1330 | 1417 |
1331 backing_store_.AddBlock(bytes); | 1418 backing_store_.AddBlock(bytes); |
1332 return backing_store_.EndSequence().start(); | 1419 return backing_store_.EndSequence().start(); |
1333 } | 1420 } |
1334 | 1421 |
1335 } } // namespace v8::internal | 1422 } } // namespace v8::internal |
OLD | NEW |