| OLD | NEW |
| 1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. |
| 6 | 6 |
| 7 #include "src/parsing/scanner.h" | 7 #include "src/parsing/scanner.h" |
| 8 | 8 |
| 9 #include <stdint.h> | 9 #include <stdint.h> |
| 10 | 10 |
| (...skipping 44 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 55 // Need to capture identifiers in order to recognize "get" and "set" | 55 // Need to capture identifiers in order to recognize "get" and "set" |
| 56 // in object literals. | 56 // in object literals. |
| 57 Init(); | 57 Init(); |
| 58 // Skip initial whitespace allowing HTML comment ends just like | 58 // Skip initial whitespace allowing HTML comment ends just like |
| 59 // after a newline and scan first token. | 59 // after a newline and scan first token. |
| 60 has_line_terminator_before_next_ = true; | 60 has_line_terminator_before_next_ = true; |
| 61 SkipWhiteSpace(); | 61 SkipWhiteSpace(); |
| 62 Scan(); | 62 Scan(); |
| 63 } | 63 } |
| 64 | 64 |
| 65 | 65 template <bool capture_raw, bool unicode> |
| 66 template <bool capture_raw> | |
| 67 uc32 Scanner::ScanHexNumber(int expected_length) { | 66 uc32 Scanner::ScanHexNumber(int expected_length) { |
| 68 DCHECK(expected_length <= 4); // prevent overflow | 67 DCHECK(expected_length <= 4); // prevent overflow |
| 69 | 68 |
| 69 int begin = source_pos() - 2; |
| 70 uc32 x = 0; | 70 uc32 x = 0; |
| 71 for (int i = 0; i < expected_length; i++) { | 71 for (int i = 0; i < expected_length; i++) { |
| 72 int d = HexValue(c0_); | 72 int d = HexValue(c0_); |
| 73 if (d < 0) { | 73 if (d < 0) { |
| 74 ReportScannerError(Location(begin, begin + expected_length + 2), |
| 75 unicode |
| 76 ? MessageTemplate::kInvalidUnicodeEscapeSequence |
| 77 : MessageTemplate::kInvalidHexEscapeSequence); |
| 74 return -1; | 78 return -1; |
| 75 } | 79 } |
| 76 x = x * 16 + d; | 80 x = x * 16 + d; |
| 77 Advance<capture_raw>(); | 81 Advance<capture_raw>(); |
| 78 } | 82 } |
| 79 | 83 |
| 80 return x; | 84 return x; |
| 81 } | 85 } |
| 82 | 86 |
| 83 | |
| 84 template <bool capture_raw> | 87 template <bool capture_raw> |
| 85 uc32 Scanner::ScanUnlimitedLengthHexNumber(int max_value) { | 88 uc32 Scanner::ScanUnlimitedLengthHexNumber(int max_value, int beg_pos) { |
| 86 uc32 x = 0; | 89 uc32 x = 0; |
| 87 int d = HexValue(c0_); | 90 int d = HexValue(c0_); |
| 88 if (d < 0) { | 91 if (d < 0) return -1; |
| 89 return -1; | 92 |
| 90 } | |
| 91 while (d >= 0) { | 93 while (d >= 0) { |
| 92 x = x * 16 + d; | 94 x = x * 16 + d; |
| 93 if (x > max_value) return -1; | 95 if (x > max_value) { |
| 96 ReportScannerError(Location(beg_pos, source_pos() + 1), |
| 97 MessageTemplate::kUndefinedUnicodeCodePoint); |
| 98 return -1; |
| 99 } |
| 94 Advance<capture_raw>(); | 100 Advance<capture_raw>(); |
| 95 d = HexValue(c0_); | 101 d = HexValue(c0_); |
| 96 } | 102 } |
| 103 |
| 97 return x; | 104 return x; |
| 98 } | 105 } |
| 99 | 106 |
| 100 | 107 |
| 101 // Ensure that tokens can be stored in a byte. | 108 // Ensure that tokens can be stored in a byte. |
| 102 STATIC_ASSERT(Token::NUM_TOKENS <= 0x100); | 109 STATIC_ASSERT(Token::NUM_TOKENS <= 0x100); |
| 103 | 110 |
| 104 // Table of one-character tokens, by character (0x00..0x7f only). | 111 // Table of one-character tokens, by character (0x00..0x7f only). |
| 105 static const byte one_char_tokens[] = { | 112 static const byte one_char_tokens[] = { |
| 106 Token::ILLEGAL, | 113 Token::ILLEGAL, |
| (...skipping 742 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 849 if (c == '\\') break; | 856 if (c == '\\') break; |
| 850 Advance<false, false>(); | 857 Advance<false, false>(); |
| 851 AddLiteralChar(c); | 858 AddLiteralChar(c); |
| 852 } | 859 } |
| 853 | 860 |
| 854 while (c0_ != quote && c0_ >= 0 | 861 while (c0_ != quote && c0_ >= 0 |
| 855 && !unicode_cache_->IsLineTerminator(c0_)) { | 862 && !unicode_cache_->IsLineTerminator(c0_)) { |
| 856 uc32 c = c0_; | 863 uc32 c = c0_; |
| 857 Advance(); | 864 Advance(); |
| 858 if (c == '\\') { | 865 if (c == '\\') { |
| 859 if (c0_ < 0 || !ScanEscape<false, false>()) return Token::ILLEGAL; | 866 if (c0_ < 0 || !ScanEscape<false, false>()) { |
| 867 return Token::ILLEGAL; |
| 868 } |
| 860 } else { | 869 } else { |
| 861 AddLiteralChar(c); | 870 AddLiteralChar(c); |
| 862 } | 871 } |
| 863 } | 872 } |
| 864 if (c0_ != quote) return Token::ILLEGAL; | 873 if (c0_ != quote) return Token::ILLEGAL; |
| 865 literal.Complete(); | 874 literal.Complete(); |
| 866 | 875 |
| 867 Advance(); // consume quote | 876 Advance(); // consume quote |
| 868 return Token::STRING; | 877 return Token::STRING; |
| 869 } | 878 } |
| (...skipping 11 matching lines...) Expand all Loading... |
| 881 // | 890 // |
| 882 // A TEMPLATE_SPAN should always be followed by an Expression, while a | 891 // A TEMPLATE_SPAN should always be followed by an Expression, while a |
| 883 // TEMPLATE_TAIL terminates a TemplateLiteral and does not need to be | 892 // TEMPLATE_TAIL terminates a TemplateLiteral and does not need to be |
| 884 // followed by an Expression. | 893 // followed by an Expression. |
| 885 | 894 |
| 886 Token::Value result = Token::TEMPLATE_SPAN; | 895 Token::Value result = Token::TEMPLATE_SPAN; |
| 887 LiteralScope literal(this); | 896 LiteralScope literal(this); |
| 888 StartRawLiteral(); | 897 StartRawLiteral(); |
| 889 const bool capture_raw = true; | 898 const bool capture_raw = true; |
| 890 const bool in_template_literal = true; | 899 const bool in_template_literal = true; |
| 891 | |
| 892 while (true) { | 900 while (true) { |
| 893 uc32 c = c0_; | 901 uc32 c = c0_; |
| 894 Advance<capture_raw>(); | 902 Advance<capture_raw>(); |
| 895 if (c == '`') { | 903 if (c == '`') { |
| 896 result = Token::TEMPLATE_TAIL; | 904 result = Token::TEMPLATE_TAIL; |
| 897 ReduceRawLiteralLength(1); | 905 ReduceRawLiteralLength(1); |
| 898 break; | 906 break; |
| 899 } else if (c == '$' && c0_ == '{') { | 907 } else if (c == '$' && c0_ == '{') { |
| 900 Advance<capture_raw>(); // Consume '{' | 908 Advance<capture_raw>(); // Consume '{' |
| 901 ReduceRawLiteralLength(2); | 909 ReduceRawLiteralLength(2); |
| (...skipping 199 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1101 Advance(); | 1109 Advance(); |
| 1102 return ScanUnicodeEscape<false>(); | 1110 return ScanUnicodeEscape<false>(); |
| 1103 } | 1111 } |
| 1104 | 1112 |
| 1105 | 1113 |
| 1106 template <bool capture_raw> | 1114 template <bool capture_raw> |
| 1107 uc32 Scanner::ScanUnicodeEscape() { | 1115 uc32 Scanner::ScanUnicodeEscape() { |
| 1108 // Accept both \uxxxx and \u{xxxxxx}. In the latter case, the number of | 1116 // Accept both \uxxxx and \u{xxxxxx}. In the latter case, the number of |
| 1109 // hex digits between { } is arbitrary. \ and u have already been read. | 1117 // hex digits between { } is arbitrary. \ and u have already been read. |
| 1110 if (c0_ == '{') { | 1118 if (c0_ == '{') { |
| 1119 int begin = source_pos() - 2; |
| 1111 Advance<capture_raw>(); | 1120 Advance<capture_raw>(); |
| 1112 uc32 cp = ScanUnlimitedLengthHexNumber<capture_raw>(0x10ffff); | 1121 uc32 cp = ScanUnlimitedLengthHexNumber<capture_raw>(0x10ffff, begin); |
| 1113 if (cp < 0) { | 1122 if (cp < 0 || c0_ != '}') { |
| 1114 return -1; | 1123 ReportScannerError(source_pos(), |
| 1115 } | 1124 MessageTemplate::kInvalidUnicodeEscapeSequence); |
| 1116 if (c0_ != '}') { | |
| 1117 return -1; | 1125 return -1; |
| 1118 } | 1126 } |
| 1119 Advance<capture_raw>(); | 1127 Advance<capture_raw>(); |
| 1120 return cp; | 1128 return cp; |
| 1121 } | 1129 } |
| 1122 return ScanHexNumber<capture_raw>(4); | 1130 const bool unicode = true; |
| 1131 return ScanHexNumber<capture_raw, unicode>(4); |
| 1123 } | 1132 } |
| 1124 | 1133 |
| 1125 | 1134 |
| 1126 // ---------------------------------------------------------------------------- | 1135 // ---------------------------------------------------------------------------- |
| 1127 // Keyword Matcher | 1136 // Keyword Matcher |
| 1128 | 1137 |
| 1129 #define KEYWORDS(KEYWORD_GROUP, KEYWORD) \ | 1138 #define KEYWORDS(KEYWORD_GROUP, KEYWORD) \ |
| 1130 KEYWORD_GROUP('b') \ | 1139 KEYWORD_GROUP('b') \ |
| 1131 KEYWORD("break", Token::BREAK) \ | 1140 KEYWORD("break", Token::BREAK) \ |
| 1132 KEYWORD_GROUP('c') \ | 1141 KEYWORD_GROUP('c') \ |
| (...skipping 546 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1679 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); | 1688 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); |
| 1680 } | 1689 } |
| 1681 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); | 1690 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); |
| 1682 | 1691 |
| 1683 backing_store_.AddBlock(bytes); | 1692 backing_store_.AddBlock(bytes); |
| 1684 return backing_store_.EndSequence().start(); | 1693 return backing_store_.EndSequence().start(); |
| 1685 } | 1694 } |
| 1686 | 1695 |
| 1687 } // namespace internal | 1696 } // namespace internal |
| 1688 } // namespace v8 | 1697 } // namespace v8 |
| OLD | NEW |