OLD | NEW |
1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. |
6 | 6 |
7 #include "src/parsing/scanner.h" | 7 #include "src/parsing/scanner.h" |
8 | 8 |
9 #include <stdint.h> | 9 #include <stdint.h> |
10 | 10 |
(...skipping 44 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
55 // Need to capture identifiers in order to recognize "get" and "set" | 55 // Need to capture identifiers in order to recognize "get" and "set" |
56 // in object literals. | 56 // in object literals. |
57 Init(); | 57 Init(); |
58 // Skip initial whitespace allowing HTML comment ends just like | 58 // Skip initial whitespace allowing HTML comment ends just like |
59 // after a newline and scan first token. | 59 // after a newline and scan first token. |
60 has_line_terminator_before_next_ = true; | 60 has_line_terminator_before_next_ = true; |
61 SkipWhiteSpace(); | 61 SkipWhiteSpace(); |
62 Scan(); | 62 Scan(); |
63 } | 63 } |
64 | 64 |
65 | 65 template <bool capture_raw, bool unicode> |
66 template <bool capture_raw> | |
67 uc32 Scanner::ScanHexNumber(int expected_length) { | 66 uc32 Scanner::ScanHexNumber(int expected_length) { |
68 DCHECK(expected_length <= 4); // prevent overflow | 67 DCHECK(expected_length <= 4); // prevent overflow |
69 | 68 |
| 69 int begin = source_pos() - 2; |
70 uc32 x = 0; | 70 uc32 x = 0; |
71 for (int i = 0; i < expected_length; i++) { | 71 for (int i = 0; i < expected_length; i++) { |
72 int d = HexValue(c0_); | 72 int d = HexValue(c0_); |
73 if (d < 0) { | 73 if (d < 0) { |
| 74 ReportScannerError(Location(begin, begin + expected_length + 2), |
| 75 unicode |
| 76 ? MessageTemplate::kInvalidUnicodeEscapeSequence |
| 77 : MessageTemplate::kInvalidHexEscapeSequence); |
74 return -1; | 78 return -1; |
75 } | 79 } |
76 x = x * 16 + d; | 80 x = x * 16 + d; |
77 Advance<capture_raw>(); | 81 Advance<capture_raw>(); |
78 } | 82 } |
79 | 83 |
80 return x; | 84 return x; |
81 } | 85 } |
82 | 86 |
83 | |
84 template <bool capture_raw> | 87 template <bool capture_raw> |
85 uc32 Scanner::ScanUnlimitedLengthHexNumber(int max_value) { | 88 uc32 Scanner::ScanUnlimitedLengthHexNumber(int max_value, int beg_pos) { |
86 uc32 x = 0; | 89 uc32 x = 0; |
87 int d = HexValue(c0_); | 90 int d = HexValue(c0_); |
88 if (d < 0) { | 91 if (d < 0) return -1; |
89 return -1; | 92 |
90 } | |
91 while (d >= 0) { | 93 while (d >= 0) { |
92 x = x * 16 + d; | 94 x = x * 16 + d; |
93 if (x > max_value) return -1; | 95 if (x > max_value) { |
| 96 ReportScannerError(Location(beg_pos, source_pos() + 1), |
| 97 MessageTemplate::kUndefinedUnicodeCodePoint); |
| 98 return -1; |
| 99 } |
94 Advance<capture_raw>(); | 100 Advance<capture_raw>(); |
95 d = HexValue(c0_); | 101 d = HexValue(c0_); |
96 } | 102 } |
| 103 |
97 return x; | 104 return x; |
98 } | 105 } |
99 | 106 |
100 | 107 |
101 // Ensure that tokens can be stored in a byte. | 108 // Ensure that tokens can be stored in a byte. |
102 STATIC_ASSERT(Token::NUM_TOKENS <= 0x100); | 109 STATIC_ASSERT(Token::NUM_TOKENS <= 0x100); |
103 | 110 |
104 // Table of one-character tokens, by character (0x00..0x7f only). | 111 // Table of one-character tokens, by character (0x00..0x7f only). |
105 static const byte one_char_tokens[] = { | 112 static const byte one_char_tokens[] = { |
106 Token::ILLEGAL, | 113 Token::ILLEGAL, |
(...skipping 742 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
849 if (c == '\\') break; | 856 if (c == '\\') break; |
850 Advance<false, false>(); | 857 Advance<false, false>(); |
851 AddLiteralChar(c); | 858 AddLiteralChar(c); |
852 } | 859 } |
853 | 860 |
854 while (c0_ != quote && c0_ >= 0 | 861 while (c0_ != quote && c0_ >= 0 |
855 && !unicode_cache_->IsLineTerminator(c0_)) { | 862 && !unicode_cache_->IsLineTerminator(c0_)) { |
856 uc32 c = c0_; | 863 uc32 c = c0_; |
857 Advance(); | 864 Advance(); |
858 if (c == '\\') { | 865 if (c == '\\') { |
859 if (c0_ < 0 || !ScanEscape<false, false>()) return Token::ILLEGAL; | 866 if (c0_ < 0 || !ScanEscape<false, false>()) { |
| 867 return Token::ILLEGAL; |
| 868 } |
860 } else { | 869 } else { |
861 AddLiteralChar(c); | 870 AddLiteralChar(c); |
862 } | 871 } |
863 } | 872 } |
864 if (c0_ != quote) return Token::ILLEGAL; | 873 if (c0_ != quote) return Token::ILLEGAL; |
865 literal.Complete(); | 874 literal.Complete(); |
866 | 875 |
867 Advance(); // consume quote | 876 Advance(); // consume quote |
868 return Token::STRING; | 877 return Token::STRING; |
869 } | 878 } |
(...skipping 11 matching lines...) Expand all Loading... |
881 // | 890 // |
882 // A TEMPLATE_SPAN should always be followed by an Expression, while a | 891 // A TEMPLATE_SPAN should always be followed by an Expression, while a |
883 // TEMPLATE_TAIL terminates a TemplateLiteral and does not need to be | 892 // TEMPLATE_TAIL terminates a TemplateLiteral and does not need to be |
884 // followed by an Expression. | 893 // followed by an Expression. |
885 | 894 |
886 Token::Value result = Token::TEMPLATE_SPAN; | 895 Token::Value result = Token::TEMPLATE_SPAN; |
887 LiteralScope literal(this); | 896 LiteralScope literal(this); |
888 StartRawLiteral(); | 897 StartRawLiteral(); |
889 const bool capture_raw = true; | 898 const bool capture_raw = true; |
890 const bool in_template_literal = true; | 899 const bool in_template_literal = true; |
891 | |
892 while (true) { | 900 while (true) { |
893 uc32 c = c0_; | 901 uc32 c = c0_; |
894 Advance<capture_raw>(); | 902 Advance<capture_raw>(); |
895 if (c == '`') { | 903 if (c == '`') { |
896 result = Token::TEMPLATE_TAIL; | 904 result = Token::TEMPLATE_TAIL; |
897 ReduceRawLiteralLength(1); | 905 ReduceRawLiteralLength(1); |
898 break; | 906 break; |
899 } else if (c == '$' && c0_ == '{') { | 907 } else if (c == '$' && c0_ == '{') { |
900 Advance<capture_raw>(); // Consume '{' | 908 Advance<capture_raw>(); // Consume '{' |
901 ReduceRawLiteralLength(2); | 909 ReduceRawLiteralLength(2); |
(...skipping 199 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1101 Advance(); | 1109 Advance(); |
1102 return ScanUnicodeEscape<false>(); | 1110 return ScanUnicodeEscape<false>(); |
1103 } | 1111 } |
1104 | 1112 |
1105 | 1113 |
1106 template <bool capture_raw> | 1114 template <bool capture_raw> |
1107 uc32 Scanner::ScanUnicodeEscape() { | 1115 uc32 Scanner::ScanUnicodeEscape() { |
1108 // Accept both \uxxxx and \u{xxxxxx}. In the latter case, the number of | 1116 // Accept both \uxxxx and \u{xxxxxx}. In the latter case, the number of |
1109 // hex digits between { } is arbitrary. \ and u have already been read. | 1117 // hex digits between { } is arbitrary. \ and u have already been read. |
1110 if (c0_ == '{') { | 1118 if (c0_ == '{') { |
| 1119 int begin = source_pos() - 2; |
1111 Advance<capture_raw>(); | 1120 Advance<capture_raw>(); |
1112 uc32 cp = ScanUnlimitedLengthHexNumber<capture_raw>(0x10ffff); | 1121 uc32 cp = ScanUnlimitedLengthHexNumber<capture_raw>(0x10ffff, begin); |
1113 if (cp < 0) { | 1122 if (cp < 0 || c0_ != '}') { |
1114 return -1; | 1123 ReportScannerError(source_pos(), |
1115 } | 1124 MessageTemplate::kInvalidUnicodeEscapeSequence); |
1116 if (c0_ != '}') { | |
1117 return -1; | 1125 return -1; |
1118 } | 1126 } |
1119 Advance<capture_raw>(); | 1127 Advance<capture_raw>(); |
1120 return cp; | 1128 return cp; |
1121 } | 1129 } |
1122 return ScanHexNumber<capture_raw>(4); | 1130 const bool unicode = true; |
| 1131 return ScanHexNumber<capture_raw, unicode>(4); |
1123 } | 1132 } |
1124 | 1133 |
1125 | 1134 |
1126 // ---------------------------------------------------------------------------- | 1135 // ---------------------------------------------------------------------------- |
1127 // Keyword Matcher | 1136 // Keyword Matcher |
1128 | 1137 |
1129 #define KEYWORDS(KEYWORD_GROUP, KEYWORD) \ | 1138 #define KEYWORDS(KEYWORD_GROUP, KEYWORD) \ |
1130 KEYWORD_GROUP('b') \ | 1139 KEYWORD_GROUP('b') \ |
1131 KEYWORD("break", Token::BREAK) \ | 1140 KEYWORD("break", Token::BREAK) \ |
1132 KEYWORD_GROUP('c') \ | 1141 KEYWORD_GROUP('c') \ |
(...skipping 546 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1679 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); | 1688 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); |
1680 } | 1689 } |
1681 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); | 1690 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); |
1682 | 1691 |
1683 backing_store_.AddBlock(bytes); | 1692 backing_store_.AddBlock(bytes); |
1684 return backing_store_.EndSequence().start(); | 1693 return backing_store_.EndSequence().start(); |
1685 } | 1694 } |
1686 | 1695 |
1687 } // namespace internal | 1696 } // namespace internal |
1688 } // namespace v8 | 1697 } // namespace v8 |
OLD | NEW |