OLD | NEW |
1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. |
6 | 6 |
7 #include "src/parsing/scanner.h" | 7 #include "src/parsing/scanner.h" |
8 | 8 |
9 #include <stdint.h> | 9 #include <stdint.h> |
10 | 10 |
(...skipping 82 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
93 } | 93 } |
94 return x; | 94 return x; |
95 } | 95 } |
96 | 96 |
97 | 97 |
98 // Ensure that tokens can be stored in a byte. | 98 // Ensure that tokens can be stored in a byte. |
99 STATIC_ASSERT(Token::NUM_TOKENS <= 0x100); | 99 STATIC_ASSERT(Token::NUM_TOKENS <= 0x100); |
100 | 100 |
101 // Table of one-character tokens, by character (0x00..0x7f only). | 101 // Table of one-character tokens, by character (0x00..0x7f only). |
102 static const byte one_char_tokens[] = { | 102 static const byte one_char_tokens[] = { |
103 Token::ILLEGAL, | 103 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
104 Token::ILLEGAL, | 104 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
105 Token::ILLEGAL, | 105 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
106 Token::ILLEGAL, | 106 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
107 Token::ILLEGAL, | 107 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
108 Token::ILLEGAL, | 108 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
109 Token::ILLEGAL, | 109 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
110 Token::ILLEGAL, | 110 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
111 Token::ILLEGAL, | 111 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
112 Token::ILLEGAL, | 112 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
113 Token::ILLEGAL, | 113 Token::LPAREN, // 0x28 |
114 Token::ILLEGAL, | 114 Token::RPAREN, // 0x29 |
115 Token::ILLEGAL, | 115 Token::ILLEGAL, Token::ILLEGAL, |
116 Token::ILLEGAL, | 116 Token::COMMA, // 0x2c |
117 Token::ILLEGAL, | 117 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
118 Token::ILLEGAL, | 118 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
119 Token::ILLEGAL, | 119 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
120 Token::ILLEGAL, | 120 Token::ILLEGAL, |
121 Token::ILLEGAL, | 121 Token::COLON, // 0x3a |
122 Token::ILLEGAL, | 122 Token::SEMICOLON, // 0x3b |
123 Token::ILLEGAL, | 123 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
124 Token::ILLEGAL, | 124 Token::CONDITIONAL, // 0x3f |
125 Token::ILLEGAL, | 125 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
126 Token::ILLEGAL, | 126 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
127 Token::ILLEGAL, | 127 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
128 Token::ILLEGAL, | 128 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
129 Token::ILLEGAL, | 129 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
130 Token::ILLEGAL, | 130 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
131 Token::ILLEGAL, | 131 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
132 Token::ILLEGAL, | 132 Token::LBRACK, // 0x5b |
133 Token::ILLEGAL, | 133 Token::ILLEGAL, |
134 Token::ILLEGAL, | 134 Token::RBRACK, // 0x5d |
135 Token::ILLEGAL, | 135 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
136 Token::ILLEGAL, | 136 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
137 Token::ILLEGAL, | 137 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
138 Token::ILLEGAL, | 138 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
139 Token::ILLEGAL, | 139 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
140 Token::ILLEGAL, | 140 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
141 Token::ILLEGAL, | 141 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
142 Token::ILLEGAL, | 142 Token::ILLEGAL, |
143 Token::LPAREN, // 0x28 | 143 Token::LBRACE, // 0x7b |
144 Token::RPAREN, // 0x29 | 144 Token::ILLEGAL, |
145 Token::ILLEGAL, | 145 Token::RBRACE, // 0x7d |
146 Token::ILLEGAL, | 146 Token::BIT_NOT, // 0x7e |
147 Token::COMMA, // 0x2c | 147 Token::ILLEGAL}; |
148 Token::ILLEGAL, | |
149 Token::ILLEGAL, | |
150 Token::ILLEGAL, | |
151 Token::ILLEGAL, | |
152 Token::ILLEGAL, | |
153 Token::ILLEGAL, | |
154 Token::ILLEGAL, | |
155 Token::ILLEGAL, | |
156 Token::ILLEGAL, | |
157 Token::ILLEGAL, | |
158 Token::ILLEGAL, | |
159 Token::ILLEGAL, | |
160 Token::ILLEGAL, | |
161 Token::COLON, // 0x3a | |
162 Token::SEMICOLON, // 0x3b | |
163 Token::ILLEGAL, | |
164 Token::ILLEGAL, | |
165 Token::ILLEGAL, | |
166 Token::CONDITIONAL, // 0x3f | |
167 Token::ILLEGAL, | |
168 Token::ILLEGAL, | |
169 Token::ILLEGAL, | |
170 Token::ILLEGAL, | |
171 Token::ILLEGAL, | |
172 Token::ILLEGAL, | |
173 Token::ILLEGAL, | |
174 Token::ILLEGAL, | |
175 Token::ILLEGAL, | |
176 Token::ILLEGAL, | |
177 Token::ILLEGAL, | |
178 Token::ILLEGAL, | |
179 Token::ILLEGAL, | |
180 Token::ILLEGAL, | |
181 Token::ILLEGAL, | |
182 Token::ILLEGAL, | |
183 Token::ILLEGAL, | |
184 Token::ILLEGAL, | |
185 Token::ILLEGAL, | |
186 Token::ILLEGAL, | |
187 Token::ILLEGAL, | |
188 Token::ILLEGAL, | |
189 Token::ILLEGAL, | |
190 Token::ILLEGAL, | |
191 Token::ILLEGAL, | |
192 Token::ILLEGAL, | |
193 Token::ILLEGAL, | |
194 Token::LBRACK, // 0x5b | |
195 Token::ILLEGAL, | |
196 Token::RBRACK, // 0x5d | |
197 Token::ILLEGAL, | |
198 Token::ILLEGAL, | |
199 Token::ILLEGAL, | |
200 Token::ILLEGAL, | |
201 Token::ILLEGAL, | |
202 Token::ILLEGAL, | |
203 Token::ILLEGAL, | |
204 Token::ILLEGAL, | |
205 Token::ILLEGAL, | |
206 Token::ILLEGAL, | |
207 Token::ILLEGAL, | |
208 Token::ILLEGAL, | |
209 Token::ILLEGAL, | |
210 Token::ILLEGAL, | |
211 Token::ILLEGAL, | |
212 Token::ILLEGAL, | |
213 Token::ILLEGAL, | |
214 Token::ILLEGAL, | |
215 Token::ILLEGAL, | |
216 Token::ILLEGAL, | |
217 Token::ILLEGAL, | |
218 Token::ILLEGAL, | |
219 Token::ILLEGAL, | |
220 Token::ILLEGAL, | |
221 Token::ILLEGAL, | |
222 Token::ILLEGAL, | |
223 Token::ILLEGAL, | |
224 Token::ILLEGAL, | |
225 Token::ILLEGAL, | |
226 Token::LBRACE, // 0x7b | |
227 Token::ILLEGAL, | |
228 Token::RBRACE, // 0x7d | |
229 Token::BIT_NOT, // 0x7e | |
230 Token::ILLEGAL | |
231 }; | |
232 | 148 |
233 | 149 |
234 Token::Value Scanner::Next() { | 150 Token::Value Scanner::Next() { |
235 if (next_.token == Token::EOS) { | 151 if (next_.token == Token::EOS) { |
236 next_.location.beg_pos = current_.location.beg_pos; | 152 next_.location.beg_pos = current_.location.beg_pos; |
237 next_.location.end_pos = current_.location.end_pos; | 153 next_.location.end_pos = current_.location.end_pos; |
238 } | 154 } |
239 current_ = next_; | 155 current_ = next_; |
240 if (V8_UNLIKELY(next_next_.token != Token::UNINITIALIZED)) { | 156 if (V8_UNLIKELY(next_next_.token != Token::UNINITIALIZED)) { |
241 next_ = next_next_; | 157 next_ = next_next_; |
(...skipping 127 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
369 if (!name.is_one_byte()) return; | 285 if (!name.is_one_byte()) return; |
370 Vector<const uint8_t> name_literal = name.one_byte_literal(); | 286 Vector<const uint8_t> name_literal = name.one_byte_literal(); |
371 LiteralBuffer* value; | 287 LiteralBuffer* value; |
372 if (name_literal == STATIC_CHAR_VECTOR("sourceURL")) { | 288 if (name_literal == STATIC_CHAR_VECTOR("sourceURL")) { |
373 value = &source_url_; | 289 value = &source_url_; |
374 } else if (name_literal == STATIC_CHAR_VECTOR("sourceMappingURL")) { | 290 } else if (name_literal == STATIC_CHAR_VECTOR("sourceMappingURL")) { |
375 value = &source_mapping_url_; | 291 value = &source_mapping_url_; |
376 } else { | 292 } else { |
377 return; | 293 return; |
378 } | 294 } |
379 if (c0_ != '=') | 295 if (c0_ != '=') return; |
380 return; | |
381 Advance(); | 296 Advance(); |
382 value->Reset(); | 297 value->Reset(); |
383 while (c0_ >= 0 && unicode_cache_->IsWhiteSpace(c0_)) { | 298 while (c0_ >= 0 && unicode_cache_->IsWhiteSpace(c0_)) { |
384 Advance(); | 299 Advance(); |
385 } | 300 } |
386 while (c0_ >= 0 && !unicode_cache_->IsLineTerminator(c0_)) { | 301 while (c0_ >= 0 && !unicode_cache_->IsLineTerminator(c0_)) { |
387 // Disallowed characters. | 302 // Disallowed characters. |
388 if (c0_ == '"' || c0_ == '\'') { | 303 if (c0_ == '"' || c0_ == '\'') { |
389 value->Reset(); | 304 value->Reset(); |
390 return; | 305 return; |
(...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
461 Advance(); | 376 Advance(); |
462 token = Token::WHITESPACE; | 377 token = Token::WHITESPACE; |
463 break; | 378 break; |
464 | 379 |
465 case '\n': | 380 case '\n': |
466 Advance(); | 381 Advance(); |
467 has_line_terminator_before_next_ = true; | 382 has_line_terminator_before_next_ = true; |
468 token = Token::WHITESPACE; | 383 token = Token::WHITESPACE; |
469 break; | 384 break; |
470 | 385 |
471 case '"': case '\'': | 386 case '"': |
| 387 case '\'': |
472 token = ScanString(); | 388 token = ScanString(); |
473 break; | 389 break; |
474 | 390 |
475 case '<': | 391 case '<': |
476 // < <= << <<= <!-- | 392 // < <= << <<= <!-- |
477 Advance(); | 393 Advance(); |
478 if (c0_ == '=') { | 394 if (c0_ == '=') { |
479 token = Select(Token::LTE); | 395 token = Select(Token::LTE); |
480 } else if (c0_ == '<') { | 396 } else if (c0_ == '<') { |
481 token = Select('=', Token::ASSIGN_SHL, Token::SHL); | 397 token = Select('=', Token::ASSIGN_SHL, Token::SHL); |
(...skipping 258 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
740 if (!in_template_literal && c0_ >= 0 && unicode_cache_->IsLineTerminator(c)) { | 656 if (!in_template_literal && c0_ >= 0 && unicode_cache_->IsLineTerminator(c)) { |
741 // Allow CR+LF newlines in multiline string literals. | 657 // Allow CR+LF newlines in multiline string literals. |
742 if (IsCarriageReturn(c) && IsLineFeed(c0_)) Advance<capture_raw>(); | 658 if (IsCarriageReturn(c) && IsLineFeed(c0_)) Advance<capture_raw>(); |
743 // Allow LF+CR newlines in multiline string literals. | 659 // Allow LF+CR newlines in multiline string literals. |
744 if (IsLineFeed(c) && IsCarriageReturn(c0_)) Advance<capture_raw>(); | 660 if (IsLineFeed(c) && IsCarriageReturn(c0_)) Advance<capture_raw>(); |
745 return true; | 661 return true; |
746 } | 662 } |
747 | 663 |
748 switch (c) { | 664 switch (c) { |
749 case '\'': // fall through | 665 case '\'': // fall through |
750 case '"' : // fall through | 666 case '"': // fall through |
751 case '\\': break; | 667 case '\\': |
752 case 'b' : c = '\b'; break; | 668 break; |
753 case 'f' : c = '\f'; break; | 669 case 'b': |
754 case 'n' : c = '\n'; break; | 670 c = '\b'; |
755 case 'r' : c = '\r'; break; | 671 break; |
756 case 't' : c = '\t'; break; | 672 case 'f': |
757 case 'u' : { | 673 c = '\f'; |
| 674 break; |
| 675 case 'n': |
| 676 c = '\n'; |
| 677 break; |
| 678 case 'r': |
| 679 c = '\r'; |
| 680 break; |
| 681 case 't': |
| 682 c = '\t'; |
| 683 break; |
| 684 case 'u': { |
758 c = ScanUnicodeEscape<capture_raw>(); | 685 c = ScanUnicodeEscape<capture_raw>(); |
759 if (c < 0) return false; | 686 if (c < 0) return false; |
760 break; | 687 break; |
761 } | 688 } |
762 case 'v': | 689 case 'v': |
763 c = '\v'; | 690 c = '\v'; |
764 break; | 691 break; |
765 case 'x': { | 692 case 'x': { |
766 c = ScanHexNumber<capture_raw>(2); | 693 c = ScanHexNumber<capture_raw>(2); |
767 if (c < 0) return false; | 694 if (c < 0) return false; |
(...skipping 63 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
831 literal.Complete(); | 758 literal.Complete(); |
832 Advance<false, false>(); | 759 Advance<false, false>(); |
833 return Token::STRING; | 760 return Token::STRING; |
834 } | 761 } |
835 uc32 c = c0_; | 762 uc32 c = c0_; |
836 if (c == '\\') break; | 763 if (c == '\\') break; |
837 Advance<false, false>(); | 764 Advance<false, false>(); |
838 AddLiteralChar(c); | 765 AddLiteralChar(c); |
839 } | 766 } |
840 | 767 |
841 while (c0_ != quote && c0_ >= 0 | 768 while (c0_ != quote && c0_ >= 0 && !unicode_cache_->IsLineTerminator(c0_)) { |
842 && !unicode_cache_->IsLineTerminator(c0_)) { | |
843 uc32 c = c0_; | 769 uc32 c = c0_; |
844 Advance(); | 770 Advance(); |
845 if (c == '\\') { | 771 if (c == '\\') { |
846 if (c0_ < 0 || !ScanEscape<false, false>()) return Token::ILLEGAL; | 772 if (c0_ < 0 || !ScanEscape<false, false>()) return Token::ILLEGAL; |
847 } else { | 773 } else { |
848 AddLiteralChar(c); | 774 AddLiteralChar(c); |
849 } | 775 } |
850 } | 776 } |
851 if (c0_ != quote) return Token::ILLEGAL; | 777 if (c0_ != quote) return Token::ILLEGAL; |
852 literal.Complete(); | 778 literal.Complete(); |
(...skipping 87 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
940 | 866 |
941 | 867 |
942 Token::Value Scanner::ScanTemplateContinuation() { | 868 Token::Value Scanner::ScanTemplateContinuation() { |
943 DCHECK_EQ(next_.token, Token::RBRACE); | 869 DCHECK_EQ(next_.token, Token::RBRACE); |
944 next_.location.beg_pos = source_pos() - 1; // We already consumed } | 870 next_.location.beg_pos = source_pos() - 1; // We already consumed } |
945 return ScanTemplateSpan(); | 871 return ScanTemplateSpan(); |
946 } | 872 } |
947 | 873 |
948 | 874 |
949 void Scanner::ScanDecimalDigits() { | 875 void Scanner::ScanDecimalDigits() { |
950 while (IsDecimalDigit(c0_)) | 876 while (IsDecimalDigit(c0_)) AddLiteralCharAdvance(); |
951 AddLiteralCharAdvance(); | |
952 } | 877 } |
953 | 878 |
954 | 879 |
955 Token::Value Scanner::ScanNumber(bool seen_period) { | 880 Token::Value Scanner::ScanNumber(bool seen_period) { |
956 DCHECK(IsDecimalDigit(c0_)); // the first digit of the number or the fraction | 881 DCHECK(IsDecimalDigit(c0_)); // the first digit of the number or the fraction |
957 | 882 |
958 enum { DECIMAL, HEX, OCTAL, IMPLICIT_OCTAL, BINARY } kind = DECIMAL; | 883 enum { DECIMAL, HEX, OCTAL, IMPLICIT_OCTAL, BINARY } kind = DECIMAL; |
959 | 884 |
960 LiteralScope literal(this); | 885 LiteralScope literal(this); |
961 bool at_start = !seen_period; | 886 bool at_start = !seen_period; |
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1005 } | 930 } |
1006 } else if ('0' <= c0_ && c0_ <= '7') { | 931 } else if ('0' <= c0_ && c0_ <= '7') { |
1007 // (possible) octal number | 932 // (possible) octal number |
1008 kind = IMPLICIT_OCTAL; | 933 kind = IMPLICIT_OCTAL; |
1009 while (true) { | 934 while (true) { |
1010 if (c0_ == '8' || c0_ == '9') { | 935 if (c0_ == '8' || c0_ == '9') { |
1011 at_start = false; | 936 at_start = false; |
1012 kind = DECIMAL; | 937 kind = DECIMAL; |
1013 break; | 938 break; |
1014 } | 939 } |
1015 if (c0_ < '0' || '7' < c0_) { | 940 if (c0_ < '0' || '7' < c0_) { |
1016 // Octal literal finished. | 941 // Octal literal finished. |
1017 octal_pos_ = Location(start_pos, source_pos()); | 942 octal_pos_ = Location(start_pos, source_pos()); |
1018 break; | 943 break; |
1019 } | 944 } |
1020 AddLiteralCharAdvance(); | 945 AddLiteralCharAdvance(); |
1021 } | 946 } |
1022 } | 947 } |
1023 } | 948 } |
1024 | 949 |
1025 // Parse decimal digits and allow trailing fractional part. | 950 // Parse decimal digits and allow trailing fractional part. |
(...skipping 26 matching lines...) Expand all Loading... |
1052 } | 977 } |
1053 } | 978 } |
1054 } | 979 } |
1055 | 980 |
1056 // scan exponent, if any | 981 // scan exponent, if any |
1057 if (c0_ == 'e' || c0_ == 'E') { | 982 if (c0_ == 'e' || c0_ == 'E') { |
1058 DCHECK(kind != HEX); // 'e'/'E' must be scanned as part of the hex number | 983 DCHECK(kind != HEX); // 'e'/'E' must be scanned as part of the hex number |
1059 if (kind != DECIMAL) return Token::ILLEGAL; | 984 if (kind != DECIMAL) return Token::ILLEGAL; |
1060 // scan exponent | 985 // scan exponent |
1061 AddLiteralCharAdvance(); | 986 AddLiteralCharAdvance(); |
1062 if (c0_ == '+' || c0_ == '-') | 987 if (c0_ == '+' || c0_ == '-') AddLiteralCharAdvance(); |
1063 AddLiteralCharAdvance(); | |
1064 if (!IsDecimalDigit(c0_)) { | 988 if (!IsDecimalDigit(c0_)) { |
1065 // we must have at least one decimal digit after 'e'/'E' | 989 // we must have at least one decimal digit after 'e'/'E' |
1066 return Token::ILLEGAL; | 990 return Token::ILLEGAL; |
1067 } | 991 } |
1068 ScanDecimalDigits(); | 992 ScanDecimalDigits(); |
1069 } | 993 } |
1070 | 994 |
1071 // The source character immediately following a numeric literal must | 995 // The source character immediately following a numeric literal must |
1072 // not be an identifier start or a decimal digit; see ECMA-262 | 996 // not be an identifier start or a decimal digit; see ECMA-262 |
1073 // section 7.8.3, page 17 (note that we read only one decimal digit | 997 // section 7.8.3, page 17 (note that we read only one decimal digit |
(...skipping 105 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1179 static Token::Value KeywordOrIdentifierToken(const uint8_t* input, | 1103 static Token::Value KeywordOrIdentifierToken(const uint8_t* input, |
1180 int input_length, bool escaped) { | 1104 int input_length, bool escaped) { |
1181 DCHECK(input_length >= 1); | 1105 DCHECK(input_length >= 1); |
1182 const int kMinLength = 2; | 1106 const int kMinLength = 2; |
1183 const int kMaxLength = 10; | 1107 const int kMaxLength = 10; |
1184 if (input_length < kMinLength || input_length > kMaxLength) { | 1108 if (input_length < kMinLength || input_length > kMaxLength) { |
1185 return Token::IDENTIFIER; | 1109 return Token::IDENTIFIER; |
1186 } | 1110 } |
1187 switch (input[0]) { | 1111 switch (input[0]) { |
1188 default: | 1112 default: |
1189 #define KEYWORD_GROUP_CASE(ch) \ | 1113 #define KEYWORD_GROUP_CASE(ch) \ |
1190 break; \ | 1114 break; \ |
1191 case ch: | 1115 case ch: |
1192 #define KEYWORD(keyword, token) \ | 1116 #define KEYWORD(keyword, token) \ |
1193 { \ | 1117 { \ |
1194 /* 'keyword' is a char array, so sizeof(keyword) is */ \ | 1118 /* 'keyword' is a char array, so sizeof(keyword) is */ \ |
1195 /* strlen(keyword) plus 1 for the NUL char. */ \ | 1119 /* strlen(keyword) plus 1 for the NUL char. */ \ |
1196 const int keyword_length = sizeof(keyword) - 1; \ | 1120 const int keyword_length = sizeof(keyword) - 1; \ |
1197 STATIC_ASSERT(keyword_length >= kMinLength); \ | 1121 STATIC_ASSERT(keyword_length >= kMinLength); \ |
1198 STATIC_ASSERT(keyword_length <= kMaxLength); \ | 1122 STATIC_ASSERT(keyword_length <= kMaxLength); \ |
1199 if (input_length == keyword_length && input[1] == keyword[1] && \ | 1123 if (input_length == keyword_length && input[1] == keyword[1] && \ |
1200 (keyword_length <= 2 || input[2] == keyword[2]) && \ | 1124 (keyword_length <= 2 || input[2] == keyword[2]) && \ |
1201 (keyword_length <= 3 || input[3] == keyword[3]) && \ | 1125 (keyword_length <= 3 || input[3] == keyword[3]) && \ |
1202 (keyword_length <= 4 || input[4] == keyword[4]) && \ | 1126 (keyword_length <= 4 || input[4] == keyword[4]) && \ |
1203 (keyword_length <= 5 || input[5] == keyword[5]) && \ | 1127 (keyword_length <= 5 || input[5] == keyword[5]) && \ |
1204 (keyword_length <= 6 || input[6] == keyword[6]) && \ | 1128 (keyword_length <= 6 || input[6] == keyword[6]) && \ |
1205 (keyword_length <= 7 || input[7] == keyword[7]) && \ | 1129 (keyword_length <= 7 || input[7] == keyword[7]) && \ |
1206 (keyword_length <= 8 || input[8] == keyword[8]) && \ | 1130 (keyword_length <= 8 || input[8] == keyword[8]) && \ |
1207 (keyword_length <= 9 || input[9] == keyword[9])) { \ | 1131 (keyword_length <= 9 || input[9] == keyword[9])) { \ |
1208 if (escaped) { \ | 1132 if (escaped) { \ |
1209 return token == Token::FUTURE_STRICT_RESERVED_WORD \ | 1133 return token == Token::FUTURE_STRICT_RESERVED_WORD \ |
1210 ? Token::ESCAPED_STRICT_RESERVED_WORD \ | 1134 ? Token::ESCAPED_STRICT_RESERVED_WORD \ |
1211 : Token::ESCAPED_KEYWORD; \ | 1135 : Token::ESCAPED_KEYWORD; \ |
1212 } \ | 1136 } \ |
1213 return token; \ | 1137 return token; \ |
1214 } \ | 1138 } \ |
1215 } | 1139 } |
1216 KEYWORDS(KEYWORD_GROUP_CASE, KEYWORD) | 1140 KEYWORDS(KEYWORD_GROUP_CASE, KEYWORD) |
1217 } | 1141 } |
1218 return Token::IDENTIFIER; | 1142 return Token::IDENTIFIER; |
1219 } | 1143 } |
1220 | 1144 |
1221 | 1145 |
1222 bool Scanner::IdentifierIsFutureStrictReserved( | 1146 bool Scanner::IdentifierIsFutureStrictReserved( |
1223 const AstRawString* string) const { | 1147 const AstRawString* string) const { |
1224 // Keywords are always 1-byte strings. | 1148 // Keywords are always 1-byte strings. |
1225 if (!string->is_one_byte()) return false; | 1149 if (!string->is_one_byte()) return false; |
1226 if (string->IsOneByteEqualTo("let") || string->IsOneByteEqualTo("static") || | 1150 if (string->IsOneByteEqualTo("let") || string->IsOneByteEqualTo("static") || |
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1275 if (c0_ <= kMaxAscii && c0_ != '\\') { | 1199 if (c0_ <= kMaxAscii && c0_ != '\\') { |
1276 literal.Complete(); | 1200 literal.Complete(); |
1277 return Token::IDENTIFIER; | 1201 return Token::IDENTIFIER; |
1278 } | 1202 } |
1279 | 1203 |
1280 HandleLeadSurrogate(); | 1204 HandleLeadSurrogate(); |
1281 } else if (c0_ == '\\') { | 1205 } else if (c0_ == '\\') { |
1282 // Scan identifier start character. | 1206 // Scan identifier start character. |
1283 uc32 c = ScanIdentifierUnicodeEscape(); | 1207 uc32 c = ScanIdentifierUnicodeEscape(); |
1284 // Only allow legal identifier start characters. | 1208 // Only allow legal identifier start characters. |
1285 if (c < 0 || | 1209 if (c < 0 || c == '\\' || // No recursive escapes. |
1286 c == '\\' || // No recursive escapes. | |
1287 !unicode_cache_->IsIdentifierStart(c)) { | 1210 !unicode_cache_->IsIdentifierStart(c)) { |
1288 return Token::ILLEGAL; | 1211 return Token::ILLEGAL; |
1289 } | 1212 } |
1290 AddLiteralChar(c); | 1213 AddLiteralChar(c); |
1291 return ScanIdentifierSuffix(&literal, true); | 1214 return ScanIdentifierSuffix(&literal, true); |
1292 } else { | 1215 } else { |
1293 uc32 first_char = c0_; | 1216 uc32 first_char = c0_; |
1294 Advance(); | 1217 Advance(); |
1295 AddLiteralChar(first_char); | 1218 AddLiteralChar(first_char); |
1296 } | 1219 } |
(...skipping 21 matching lines...) Expand all Loading... |
1318 | 1241 |
1319 | 1242 |
1320 Token::Value Scanner::ScanIdentifierSuffix(LiteralScope* literal, | 1243 Token::Value Scanner::ScanIdentifierSuffix(LiteralScope* literal, |
1321 bool escaped) { | 1244 bool escaped) { |
1322 // Scan the rest of the identifier characters. | 1245 // Scan the rest of the identifier characters. |
1323 while (c0_ >= 0 && unicode_cache_->IsIdentifierPart(c0_)) { | 1246 while (c0_ >= 0 && unicode_cache_->IsIdentifierPart(c0_)) { |
1324 if (c0_ == '\\') { | 1247 if (c0_ == '\\') { |
1325 uc32 c = ScanIdentifierUnicodeEscape(); | 1248 uc32 c = ScanIdentifierUnicodeEscape(); |
1326 escaped = true; | 1249 escaped = true; |
1327 // Only allow legal identifier part characters. | 1250 // Only allow legal identifier part characters. |
1328 if (c < 0 || | 1251 if (c < 0 || c == '\\' || !unicode_cache_->IsIdentifierPart(c)) { |
1329 c == '\\' || | |
1330 !unicode_cache_->IsIdentifierPart(c)) { | |
1331 return Token::ILLEGAL; | 1252 return Token::ILLEGAL; |
1332 } | 1253 } |
1333 AddLiteralChar(c); | 1254 AddLiteralChar(c); |
1334 } else { | 1255 } else { |
1335 AddLiteralChar(c0_); | 1256 AddLiteralChar(c0_); |
1336 Advance(); | 1257 Advance(); |
1337 } | 1258 } |
1338 } | 1259 } |
1339 literal->Complete(); | 1260 literal->Complete(); |
1340 | 1261 |
(...skipping 111 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1452 if (is_raw_literal_one_byte()) { | 1373 if (is_raw_literal_one_byte()) { |
1453 return ast_value_factory->GetOneByteString(raw_literal_one_byte_string()); | 1374 return ast_value_factory->GetOneByteString(raw_literal_one_byte_string()); |
1454 } | 1375 } |
1455 return ast_value_factory->GetTwoByteString(raw_literal_two_byte_string()); | 1376 return ast_value_factory->GetTwoByteString(raw_literal_two_byte_string()); |
1456 } | 1377 } |
1457 | 1378 |
1458 | 1379 |
1459 double Scanner::DoubleValue() { | 1380 double Scanner::DoubleValue() { |
1460 DCHECK(is_literal_one_byte()); | 1381 DCHECK(is_literal_one_byte()); |
1461 return StringToDouble( | 1382 return StringToDouble( |
1462 unicode_cache_, | 1383 unicode_cache_, literal_one_byte_string(), |
1463 literal_one_byte_string(), | |
1464 ALLOW_HEX | ALLOW_OCTAL | ALLOW_IMPLICIT_OCTAL | ALLOW_BINARY); | 1384 ALLOW_HEX | ALLOW_OCTAL | ALLOW_IMPLICIT_OCTAL | ALLOW_BINARY); |
1465 } | 1385 } |
1466 | 1386 |
1467 | 1387 |
1468 bool Scanner::ContainsDot() { | 1388 bool Scanner::ContainsDot() { |
1469 DCHECK(is_literal_one_byte()); | 1389 DCHECK(is_literal_one_byte()); |
1470 Vector<const uint8_t> str = literal_one_byte_string(); | 1390 Vector<const uint8_t> str = literal_one_byte_string(); |
1471 return std::find(str.begin(), str.end(), '.') != str.end(); | 1391 return std::find(str.begin(), str.end(), '.') != str.end(); |
1472 } | 1392 } |
1473 | 1393 |
(...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1533 int DuplicateFinder::AddOneByteSymbol(Vector<const uint8_t> key, int value) { | 1453 int DuplicateFinder::AddOneByteSymbol(Vector<const uint8_t> key, int value) { |
1534 return AddSymbol(key, true, value); | 1454 return AddSymbol(key, true, value); |
1535 } | 1455 } |
1536 | 1456 |
1537 | 1457 |
1538 int DuplicateFinder::AddTwoByteSymbol(Vector<const uint16_t> key, int value) { | 1458 int DuplicateFinder::AddTwoByteSymbol(Vector<const uint16_t> key, int value) { |
1539 return AddSymbol(Vector<const uint8_t>::cast(key), false, value); | 1459 return AddSymbol(Vector<const uint8_t>::cast(key), false, value); |
1540 } | 1460 } |
1541 | 1461 |
1542 | 1462 |
1543 int DuplicateFinder::AddSymbol(Vector<const uint8_t> key, | 1463 int DuplicateFinder::AddSymbol(Vector<const uint8_t> key, bool is_one_byte, |
1544 bool is_one_byte, | |
1545 int value) { | 1464 int value) { |
1546 uint32_t hash = Hash(key, is_one_byte); | 1465 uint32_t hash = Hash(key, is_one_byte); |
1547 byte* encoding = BackupKey(key, is_one_byte); | 1466 byte* encoding = BackupKey(key, is_one_byte); |
1548 HashMap::Entry* entry = map_.LookupOrInsert(encoding, hash); | 1467 HashMap::Entry* entry = map_.LookupOrInsert(encoding, hash); |
1549 int old_value = static_cast<int>(reinterpret_cast<intptr_t>(entry->value)); | 1468 int old_value = static_cast<int>(reinterpret_cast<intptr_t>(entry->value)); |
1550 entry->value = | 1469 entry->value = |
1551 reinterpret_cast<void*>(static_cast<intptr_t>(value | old_value)); | 1470 reinterpret_cast<void*>(static_cast<intptr_t>(value | old_value)); |
1552 return old_value; | 1471 return old_value; |
1553 } | 1472 } |
1554 | 1473 |
1555 | 1474 |
1556 int DuplicateFinder::AddNumber(Vector<const uint8_t> key, int value) { | 1475 int DuplicateFinder::AddNumber(Vector<const uint8_t> key, int value) { |
1557 DCHECK(key.length() > 0); | 1476 DCHECK(key.length() > 0); |
1558 // Quick check for already being in canonical form. | 1477 // Quick check for already being in canonical form. |
1559 if (IsNumberCanonical(key)) { | 1478 if (IsNumberCanonical(key)) { |
1560 return AddOneByteSymbol(key, value); | 1479 return AddOneByteSymbol(key, value); |
1561 } | 1480 } |
1562 | 1481 |
1563 int flags = ALLOW_HEX | ALLOW_OCTAL | ALLOW_IMPLICIT_OCTAL | ALLOW_BINARY; | 1482 int flags = ALLOW_HEX | ALLOW_OCTAL | ALLOW_IMPLICIT_OCTAL | ALLOW_BINARY; |
1564 double double_value = StringToDouble( | 1483 double double_value = StringToDouble(unicode_constants_, key, flags, 0.0); |
1565 unicode_constants_, key, flags, 0.0); | |
1566 int length; | 1484 int length; |
1567 const char* string; | 1485 const char* string; |
1568 if (!std::isfinite(double_value)) { | 1486 if (!std::isfinite(double_value)) { |
1569 string = "Infinity"; | 1487 string = "Infinity"; |
1570 length = 8; // strlen("Infinity"); | 1488 length = 8; // strlen("Infinity"); |
1571 } else { | 1489 } else { |
1572 string = DoubleToCString(double_value, | 1490 string = DoubleToCString(double_value, |
1573 Vector<char>(number_buffer_, kBufferSize)); | 1491 Vector<char>(number_buffer_, kBufferSize)); |
1574 length = StrLength(string); | 1492 length = StrLength(string); |
1575 } | 1493 } |
1576 return AddSymbol(Vector<const byte>(reinterpret_cast<const byte*>(string), | 1494 return AddSymbol( |
1577 length), true, value); | 1495 Vector<const byte>(reinterpret_cast<const byte*>(string), length), true, |
| 1496 value); |
1578 } | 1497 } |
1579 | 1498 |
1580 | 1499 |
1581 bool DuplicateFinder::IsNumberCanonical(Vector<const uint8_t> number) { | 1500 bool DuplicateFinder::IsNumberCanonical(Vector<const uint8_t> number) { |
1582 // Test for a safe approximation of number literals that are already | 1501 // Test for a safe approximation of number literals that are already |
1583 // in canonical form: max 15 digits, no leading zeroes, except an | 1502 // in canonical form: max 15 digits, no leading zeroes, except an |
1584 // integer part that is a single zero, and no trailing zeros below | 1503 // integer part that is a single zero, and no trailing zeros below |
1585 // the decimal point. | 1504 // the decimal point. |
1586 int pos = 0; | 1505 int pos = 0; |
1587 int length = number.length(); | 1506 int length = number.length(); |
1588 if (number.length() > 15) return false; | 1507 if (number.length() > 15) return false; |
1589 if (number[pos] == '0') { | 1508 if (number[pos] == '0') { |
1590 pos++; | 1509 pos++; |
1591 } else { | 1510 } else { |
1592 while (pos < length && | 1511 while (pos < length && |
1593 static_cast<unsigned>(number[pos] - '0') <= ('9' - '0')) pos++; | 1512 static_cast<unsigned>(number[pos] - '0') <= ('9' - '0')) |
| 1513 pos++; |
1594 } | 1514 } |
1595 if (length == pos) return true; | 1515 if (length == pos) return true; |
1596 if (number[pos] != '.') return false; | 1516 if (number[pos] != '.') return false; |
1597 pos++; | 1517 pos++; |
1598 bool invalid_last_digit = true; | 1518 bool invalid_last_digit = true; |
1599 while (pos < length) { | 1519 while (pos < length) { |
1600 uint8_t digit = number[pos] - '0'; | 1520 uint8_t digit = number[pos] - '0'; |
1601 if (digit > '9' - '0') return false; | 1521 if (digit > '9' - '0') return false; |
1602 invalid_last_digit = (digit == 0); | 1522 invalid_last_digit = (digit == 0); |
1603 pos++; | 1523 pos++; |
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1651 if (one_byte_length >= (1 << 7)) { | 1571 if (one_byte_length >= (1 << 7)) { |
1652 if (one_byte_length >= (1 << 14)) { | 1572 if (one_byte_length >= (1 << 14)) { |
1653 if (one_byte_length >= (1 << 21)) { | 1573 if (one_byte_length >= (1 << 21)) { |
1654 if (one_byte_length >= (1 << 28)) { | 1574 if (one_byte_length >= (1 << 28)) { |
1655 backing_store_.Add( | 1575 backing_store_.Add( |
1656 static_cast<uint8_t>((one_byte_length >> 28) | 0x80)); | 1576 static_cast<uint8_t>((one_byte_length >> 28) | 0x80)); |
1657 } | 1577 } |
1658 backing_store_.Add( | 1578 backing_store_.Add( |
1659 static_cast<uint8_t>((one_byte_length >> 21) | 0x80u)); | 1579 static_cast<uint8_t>((one_byte_length >> 21) | 0x80u)); |
1660 } | 1580 } |
1661 backing_store_.Add( | 1581 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 14) | 0x80u)); |
1662 static_cast<uint8_t>((one_byte_length >> 14) | 0x80u)); | |
1663 } | 1582 } |
1664 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); | 1583 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); |
1665 } | 1584 } |
1666 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); | 1585 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); |
1667 | 1586 |
1668 backing_store_.AddBlock(bytes); | 1587 backing_store_.AddBlock(bytes); |
1669 return backing_store_.EndSequence().start(); | 1588 return backing_store_.EndSequence().start(); |
1670 } | 1589 } |
1671 | 1590 |
1672 } // namespace internal | 1591 } // namespace internal |
1673 } // namespace v8 | 1592 } // namespace v8 |
OLD | NEW |