| OLD | NEW |
| 1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. |
| 6 | 6 |
| 7 #include "src/parsing/scanner.h" | 7 #include "src/parsing/scanner.h" |
| 8 | 8 |
| 9 #include <stdint.h> | 9 #include <stdint.h> |
| 10 | 10 |
| (...skipping 82 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 93 } | 93 } |
| 94 return x; | 94 return x; |
| 95 } | 95 } |
| 96 | 96 |
| 97 | 97 |
| 98 // Ensure that tokens can be stored in a byte. | 98 // Ensure that tokens can be stored in a byte. |
| 99 STATIC_ASSERT(Token::NUM_TOKENS <= 0x100); | 99 STATIC_ASSERT(Token::NUM_TOKENS <= 0x100); |
| 100 | 100 |
| 101 // Table of one-character tokens, by character (0x00..0x7f only). | 101 // Table of one-character tokens, by character (0x00..0x7f only). |
| 102 static const byte one_char_tokens[] = { | 102 static const byte one_char_tokens[] = { |
| 103 Token::ILLEGAL, | 103 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 104 Token::ILLEGAL, | 104 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 105 Token::ILLEGAL, | 105 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 106 Token::ILLEGAL, | 106 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 107 Token::ILLEGAL, | 107 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 108 Token::ILLEGAL, | 108 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 109 Token::ILLEGAL, | 109 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 110 Token::ILLEGAL, | 110 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 111 Token::ILLEGAL, | 111 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 112 Token::ILLEGAL, | 112 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 113 Token::ILLEGAL, | 113 Token::LPAREN, // 0x28 |
| 114 Token::ILLEGAL, | 114 Token::RPAREN, // 0x29 |
| 115 Token::ILLEGAL, | 115 Token::ILLEGAL, Token::ILLEGAL, |
| 116 Token::ILLEGAL, | 116 Token::COMMA, // 0x2c |
| 117 Token::ILLEGAL, | 117 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 118 Token::ILLEGAL, | 118 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 119 Token::ILLEGAL, | 119 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 120 Token::ILLEGAL, | 120 Token::ILLEGAL, |
| 121 Token::ILLEGAL, | 121 Token::COLON, // 0x3a |
| 122 Token::ILLEGAL, | 122 Token::SEMICOLON, // 0x3b |
| 123 Token::ILLEGAL, | 123 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 124 Token::ILLEGAL, | 124 Token::CONDITIONAL, // 0x3f |
| 125 Token::ILLEGAL, | 125 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 126 Token::ILLEGAL, | 126 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 127 Token::ILLEGAL, | 127 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 128 Token::ILLEGAL, | 128 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 129 Token::ILLEGAL, | 129 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 130 Token::ILLEGAL, | 130 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 131 Token::ILLEGAL, | 131 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 132 Token::ILLEGAL, | 132 Token::LBRACK, // 0x5b |
| 133 Token::ILLEGAL, | 133 Token::ILLEGAL, |
| 134 Token::ILLEGAL, | 134 Token::RBRACK, // 0x5d |
| 135 Token::ILLEGAL, | 135 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 136 Token::ILLEGAL, | 136 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 137 Token::ILLEGAL, | 137 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 138 Token::ILLEGAL, | 138 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 139 Token::ILLEGAL, | 139 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 140 Token::ILLEGAL, | 140 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 141 Token::ILLEGAL, | 141 Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, Token::ILLEGAL, |
| 142 Token::ILLEGAL, | 142 Token::ILLEGAL, |
| 143 Token::LPAREN, // 0x28 | 143 Token::LBRACE, // 0x7b |
| 144 Token::RPAREN, // 0x29 | 144 Token::ILLEGAL, |
| 145 Token::ILLEGAL, | 145 Token::RBRACE, // 0x7d |
| 146 Token::ILLEGAL, | 146 Token::BIT_NOT, // 0x7e |
| 147 Token::COMMA, // 0x2c | 147 Token::ILLEGAL}; |
| 148 Token::ILLEGAL, | |
| 149 Token::ILLEGAL, | |
| 150 Token::ILLEGAL, | |
| 151 Token::ILLEGAL, | |
| 152 Token::ILLEGAL, | |
| 153 Token::ILLEGAL, | |
| 154 Token::ILLEGAL, | |
| 155 Token::ILLEGAL, | |
| 156 Token::ILLEGAL, | |
| 157 Token::ILLEGAL, | |
| 158 Token::ILLEGAL, | |
| 159 Token::ILLEGAL, | |
| 160 Token::ILLEGAL, | |
| 161 Token::COLON, // 0x3a | |
| 162 Token::SEMICOLON, // 0x3b | |
| 163 Token::ILLEGAL, | |
| 164 Token::ILLEGAL, | |
| 165 Token::ILLEGAL, | |
| 166 Token::CONDITIONAL, // 0x3f | |
| 167 Token::ILLEGAL, | |
| 168 Token::ILLEGAL, | |
| 169 Token::ILLEGAL, | |
| 170 Token::ILLEGAL, | |
| 171 Token::ILLEGAL, | |
| 172 Token::ILLEGAL, | |
| 173 Token::ILLEGAL, | |
| 174 Token::ILLEGAL, | |
| 175 Token::ILLEGAL, | |
| 176 Token::ILLEGAL, | |
| 177 Token::ILLEGAL, | |
| 178 Token::ILLEGAL, | |
| 179 Token::ILLEGAL, | |
| 180 Token::ILLEGAL, | |
| 181 Token::ILLEGAL, | |
| 182 Token::ILLEGAL, | |
| 183 Token::ILLEGAL, | |
| 184 Token::ILLEGAL, | |
| 185 Token::ILLEGAL, | |
| 186 Token::ILLEGAL, | |
| 187 Token::ILLEGAL, | |
| 188 Token::ILLEGAL, | |
| 189 Token::ILLEGAL, | |
| 190 Token::ILLEGAL, | |
| 191 Token::ILLEGAL, | |
| 192 Token::ILLEGAL, | |
| 193 Token::ILLEGAL, | |
| 194 Token::LBRACK, // 0x5b | |
| 195 Token::ILLEGAL, | |
| 196 Token::RBRACK, // 0x5d | |
| 197 Token::ILLEGAL, | |
| 198 Token::ILLEGAL, | |
| 199 Token::ILLEGAL, | |
| 200 Token::ILLEGAL, | |
| 201 Token::ILLEGAL, | |
| 202 Token::ILLEGAL, | |
| 203 Token::ILLEGAL, | |
| 204 Token::ILLEGAL, | |
| 205 Token::ILLEGAL, | |
| 206 Token::ILLEGAL, | |
| 207 Token::ILLEGAL, | |
| 208 Token::ILLEGAL, | |
| 209 Token::ILLEGAL, | |
| 210 Token::ILLEGAL, | |
| 211 Token::ILLEGAL, | |
| 212 Token::ILLEGAL, | |
| 213 Token::ILLEGAL, | |
| 214 Token::ILLEGAL, | |
| 215 Token::ILLEGAL, | |
| 216 Token::ILLEGAL, | |
| 217 Token::ILLEGAL, | |
| 218 Token::ILLEGAL, | |
| 219 Token::ILLEGAL, | |
| 220 Token::ILLEGAL, | |
| 221 Token::ILLEGAL, | |
| 222 Token::ILLEGAL, | |
| 223 Token::ILLEGAL, | |
| 224 Token::ILLEGAL, | |
| 225 Token::ILLEGAL, | |
| 226 Token::LBRACE, // 0x7b | |
| 227 Token::ILLEGAL, | |
| 228 Token::RBRACE, // 0x7d | |
| 229 Token::BIT_NOT, // 0x7e | |
| 230 Token::ILLEGAL | |
| 231 }; | |
| 232 | 148 |
| 233 | 149 |
| 234 Token::Value Scanner::Next() { | 150 Token::Value Scanner::Next() { |
| 235 if (next_.token == Token::EOS) { | 151 if (next_.token == Token::EOS) { |
| 236 next_.location.beg_pos = current_.location.beg_pos; | 152 next_.location.beg_pos = current_.location.beg_pos; |
| 237 next_.location.end_pos = current_.location.end_pos; | 153 next_.location.end_pos = current_.location.end_pos; |
| 238 } | 154 } |
| 239 current_ = next_; | 155 current_ = next_; |
| 240 if (V8_UNLIKELY(next_next_.token != Token::UNINITIALIZED)) { | 156 if (V8_UNLIKELY(next_next_.token != Token::UNINITIALIZED)) { |
| 241 next_ = next_next_; | 157 next_ = next_next_; |
| (...skipping 127 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 369 if (!name.is_one_byte()) return; | 285 if (!name.is_one_byte()) return; |
| 370 Vector<const uint8_t> name_literal = name.one_byte_literal(); | 286 Vector<const uint8_t> name_literal = name.one_byte_literal(); |
| 371 LiteralBuffer* value; | 287 LiteralBuffer* value; |
| 372 if (name_literal == STATIC_CHAR_VECTOR("sourceURL")) { | 288 if (name_literal == STATIC_CHAR_VECTOR("sourceURL")) { |
| 373 value = &source_url_; | 289 value = &source_url_; |
| 374 } else if (name_literal == STATIC_CHAR_VECTOR("sourceMappingURL")) { | 290 } else if (name_literal == STATIC_CHAR_VECTOR("sourceMappingURL")) { |
| 375 value = &source_mapping_url_; | 291 value = &source_mapping_url_; |
| 376 } else { | 292 } else { |
| 377 return; | 293 return; |
| 378 } | 294 } |
| 379 if (c0_ != '=') | 295 if (c0_ != '=') return; |
| 380 return; | |
| 381 Advance(); | 296 Advance(); |
| 382 value->Reset(); | 297 value->Reset(); |
| 383 while (c0_ >= 0 && unicode_cache_->IsWhiteSpace(c0_)) { | 298 while (c0_ >= 0 && unicode_cache_->IsWhiteSpace(c0_)) { |
| 384 Advance(); | 299 Advance(); |
| 385 } | 300 } |
| 386 while (c0_ >= 0 && !unicode_cache_->IsLineTerminator(c0_)) { | 301 while (c0_ >= 0 && !unicode_cache_->IsLineTerminator(c0_)) { |
| 387 // Disallowed characters. | 302 // Disallowed characters. |
| 388 if (c0_ == '"' || c0_ == '\'') { | 303 if (c0_ == '"' || c0_ == '\'') { |
| 389 value->Reset(); | 304 value->Reset(); |
| 390 return; | 305 return; |
| (...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 461 Advance(); | 376 Advance(); |
| 462 token = Token::WHITESPACE; | 377 token = Token::WHITESPACE; |
| 463 break; | 378 break; |
| 464 | 379 |
| 465 case '\n': | 380 case '\n': |
| 466 Advance(); | 381 Advance(); |
| 467 has_line_terminator_before_next_ = true; | 382 has_line_terminator_before_next_ = true; |
| 468 token = Token::WHITESPACE; | 383 token = Token::WHITESPACE; |
| 469 break; | 384 break; |
| 470 | 385 |
| 471 case '"': case '\'': | 386 case '"': |
| 387 case '\'': |
| 472 token = ScanString(); | 388 token = ScanString(); |
| 473 break; | 389 break; |
| 474 | 390 |
| 475 case '<': | 391 case '<': |
| 476 // < <= << <<= <!-- | 392 // < <= << <<= <!-- |
| 477 Advance(); | 393 Advance(); |
| 478 if (c0_ == '=') { | 394 if (c0_ == '=') { |
| 479 token = Select(Token::LTE); | 395 token = Select(Token::LTE); |
| 480 } else if (c0_ == '<') { | 396 } else if (c0_ == '<') { |
| 481 token = Select('=', Token::ASSIGN_SHL, Token::SHL); | 397 token = Select('=', Token::ASSIGN_SHL, Token::SHL); |
| (...skipping 258 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 740 if (!in_template_literal && c0_ >= 0 && unicode_cache_->IsLineTerminator(c)) { | 656 if (!in_template_literal && c0_ >= 0 && unicode_cache_->IsLineTerminator(c)) { |
| 741 // Allow CR+LF newlines in multiline string literals. | 657 // Allow CR+LF newlines in multiline string literals. |
| 742 if (IsCarriageReturn(c) && IsLineFeed(c0_)) Advance<capture_raw>(); | 658 if (IsCarriageReturn(c) && IsLineFeed(c0_)) Advance<capture_raw>(); |
| 743 // Allow LF+CR newlines in multiline string literals. | 659 // Allow LF+CR newlines in multiline string literals. |
| 744 if (IsLineFeed(c) && IsCarriageReturn(c0_)) Advance<capture_raw>(); | 660 if (IsLineFeed(c) && IsCarriageReturn(c0_)) Advance<capture_raw>(); |
| 745 return true; | 661 return true; |
| 746 } | 662 } |
| 747 | 663 |
| 748 switch (c) { | 664 switch (c) { |
| 749 case '\'': // fall through | 665 case '\'': // fall through |
| 750 case '"' : // fall through | 666 case '"': // fall through |
| 751 case '\\': break; | 667 case '\\': |
| 752 case 'b' : c = '\b'; break; | 668 break; |
| 753 case 'f' : c = '\f'; break; | 669 case 'b': |
| 754 case 'n' : c = '\n'; break; | 670 c = '\b'; |
| 755 case 'r' : c = '\r'; break; | 671 break; |
| 756 case 't' : c = '\t'; break; | 672 case 'f': |
| 757 case 'u' : { | 673 c = '\f'; |
| 674 break; |
| 675 case 'n': |
| 676 c = '\n'; |
| 677 break; |
| 678 case 'r': |
| 679 c = '\r'; |
| 680 break; |
| 681 case 't': |
| 682 c = '\t'; |
| 683 break; |
| 684 case 'u': { |
| 758 c = ScanUnicodeEscape<capture_raw>(); | 685 c = ScanUnicodeEscape<capture_raw>(); |
| 759 if (c < 0) return false; | 686 if (c < 0) return false; |
| 760 break; | 687 break; |
| 761 } | 688 } |
| 762 case 'v': | 689 case 'v': |
| 763 c = '\v'; | 690 c = '\v'; |
| 764 break; | 691 break; |
| 765 case 'x': { | 692 case 'x': { |
| 766 c = ScanHexNumber<capture_raw>(2); | 693 c = ScanHexNumber<capture_raw>(2); |
| 767 if (c < 0) return false; | 694 if (c < 0) return false; |
| (...skipping 63 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 831 literal.Complete(); | 758 literal.Complete(); |
| 832 Advance<false, false>(); | 759 Advance<false, false>(); |
| 833 return Token::STRING; | 760 return Token::STRING; |
| 834 } | 761 } |
| 835 uc32 c = c0_; | 762 uc32 c = c0_; |
| 836 if (c == '\\') break; | 763 if (c == '\\') break; |
| 837 Advance<false, false>(); | 764 Advance<false, false>(); |
| 838 AddLiteralChar(c); | 765 AddLiteralChar(c); |
| 839 } | 766 } |
| 840 | 767 |
| 841 while (c0_ != quote && c0_ >= 0 | 768 while (c0_ != quote && c0_ >= 0 && !unicode_cache_->IsLineTerminator(c0_)) { |
| 842 && !unicode_cache_->IsLineTerminator(c0_)) { | |
| 843 uc32 c = c0_; | 769 uc32 c = c0_; |
| 844 Advance(); | 770 Advance(); |
| 845 if (c == '\\') { | 771 if (c == '\\') { |
| 846 if (c0_ < 0 || !ScanEscape<false, false>()) return Token::ILLEGAL; | 772 if (c0_ < 0 || !ScanEscape<false, false>()) return Token::ILLEGAL; |
| 847 } else { | 773 } else { |
| 848 AddLiteralChar(c); | 774 AddLiteralChar(c); |
| 849 } | 775 } |
| 850 } | 776 } |
| 851 if (c0_ != quote) return Token::ILLEGAL; | 777 if (c0_ != quote) return Token::ILLEGAL; |
| 852 literal.Complete(); | 778 literal.Complete(); |
| (...skipping 87 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 940 | 866 |
| 941 | 867 |
| 942 Token::Value Scanner::ScanTemplateContinuation() { | 868 Token::Value Scanner::ScanTemplateContinuation() { |
| 943 DCHECK_EQ(next_.token, Token::RBRACE); | 869 DCHECK_EQ(next_.token, Token::RBRACE); |
| 944 next_.location.beg_pos = source_pos() - 1; // We already consumed } | 870 next_.location.beg_pos = source_pos() - 1; // We already consumed } |
| 945 return ScanTemplateSpan(); | 871 return ScanTemplateSpan(); |
| 946 } | 872 } |
| 947 | 873 |
| 948 | 874 |
| 949 void Scanner::ScanDecimalDigits() { | 875 void Scanner::ScanDecimalDigits() { |
| 950 while (IsDecimalDigit(c0_)) | 876 while (IsDecimalDigit(c0_)) AddLiteralCharAdvance(); |
| 951 AddLiteralCharAdvance(); | |
| 952 } | 877 } |
| 953 | 878 |
| 954 | 879 |
| 955 Token::Value Scanner::ScanNumber(bool seen_period) { | 880 Token::Value Scanner::ScanNumber(bool seen_period) { |
| 956 DCHECK(IsDecimalDigit(c0_)); // the first digit of the number or the fraction | 881 DCHECK(IsDecimalDigit(c0_)); // the first digit of the number or the fraction |
| 957 | 882 |
| 958 enum { DECIMAL, HEX, OCTAL, IMPLICIT_OCTAL, BINARY } kind = DECIMAL; | 883 enum { DECIMAL, HEX, OCTAL, IMPLICIT_OCTAL, BINARY } kind = DECIMAL; |
| 959 | 884 |
| 960 LiteralScope literal(this); | 885 LiteralScope literal(this); |
| 961 bool at_start = !seen_period; | 886 bool at_start = !seen_period; |
| (...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1005 } | 930 } |
| 1006 } else if ('0' <= c0_ && c0_ <= '7') { | 931 } else if ('0' <= c0_ && c0_ <= '7') { |
| 1007 // (possible) octal number | 932 // (possible) octal number |
| 1008 kind = IMPLICIT_OCTAL; | 933 kind = IMPLICIT_OCTAL; |
| 1009 while (true) { | 934 while (true) { |
| 1010 if (c0_ == '8' || c0_ == '9') { | 935 if (c0_ == '8' || c0_ == '9') { |
| 1011 at_start = false; | 936 at_start = false; |
| 1012 kind = DECIMAL; | 937 kind = DECIMAL; |
| 1013 break; | 938 break; |
| 1014 } | 939 } |
| 1015 if (c0_ < '0' || '7' < c0_) { | 940 if (c0_ < '0' || '7' < c0_) { |
| 1016 // Octal literal finished. | 941 // Octal literal finished. |
| 1017 octal_pos_ = Location(start_pos, source_pos()); | 942 octal_pos_ = Location(start_pos, source_pos()); |
| 1018 break; | 943 break; |
| 1019 } | 944 } |
| 1020 AddLiteralCharAdvance(); | 945 AddLiteralCharAdvance(); |
| 1021 } | 946 } |
| 1022 } | 947 } |
| 1023 } | 948 } |
| 1024 | 949 |
| 1025 // Parse decimal digits and allow trailing fractional part. | 950 // Parse decimal digits and allow trailing fractional part. |
| (...skipping 26 matching lines...) Expand all Loading... |
| 1052 } | 977 } |
| 1053 } | 978 } |
| 1054 } | 979 } |
| 1055 | 980 |
| 1056 // scan exponent, if any | 981 // scan exponent, if any |
| 1057 if (c0_ == 'e' || c0_ == 'E') { | 982 if (c0_ == 'e' || c0_ == 'E') { |
| 1058 DCHECK(kind != HEX); // 'e'/'E' must be scanned as part of the hex number | 983 DCHECK(kind != HEX); // 'e'/'E' must be scanned as part of the hex number |
| 1059 if (kind != DECIMAL) return Token::ILLEGAL; | 984 if (kind != DECIMAL) return Token::ILLEGAL; |
| 1060 // scan exponent | 985 // scan exponent |
| 1061 AddLiteralCharAdvance(); | 986 AddLiteralCharAdvance(); |
| 1062 if (c0_ == '+' || c0_ == '-') | 987 if (c0_ == '+' || c0_ == '-') AddLiteralCharAdvance(); |
| 1063 AddLiteralCharAdvance(); | |
| 1064 if (!IsDecimalDigit(c0_)) { | 988 if (!IsDecimalDigit(c0_)) { |
| 1065 // we must have at least one decimal digit after 'e'/'E' | 989 // we must have at least one decimal digit after 'e'/'E' |
| 1066 return Token::ILLEGAL; | 990 return Token::ILLEGAL; |
| 1067 } | 991 } |
| 1068 ScanDecimalDigits(); | 992 ScanDecimalDigits(); |
| 1069 } | 993 } |
| 1070 | 994 |
| 1071 // The source character immediately following a numeric literal must | 995 // The source character immediately following a numeric literal must |
| 1072 // not be an identifier start or a decimal digit; see ECMA-262 | 996 // not be an identifier start or a decimal digit; see ECMA-262 |
| 1073 // section 7.8.3, page 17 (note that we read only one decimal digit | 997 // section 7.8.3, page 17 (note that we read only one decimal digit |
| (...skipping 105 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1179 static Token::Value KeywordOrIdentifierToken(const uint8_t* input, | 1103 static Token::Value KeywordOrIdentifierToken(const uint8_t* input, |
| 1180 int input_length, bool escaped) { | 1104 int input_length, bool escaped) { |
| 1181 DCHECK(input_length >= 1); | 1105 DCHECK(input_length >= 1); |
| 1182 const int kMinLength = 2; | 1106 const int kMinLength = 2; |
| 1183 const int kMaxLength = 10; | 1107 const int kMaxLength = 10; |
| 1184 if (input_length < kMinLength || input_length > kMaxLength) { | 1108 if (input_length < kMinLength || input_length > kMaxLength) { |
| 1185 return Token::IDENTIFIER; | 1109 return Token::IDENTIFIER; |
| 1186 } | 1110 } |
| 1187 switch (input[0]) { | 1111 switch (input[0]) { |
| 1188 default: | 1112 default: |
| 1189 #define KEYWORD_GROUP_CASE(ch) \ | 1113 #define KEYWORD_GROUP_CASE(ch) \ |
| 1190 break; \ | 1114 break; \ |
| 1191 case ch: | 1115 case ch: |
| 1192 #define KEYWORD(keyword, token) \ | 1116 #define KEYWORD(keyword, token) \ |
| 1193 { \ | 1117 { \ |
| 1194 /* 'keyword' is a char array, so sizeof(keyword) is */ \ | 1118 /* 'keyword' is a char array, so sizeof(keyword) is */ \ |
| 1195 /* strlen(keyword) plus 1 for the NUL char. */ \ | 1119 /* strlen(keyword) plus 1 for the NUL char. */ \ |
| 1196 const int keyword_length = sizeof(keyword) - 1; \ | 1120 const int keyword_length = sizeof(keyword) - 1; \ |
| 1197 STATIC_ASSERT(keyword_length >= kMinLength); \ | 1121 STATIC_ASSERT(keyword_length >= kMinLength); \ |
| 1198 STATIC_ASSERT(keyword_length <= kMaxLength); \ | 1122 STATIC_ASSERT(keyword_length <= kMaxLength); \ |
| 1199 if (input_length == keyword_length && input[1] == keyword[1] && \ | 1123 if (input_length == keyword_length && input[1] == keyword[1] && \ |
| 1200 (keyword_length <= 2 || input[2] == keyword[2]) && \ | 1124 (keyword_length <= 2 || input[2] == keyword[2]) && \ |
| 1201 (keyword_length <= 3 || input[3] == keyword[3]) && \ | 1125 (keyword_length <= 3 || input[3] == keyword[3]) && \ |
| 1202 (keyword_length <= 4 || input[4] == keyword[4]) && \ | 1126 (keyword_length <= 4 || input[4] == keyword[4]) && \ |
| 1203 (keyword_length <= 5 || input[5] == keyword[5]) && \ | 1127 (keyword_length <= 5 || input[5] == keyword[5]) && \ |
| 1204 (keyword_length <= 6 || input[6] == keyword[6]) && \ | 1128 (keyword_length <= 6 || input[6] == keyword[6]) && \ |
| 1205 (keyword_length <= 7 || input[7] == keyword[7]) && \ | 1129 (keyword_length <= 7 || input[7] == keyword[7]) && \ |
| 1206 (keyword_length <= 8 || input[8] == keyword[8]) && \ | 1130 (keyword_length <= 8 || input[8] == keyword[8]) && \ |
| 1207 (keyword_length <= 9 || input[9] == keyword[9])) { \ | 1131 (keyword_length <= 9 || input[9] == keyword[9])) { \ |
| 1208 if (escaped) { \ | 1132 if (escaped) { \ |
| 1209 return token == Token::FUTURE_STRICT_RESERVED_WORD \ | 1133 return token == Token::FUTURE_STRICT_RESERVED_WORD \ |
| 1210 ? Token::ESCAPED_STRICT_RESERVED_WORD \ | 1134 ? Token::ESCAPED_STRICT_RESERVED_WORD \ |
| 1211 : Token::ESCAPED_KEYWORD; \ | 1135 : Token::ESCAPED_KEYWORD; \ |
| 1212 } \ | 1136 } \ |
| 1213 return token; \ | 1137 return token; \ |
| 1214 } \ | 1138 } \ |
| 1215 } | 1139 } |
| 1216 KEYWORDS(KEYWORD_GROUP_CASE, KEYWORD) | 1140 KEYWORDS(KEYWORD_GROUP_CASE, KEYWORD) |
| 1217 } | 1141 } |
| 1218 return Token::IDENTIFIER; | 1142 return Token::IDENTIFIER; |
| 1219 } | 1143 } |
| 1220 | 1144 |
| 1221 | 1145 |
| 1222 bool Scanner::IdentifierIsFutureStrictReserved( | 1146 bool Scanner::IdentifierIsFutureStrictReserved( |
| 1223 const AstRawString* string) const { | 1147 const AstRawString* string) const { |
| 1224 // Keywords are always 1-byte strings. | 1148 // Keywords are always 1-byte strings. |
| 1225 if (!string->is_one_byte()) return false; | 1149 if (!string->is_one_byte()) return false; |
| 1226 if (string->IsOneByteEqualTo("let") || string->IsOneByteEqualTo("static") || | 1150 if (string->IsOneByteEqualTo("let") || string->IsOneByteEqualTo("static") || |
| (...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1275 if (c0_ <= kMaxAscii && c0_ != '\\') { | 1199 if (c0_ <= kMaxAscii && c0_ != '\\') { |
| 1276 literal.Complete(); | 1200 literal.Complete(); |
| 1277 return Token::IDENTIFIER; | 1201 return Token::IDENTIFIER; |
| 1278 } | 1202 } |
| 1279 | 1203 |
| 1280 HandleLeadSurrogate(); | 1204 HandleLeadSurrogate(); |
| 1281 } else if (c0_ == '\\') { | 1205 } else if (c0_ == '\\') { |
| 1282 // Scan identifier start character. | 1206 // Scan identifier start character. |
| 1283 uc32 c = ScanIdentifierUnicodeEscape(); | 1207 uc32 c = ScanIdentifierUnicodeEscape(); |
| 1284 // Only allow legal identifier start characters. | 1208 // Only allow legal identifier start characters. |
| 1285 if (c < 0 || | 1209 if (c < 0 || c == '\\' || // No recursive escapes. |
| 1286 c == '\\' || // No recursive escapes. | |
| 1287 !unicode_cache_->IsIdentifierStart(c)) { | 1210 !unicode_cache_->IsIdentifierStart(c)) { |
| 1288 return Token::ILLEGAL; | 1211 return Token::ILLEGAL; |
| 1289 } | 1212 } |
| 1290 AddLiteralChar(c); | 1213 AddLiteralChar(c); |
| 1291 return ScanIdentifierSuffix(&literal, true); | 1214 return ScanIdentifierSuffix(&literal, true); |
| 1292 } else { | 1215 } else { |
| 1293 uc32 first_char = c0_; | 1216 uc32 first_char = c0_; |
| 1294 Advance(); | 1217 Advance(); |
| 1295 AddLiteralChar(first_char); | 1218 AddLiteralChar(first_char); |
| 1296 } | 1219 } |
| (...skipping 21 matching lines...) Expand all Loading... |
| 1318 | 1241 |
| 1319 | 1242 |
| 1320 Token::Value Scanner::ScanIdentifierSuffix(LiteralScope* literal, | 1243 Token::Value Scanner::ScanIdentifierSuffix(LiteralScope* literal, |
| 1321 bool escaped) { | 1244 bool escaped) { |
| 1322 // Scan the rest of the identifier characters. | 1245 // Scan the rest of the identifier characters. |
| 1323 while (c0_ >= 0 && unicode_cache_->IsIdentifierPart(c0_)) { | 1246 while (c0_ >= 0 && unicode_cache_->IsIdentifierPart(c0_)) { |
| 1324 if (c0_ == '\\') { | 1247 if (c0_ == '\\') { |
| 1325 uc32 c = ScanIdentifierUnicodeEscape(); | 1248 uc32 c = ScanIdentifierUnicodeEscape(); |
| 1326 escaped = true; | 1249 escaped = true; |
| 1327 // Only allow legal identifier part characters. | 1250 // Only allow legal identifier part characters. |
| 1328 if (c < 0 || | 1251 if (c < 0 || c == '\\' || !unicode_cache_->IsIdentifierPart(c)) { |
| 1329 c == '\\' || | |
| 1330 !unicode_cache_->IsIdentifierPart(c)) { | |
| 1331 return Token::ILLEGAL; | 1252 return Token::ILLEGAL; |
| 1332 } | 1253 } |
| 1333 AddLiteralChar(c); | 1254 AddLiteralChar(c); |
| 1334 } else { | 1255 } else { |
| 1335 AddLiteralChar(c0_); | 1256 AddLiteralChar(c0_); |
| 1336 Advance(); | 1257 Advance(); |
| 1337 } | 1258 } |
| 1338 } | 1259 } |
| 1339 literal->Complete(); | 1260 literal->Complete(); |
| 1340 | 1261 |
| (...skipping 111 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1452 if (is_raw_literal_one_byte()) { | 1373 if (is_raw_literal_one_byte()) { |
| 1453 return ast_value_factory->GetOneByteString(raw_literal_one_byte_string()); | 1374 return ast_value_factory->GetOneByteString(raw_literal_one_byte_string()); |
| 1454 } | 1375 } |
| 1455 return ast_value_factory->GetTwoByteString(raw_literal_two_byte_string()); | 1376 return ast_value_factory->GetTwoByteString(raw_literal_two_byte_string()); |
| 1456 } | 1377 } |
| 1457 | 1378 |
| 1458 | 1379 |
| 1459 double Scanner::DoubleValue() { | 1380 double Scanner::DoubleValue() { |
| 1460 DCHECK(is_literal_one_byte()); | 1381 DCHECK(is_literal_one_byte()); |
| 1461 return StringToDouble( | 1382 return StringToDouble( |
| 1462 unicode_cache_, | 1383 unicode_cache_, literal_one_byte_string(), |
| 1463 literal_one_byte_string(), | |
| 1464 ALLOW_HEX | ALLOW_OCTAL | ALLOW_IMPLICIT_OCTAL | ALLOW_BINARY); | 1384 ALLOW_HEX | ALLOW_OCTAL | ALLOW_IMPLICIT_OCTAL | ALLOW_BINARY); |
| 1465 } | 1385 } |
| 1466 | 1386 |
| 1467 | 1387 |
| 1468 bool Scanner::ContainsDot() { | 1388 bool Scanner::ContainsDot() { |
| 1469 DCHECK(is_literal_one_byte()); | 1389 DCHECK(is_literal_one_byte()); |
| 1470 Vector<const uint8_t> str = literal_one_byte_string(); | 1390 Vector<const uint8_t> str = literal_one_byte_string(); |
| 1471 return std::find(str.begin(), str.end(), '.') != str.end(); | 1391 return std::find(str.begin(), str.end(), '.') != str.end(); |
| 1472 } | 1392 } |
| 1473 | 1393 |
| (...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1533 int DuplicateFinder::AddOneByteSymbol(Vector<const uint8_t> key, int value) { | 1453 int DuplicateFinder::AddOneByteSymbol(Vector<const uint8_t> key, int value) { |
| 1534 return AddSymbol(key, true, value); | 1454 return AddSymbol(key, true, value); |
| 1535 } | 1455 } |
| 1536 | 1456 |
| 1537 | 1457 |
| 1538 int DuplicateFinder::AddTwoByteSymbol(Vector<const uint16_t> key, int value) { | 1458 int DuplicateFinder::AddTwoByteSymbol(Vector<const uint16_t> key, int value) { |
| 1539 return AddSymbol(Vector<const uint8_t>::cast(key), false, value); | 1459 return AddSymbol(Vector<const uint8_t>::cast(key), false, value); |
| 1540 } | 1460 } |
| 1541 | 1461 |
| 1542 | 1462 |
| 1543 int DuplicateFinder::AddSymbol(Vector<const uint8_t> key, | 1463 int DuplicateFinder::AddSymbol(Vector<const uint8_t> key, bool is_one_byte, |
| 1544 bool is_one_byte, | |
| 1545 int value) { | 1464 int value) { |
| 1546 uint32_t hash = Hash(key, is_one_byte); | 1465 uint32_t hash = Hash(key, is_one_byte); |
| 1547 byte* encoding = BackupKey(key, is_one_byte); | 1466 byte* encoding = BackupKey(key, is_one_byte); |
| 1548 HashMap::Entry* entry = map_.LookupOrInsert(encoding, hash); | 1467 HashMap::Entry* entry = map_.LookupOrInsert(encoding, hash); |
| 1549 int old_value = static_cast<int>(reinterpret_cast<intptr_t>(entry->value)); | 1468 int old_value = static_cast<int>(reinterpret_cast<intptr_t>(entry->value)); |
| 1550 entry->value = | 1469 entry->value = |
| 1551 reinterpret_cast<void*>(static_cast<intptr_t>(value | old_value)); | 1470 reinterpret_cast<void*>(static_cast<intptr_t>(value | old_value)); |
| 1552 return old_value; | 1471 return old_value; |
| 1553 } | 1472 } |
| 1554 | 1473 |
| 1555 | 1474 |
| 1556 int DuplicateFinder::AddNumber(Vector<const uint8_t> key, int value) { | 1475 int DuplicateFinder::AddNumber(Vector<const uint8_t> key, int value) { |
| 1557 DCHECK(key.length() > 0); | 1476 DCHECK(key.length() > 0); |
| 1558 // Quick check for already being in canonical form. | 1477 // Quick check for already being in canonical form. |
| 1559 if (IsNumberCanonical(key)) { | 1478 if (IsNumberCanonical(key)) { |
| 1560 return AddOneByteSymbol(key, value); | 1479 return AddOneByteSymbol(key, value); |
| 1561 } | 1480 } |
| 1562 | 1481 |
| 1563 int flags = ALLOW_HEX | ALLOW_OCTAL | ALLOW_IMPLICIT_OCTAL | ALLOW_BINARY; | 1482 int flags = ALLOW_HEX | ALLOW_OCTAL | ALLOW_IMPLICIT_OCTAL | ALLOW_BINARY; |
| 1564 double double_value = StringToDouble( | 1483 double double_value = StringToDouble(unicode_constants_, key, flags, 0.0); |
| 1565 unicode_constants_, key, flags, 0.0); | |
| 1566 int length; | 1484 int length; |
| 1567 const char* string; | 1485 const char* string; |
| 1568 if (!std::isfinite(double_value)) { | 1486 if (!std::isfinite(double_value)) { |
| 1569 string = "Infinity"; | 1487 string = "Infinity"; |
| 1570 length = 8; // strlen("Infinity"); | 1488 length = 8; // strlen("Infinity"); |
| 1571 } else { | 1489 } else { |
| 1572 string = DoubleToCString(double_value, | 1490 string = DoubleToCString(double_value, |
| 1573 Vector<char>(number_buffer_, kBufferSize)); | 1491 Vector<char>(number_buffer_, kBufferSize)); |
| 1574 length = StrLength(string); | 1492 length = StrLength(string); |
| 1575 } | 1493 } |
| 1576 return AddSymbol(Vector<const byte>(reinterpret_cast<const byte*>(string), | 1494 return AddSymbol( |
| 1577 length), true, value); | 1495 Vector<const byte>(reinterpret_cast<const byte*>(string), length), true, |
| 1496 value); |
| 1578 } | 1497 } |
| 1579 | 1498 |
| 1580 | 1499 |
| 1581 bool DuplicateFinder::IsNumberCanonical(Vector<const uint8_t> number) { | 1500 bool DuplicateFinder::IsNumberCanonical(Vector<const uint8_t> number) { |
| 1582 // Test for a safe approximation of number literals that are already | 1501 // Test for a safe approximation of number literals that are already |
| 1583 // in canonical form: max 15 digits, no leading zeroes, except an | 1502 // in canonical form: max 15 digits, no leading zeroes, except an |
| 1584 // integer part that is a single zero, and no trailing zeros below | 1503 // integer part that is a single zero, and no trailing zeros below |
| 1585 // the decimal point. | 1504 // the decimal point. |
| 1586 int pos = 0; | 1505 int pos = 0; |
| 1587 int length = number.length(); | 1506 int length = number.length(); |
| 1588 if (number.length() > 15) return false; | 1507 if (number.length() > 15) return false; |
| 1589 if (number[pos] == '0') { | 1508 if (number[pos] == '0') { |
| 1590 pos++; | 1509 pos++; |
| 1591 } else { | 1510 } else { |
| 1592 while (pos < length && | 1511 while (pos < length && |
| 1593 static_cast<unsigned>(number[pos] - '0') <= ('9' - '0')) pos++; | 1512 static_cast<unsigned>(number[pos] - '0') <= ('9' - '0')) |
| 1513 pos++; |
| 1594 } | 1514 } |
| 1595 if (length == pos) return true; | 1515 if (length == pos) return true; |
| 1596 if (number[pos] != '.') return false; | 1516 if (number[pos] != '.') return false; |
| 1597 pos++; | 1517 pos++; |
| 1598 bool invalid_last_digit = true; | 1518 bool invalid_last_digit = true; |
| 1599 while (pos < length) { | 1519 while (pos < length) { |
| 1600 uint8_t digit = number[pos] - '0'; | 1520 uint8_t digit = number[pos] - '0'; |
| 1601 if (digit > '9' - '0') return false; | 1521 if (digit > '9' - '0') return false; |
| 1602 invalid_last_digit = (digit == 0); | 1522 invalid_last_digit = (digit == 0); |
| 1603 pos++; | 1523 pos++; |
| (...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1651 if (one_byte_length >= (1 << 7)) { | 1571 if (one_byte_length >= (1 << 7)) { |
| 1652 if (one_byte_length >= (1 << 14)) { | 1572 if (one_byte_length >= (1 << 14)) { |
| 1653 if (one_byte_length >= (1 << 21)) { | 1573 if (one_byte_length >= (1 << 21)) { |
| 1654 if (one_byte_length >= (1 << 28)) { | 1574 if (one_byte_length >= (1 << 28)) { |
| 1655 backing_store_.Add( | 1575 backing_store_.Add( |
| 1656 static_cast<uint8_t>((one_byte_length >> 28) | 0x80)); | 1576 static_cast<uint8_t>((one_byte_length >> 28) | 0x80)); |
| 1657 } | 1577 } |
| 1658 backing_store_.Add( | 1578 backing_store_.Add( |
| 1659 static_cast<uint8_t>((one_byte_length >> 21) | 0x80u)); | 1579 static_cast<uint8_t>((one_byte_length >> 21) | 0x80u)); |
| 1660 } | 1580 } |
| 1661 backing_store_.Add( | 1581 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 14) | 0x80u)); |
| 1662 static_cast<uint8_t>((one_byte_length >> 14) | 0x80u)); | |
| 1663 } | 1582 } |
| 1664 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); | 1583 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); |
| 1665 } | 1584 } |
| 1666 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); | 1585 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); |
| 1667 | 1586 |
| 1668 backing_store_.AddBlock(bytes); | 1587 backing_store_.AddBlock(bytes); |
| 1669 return backing_store_.EndSequence().start(); | 1588 return backing_store_.EndSequence().start(); |
| 1670 } | 1589 } |
| 1671 | 1590 |
| 1672 } // namespace internal | 1591 } // namespace internal |
| 1673 } // namespace v8 | 1592 } // namespace v8 |
| OLD | NEW |