OLD | NEW |
1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. |
6 | 6 |
7 #include "src/parsing/scanner.h" | 7 #include "src/parsing/scanner.h" |
8 | 8 |
9 #include <stdint.h> | 9 #include <stdint.h> |
10 | 10 |
(...skipping 22 matching lines...) Expand all Loading... |
33 void Utf16CharacterStream::ResetToBookmark() { UNREACHABLE(); } | 33 void Utf16CharacterStream::ResetToBookmark() { UNREACHABLE(); } |
34 | 34 |
35 | 35 |
36 // ---------------------------------------------------------------------------- | 36 // ---------------------------------------------------------------------------- |
37 // Scanner | 37 // Scanner |
38 | 38 |
39 Scanner::Scanner(UnicodeCache* unicode_cache) | 39 Scanner::Scanner(UnicodeCache* unicode_cache) |
40 : unicode_cache_(unicode_cache), | 40 : unicode_cache_(unicode_cache), |
41 bookmark_c0_(kNoBookmark), | 41 bookmark_c0_(kNoBookmark), |
42 octal_pos_(Location::invalid()), | 42 octal_pos_(Location::invalid()), |
43 decimal_with_leading_zero_pos_(Location::invalid()), | |
44 found_html_comment_(false), | 43 found_html_comment_(false), |
45 allow_harmony_exponentiation_operator_(false) { | 44 allow_harmony_exponentiation_operator_(false) { |
46 bookmark_current_.literal_chars = &bookmark_current_literal_; | 45 bookmark_current_.literal_chars = &bookmark_current_literal_; |
47 bookmark_current_.raw_literal_chars = &bookmark_current_raw_literal_; | 46 bookmark_current_.raw_literal_chars = &bookmark_current_raw_literal_; |
48 bookmark_next_.literal_chars = &bookmark_next_literal_; | 47 bookmark_next_.literal_chars = &bookmark_next_literal_; |
49 bookmark_next_.raw_literal_chars = &bookmark_next_raw_literal_; | 48 bookmark_next_.raw_literal_chars = &bookmark_next_raw_literal_; |
50 } | 49 } |
51 | 50 |
52 | 51 |
53 void Scanner::Initialize(Utf16CharacterStream* source) { | 52 void Scanner::Initialize(Utf16CharacterStream* source) { |
(...skipping 915 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
969 | 968 |
970 void Scanner::ScanDecimalDigits() { | 969 void Scanner::ScanDecimalDigits() { |
971 while (IsDecimalDigit(c0_)) | 970 while (IsDecimalDigit(c0_)) |
972 AddLiteralCharAdvance(); | 971 AddLiteralCharAdvance(); |
973 } | 972 } |
974 | 973 |
975 | 974 |
976 Token::Value Scanner::ScanNumber(bool seen_period) { | 975 Token::Value Scanner::ScanNumber(bool seen_period) { |
977 DCHECK(IsDecimalDigit(c0_)); // the first digit of the number or the fraction | 976 DCHECK(IsDecimalDigit(c0_)); // the first digit of the number or the fraction |
978 | 977 |
979 enum { | 978 enum { DECIMAL, HEX, OCTAL, IMPLICIT_OCTAL, BINARY } kind = DECIMAL; |
980 DECIMAL, | |
981 DECIMAL_WITH_LEADING_ZERO, | |
982 HEX, | |
983 OCTAL, | |
984 IMPLICIT_OCTAL, | |
985 BINARY | |
986 } kind = DECIMAL; | |
987 | 979 |
988 LiteralScope literal(this); | 980 LiteralScope literal(this); |
989 bool at_start = !seen_period; | 981 bool at_start = !seen_period; |
990 int start_pos; // For reporting octal positions. | |
991 if (seen_period) { | 982 if (seen_period) { |
992 // we have already seen a decimal point of the float | 983 // we have already seen a decimal point of the float |
993 AddLiteralChar('.'); | 984 AddLiteralChar('.'); |
994 ScanDecimalDigits(); // we know we have at least one digit | 985 ScanDecimalDigits(); // we know we have at least one digit |
995 | 986 |
996 } else { | 987 } else { |
997 // if the first character is '0' we must check for octals and hex | 988 // if the first character is '0' we must check for octals and hex |
998 if (c0_ == '0') { | 989 if (c0_ == '0') { |
999 start_pos = source_pos(); | 990 int start_pos = source_pos(); // For reporting octal positions. |
1000 AddLiteralCharAdvance(); | 991 AddLiteralCharAdvance(); |
1001 | 992 |
1002 // either 0, 0exxx, 0Exxx, 0.xxx, a hex number, a binary number or | 993 // either 0, 0exxx, 0Exxx, 0.xxx, a hex number, a binary number or |
1003 // an octal number. | 994 // an octal number. |
1004 if (c0_ == 'x' || c0_ == 'X') { | 995 if (c0_ == 'x' || c0_ == 'X') { |
1005 // hex number | 996 // hex number |
1006 kind = HEX; | 997 kind = HEX; |
1007 AddLiteralCharAdvance(); | 998 AddLiteralCharAdvance(); |
1008 if (!IsHexDigit(c0_)) { | 999 if (!IsHexDigit(c0_)) { |
1009 // we must have at least one hex digit after 'x'/'X' | 1000 // we must have at least one hex digit after 'x'/'X' |
(...skipping 21 matching lines...) Expand all Loading... |
1031 } | 1022 } |
1032 while (IsBinaryDigit(c0_)) { | 1023 while (IsBinaryDigit(c0_)) { |
1033 AddLiteralCharAdvance(); | 1024 AddLiteralCharAdvance(); |
1034 } | 1025 } |
1035 } else if ('0' <= c0_ && c0_ <= '7') { | 1026 } else if ('0' <= c0_ && c0_ <= '7') { |
1036 // (possible) octal number | 1027 // (possible) octal number |
1037 kind = IMPLICIT_OCTAL; | 1028 kind = IMPLICIT_OCTAL; |
1038 while (true) { | 1029 while (true) { |
1039 if (c0_ == '8' || c0_ == '9') { | 1030 if (c0_ == '8' || c0_ == '9') { |
1040 at_start = false; | 1031 at_start = false; |
1041 kind = DECIMAL_WITH_LEADING_ZERO; | 1032 kind = DECIMAL; |
1042 break; | 1033 break; |
1043 } | 1034 } |
1044 if (c0_ < '0' || '7' < c0_) { | 1035 if (c0_ < '0' || '7' < c0_) { |
1045 // Octal literal finished. | 1036 // Octal literal finished. |
1046 octal_pos_ = Location(start_pos, source_pos()); | 1037 octal_pos_ = Location(start_pos, source_pos()); |
1047 break; | 1038 break; |
1048 } | 1039 } |
1049 AddLiteralCharAdvance(); | 1040 AddLiteralCharAdvance(); |
1050 } | 1041 } |
1051 } else if (c0_ == '8' || c0_ == '9') { | |
1052 kind = DECIMAL_WITH_LEADING_ZERO; | |
1053 } | 1042 } |
1054 } | 1043 } |
1055 | 1044 |
1056 // Parse decimal digits and allow trailing fractional part. | 1045 // Parse decimal digits and allow trailing fractional part. |
1057 if (kind == DECIMAL || kind == DECIMAL_WITH_LEADING_ZERO) { | 1046 if (kind == DECIMAL) { |
1058 if (at_start) { | 1047 if (at_start) { |
1059 uint64_t value = 0; | 1048 uint64_t value = 0; |
1060 while (IsDecimalDigit(c0_)) { | 1049 while (IsDecimalDigit(c0_)) { |
1061 value = 10 * value + (c0_ - '0'); | 1050 value = 10 * value + (c0_ - '0'); |
1062 | 1051 |
1063 uc32 first_char = c0_; | 1052 uc32 first_char = c0_; |
1064 Advance<false, false>(); | 1053 Advance<false, false>(); |
1065 AddLiteralChar(first_char); | 1054 AddLiteralChar(first_char); |
1066 } | 1055 } |
1067 | 1056 |
1068 if (next_.literal_chars->one_byte_literal().length() <= 10 && | 1057 if (next_.literal_chars->one_byte_literal().length() <= 10 && |
1069 value <= Smi::kMaxValue && c0_ != '.' && c0_ != 'e' && c0_ != 'E') { | 1058 value <= Smi::kMaxValue && c0_ != '.' && c0_ != 'e' && c0_ != 'E') { |
1070 next_.smi_value_ = static_cast<int>(value); | 1059 next_.smi_value_ = static_cast<int>(value); |
1071 literal.Complete(); | 1060 literal.Complete(); |
1072 HandleLeadSurrogate(); | 1061 HandleLeadSurrogate(); |
1073 | 1062 |
1074 if (kind == DECIMAL_WITH_LEADING_ZERO) | |
1075 decimal_with_leading_zero_pos_ = Location(start_pos, source_pos()); | |
1076 return Token::SMI; | 1063 return Token::SMI; |
1077 } | 1064 } |
1078 HandleLeadSurrogate(); | 1065 HandleLeadSurrogate(); |
1079 } | 1066 } |
1080 | 1067 |
1081 ScanDecimalDigits(); // optional | 1068 ScanDecimalDigits(); // optional |
1082 if (c0_ == '.') { | 1069 if (c0_ == '.') { |
1083 AddLiteralCharAdvance(); | 1070 AddLiteralCharAdvance(); |
1084 ScanDecimalDigits(); // optional | 1071 ScanDecimalDigits(); // optional |
1085 } | 1072 } |
1086 } | 1073 } |
1087 } | 1074 } |
1088 | 1075 |
1089 // scan exponent, if any | 1076 // scan exponent, if any |
1090 if (c0_ == 'e' || c0_ == 'E') { | 1077 if (c0_ == 'e' || c0_ == 'E') { |
1091 DCHECK(kind != HEX); // 'e'/'E' must be scanned as part of the hex number | 1078 DCHECK(kind != HEX); // 'e'/'E' must be scanned as part of the hex number |
1092 if (!(kind == DECIMAL || kind == DECIMAL_WITH_LEADING_ZERO)) | 1079 if (kind != DECIMAL) return Token::ILLEGAL; |
1093 return Token::ILLEGAL; | |
1094 // scan exponent | 1080 // scan exponent |
1095 AddLiteralCharAdvance(); | 1081 AddLiteralCharAdvance(); |
1096 if (c0_ == '+' || c0_ == '-') | 1082 if (c0_ == '+' || c0_ == '-') |
1097 AddLiteralCharAdvance(); | 1083 AddLiteralCharAdvance(); |
1098 if (!IsDecimalDigit(c0_)) { | 1084 if (!IsDecimalDigit(c0_)) { |
1099 // we must have at least one decimal digit after 'e'/'E' | 1085 // we must have at least one decimal digit after 'e'/'E' |
1100 return Token::ILLEGAL; | 1086 return Token::ILLEGAL; |
1101 } | 1087 } |
1102 ScanDecimalDigits(); | 1088 ScanDecimalDigits(); |
1103 } | 1089 } |
1104 | 1090 |
1105 // The source character immediately following a numeric literal must | 1091 // The source character immediately following a numeric literal must |
1106 // not be an identifier start or a decimal digit; see ECMA-262 | 1092 // not be an identifier start or a decimal digit; see ECMA-262 |
1107 // section 7.8.3, page 17 (note that we read only one decimal digit | 1093 // section 7.8.3, page 17 (note that we read only one decimal digit |
1108 // if the value is 0). | 1094 // if the value is 0). |
1109 if (IsDecimalDigit(c0_) || | 1095 if (IsDecimalDigit(c0_) || |
1110 (c0_ >= 0 && unicode_cache_->IsIdentifierStart(c0_))) | 1096 (c0_ >= 0 && unicode_cache_->IsIdentifierStart(c0_))) |
1111 return Token::ILLEGAL; | 1097 return Token::ILLEGAL; |
1112 | 1098 |
1113 literal.Complete(); | 1099 literal.Complete(); |
1114 | 1100 |
1115 if (kind == DECIMAL_WITH_LEADING_ZERO) | |
1116 decimal_with_leading_zero_pos_ = Location(start_pos, source_pos()); | |
1117 return Token::NUMBER; | 1101 return Token::NUMBER; |
1118 } | 1102 } |
1119 | 1103 |
1120 | 1104 |
1121 uc32 Scanner::ScanIdentifierUnicodeEscape() { | 1105 uc32 Scanner::ScanIdentifierUnicodeEscape() { |
1122 Advance(); | 1106 Advance(); |
1123 if (c0_ != 'u') return -1; | 1107 if (c0_ != 'u') return -1; |
1124 Advance(); | 1108 Advance(); |
1125 return ScanUnicodeEscape<false>(); | 1109 return ScanUnicodeEscape<false>(); |
1126 } | 1110 } |
(...skipping 576 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1703 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); | 1687 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); |
1704 } | 1688 } |
1705 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); | 1689 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); |
1706 | 1690 |
1707 backing_store_.AddBlock(bytes); | 1691 backing_store_.AddBlock(bytes); |
1708 return backing_store_.EndSequence().start(); | 1692 return backing_store_.EndSequence().start(); |
1709 } | 1693 } |
1710 | 1694 |
1711 } // namespace internal | 1695 } // namespace internal |
1712 } // namespace v8 | 1696 } // namespace v8 |
OLD | NEW |