| OLD | NEW |
| 1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. |
| 6 | 6 |
| 7 #include <stdint.h> | 7 #include <stdint.h> |
| 8 | 8 |
| 9 #include <cmath> | 9 #include <cmath> |
| 10 | 10 |
| (...skipping 24 matching lines...) Expand all Loading... |
| 35 | 35 |
| 36 | 36 |
| 37 // ---------------------------------------------------------------------------- | 37 // ---------------------------------------------------------------------------- |
| 38 // Scanner | 38 // Scanner |
| 39 | 39 |
| 40 Scanner::Scanner(UnicodeCache* unicode_cache) | 40 Scanner::Scanner(UnicodeCache* unicode_cache) |
| 41 : unicode_cache_(unicode_cache), | 41 : unicode_cache_(unicode_cache), |
| 42 bookmark_c0_(kNoBookmark), | 42 bookmark_c0_(kNoBookmark), |
| 43 octal_pos_(Location::invalid()), | 43 octal_pos_(Location::invalid()), |
| 44 harmony_modules_(false), | 44 harmony_modules_(false), |
| 45 harmony_classes_(false), | |
| 46 harmony_unicode_(false) { | 45 harmony_unicode_(false) { |
| 47 bookmark_current_.literal_chars = &bookmark_current_literal_; | 46 bookmark_current_.literal_chars = &bookmark_current_literal_; |
| 48 bookmark_current_.raw_literal_chars = &bookmark_current_raw_literal_; | 47 bookmark_current_.raw_literal_chars = &bookmark_current_raw_literal_; |
| 49 bookmark_next_.literal_chars = &bookmark_next_literal_; | 48 bookmark_next_.literal_chars = &bookmark_next_literal_; |
| 50 bookmark_next_.raw_literal_chars = &bookmark_next_raw_literal_; | 49 bookmark_next_.raw_literal_chars = &bookmark_next_raw_literal_; |
| 51 } | 50 } |
| 52 | 51 |
| 53 | 52 |
| 54 void Scanner::Initialize(Utf16CharacterStream* source) { | 53 void Scanner::Initialize(Utf16CharacterStream* source) { |
| 55 source_ = source; | 54 source_ = source; |
| (...skipping 1036 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1092 Advance<capture_raw>(); | 1091 Advance<capture_raw>(); |
| 1093 return cp; | 1092 return cp; |
| 1094 } | 1093 } |
| 1095 return ScanHexNumber<capture_raw>(4); | 1094 return ScanHexNumber<capture_raw>(4); |
| 1096 } | 1095 } |
| 1097 | 1096 |
| 1098 | 1097 |
| 1099 // ---------------------------------------------------------------------------- | 1098 // ---------------------------------------------------------------------------- |
| 1100 // Keyword Matcher | 1099 // Keyword Matcher |
| 1101 | 1100 |
| 1102 #define KEYWORDS(KEYWORD_GROUP, KEYWORD) \ | 1101 #define KEYWORDS(KEYWORD_GROUP, KEYWORD) \ |
| 1103 KEYWORD_GROUP('b') \ | 1102 KEYWORD_GROUP('b') \ |
| 1104 KEYWORD("break", Token::BREAK) \ | 1103 KEYWORD("break", Token::BREAK) \ |
| 1105 KEYWORD_GROUP('c') \ | 1104 KEYWORD_GROUP('c') \ |
| 1106 KEYWORD("case", Token::CASE) \ | 1105 KEYWORD("case", Token::CASE) \ |
| 1107 KEYWORD("catch", Token::CATCH) \ | 1106 KEYWORD("catch", Token::CATCH) \ |
| 1108 KEYWORD("class", \ | 1107 KEYWORD("class", Token::CLASS) \ |
| 1109 harmony_classes ? Token::CLASS : Token::FUTURE_RESERVED_WORD) \ | 1108 KEYWORD("const", Token::CONST) \ |
| 1110 KEYWORD("const", Token::CONST) \ | 1109 KEYWORD("continue", Token::CONTINUE) \ |
| 1111 KEYWORD("continue", Token::CONTINUE) \ | 1110 KEYWORD_GROUP('d') \ |
| 1112 KEYWORD_GROUP('d') \ | 1111 KEYWORD("debugger", Token::DEBUGGER) \ |
| 1113 KEYWORD("debugger", Token::DEBUGGER) \ | 1112 KEYWORD("default", Token::DEFAULT) \ |
| 1114 KEYWORD("default", Token::DEFAULT) \ | 1113 KEYWORD("delete", Token::DELETE) \ |
| 1115 KEYWORD("delete", Token::DELETE) \ | 1114 KEYWORD("do", Token::DO) \ |
| 1116 KEYWORD("do", Token::DO) \ | 1115 KEYWORD_GROUP('e') \ |
| 1117 KEYWORD_GROUP('e') \ | 1116 KEYWORD("else", Token::ELSE) \ |
| 1118 KEYWORD("else", Token::ELSE) \ | 1117 KEYWORD("enum", Token::FUTURE_RESERVED_WORD) \ |
| 1119 KEYWORD("enum", Token::FUTURE_RESERVED_WORD) \ | 1118 KEYWORD("export", \ |
| 1120 KEYWORD("export", \ | 1119 harmony_modules ? Token::EXPORT : Token::FUTURE_RESERVED_WORD) \ |
| 1121 harmony_modules ? Token::EXPORT : Token::FUTURE_RESERVED_WORD) \ | 1120 KEYWORD("extends", Token::EXTENDS) \ |
| 1122 KEYWORD("extends", \ | 1121 KEYWORD_GROUP('f') \ |
| 1123 harmony_classes ? Token::EXTENDS : Token::FUTURE_RESERVED_WORD) \ | 1122 KEYWORD("false", Token::FALSE_LITERAL) \ |
| 1124 KEYWORD_GROUP('f') \ | 1123 KEYWORD("finally", Token::FINALLY) \ |
| 1125 KEYWORD("false", Token::FALSE_LITERAL) \ | 1124 KEYWORD("for", Token::FOR) \ |
| 1126 KEYWORD("finally", Token::FINALLY) \ | 1125 KEYWORD("function", Token::FUNCTION) \ |
| 1127 KEYWORD("for", Token::FOR) \ | 1126 KEYWORD_GROUP('i') \ |
| 1128 KEYWORD("function", Token::FUNCTION) \ | 1127 KEYWORD("if", Token::IF) \ |
| 1129 KEYWORD_GROUP('i') \ | 1128 KEYWORD("implements", Token::FUTURE_STRICT_RESERVED_WORD) \ |
| 1130 KEYWORD("if", Token::IF) \ | 1129 KEYWORD("import", \ |
| 1131 KEYWORD("implements", Token::FUTURE_STRICT_RESERVED_WORD) \ | 1130 harmony_modules ? Token::IMPORT : Token::FUTURE_RESERVED_WORD) \ |
| 1132 KEYWORD("import", \ | 1131 KEYWORD("in", Token::IN) \ |
| 1133 harmony_modules ? Token::IMPORT : Token::FUTURE_RESERVED_WORD) \ | 1132 KEYWORD("instanceof", Token::INSTANCEOF) \ |
| 1134 KEYWORD("in", Token::IN) \ | 1133 KEYWORD("interface", Token::FUTURE_STRICT_RESERVED_WORD) \ |
| 1135 KEYWORD("instanceof", Token::INSTANCEOF) \ | 1134 KEYWORD_GROUP('l') \ |
| 1136 KEYWORD("interface", Token::FUTURE_STRICT_RESERVED_WORD) \ | 1135 KEYWORD("let", Token::LET) \ |
| 1137 KEYWORD_GROUP('l') \ | 1136 KEYWORD_GROUP('n') \ |
| 1138 KEYWORD("let", Token::LET) \ | 1137 KEYWORD("new", Token::NEW) \ |
| 1139 KEYWORD_GROUP('n') \ | 1138 KEYWORD("null", Token::NULL_LITERAL) \ |
| 1140 KEYWORD("new", Token::NEW) \ | 1139 KEYWORD_GROUP('p') \ |
| 1141 KEYWORD("null", Token::NULL_LITERAL) \ | 1140 KEYWORD("package", Token::FUTURE_STRICT_RESERVED_WORD) \ |
| 1142 KEYWORD_GROUP('p') \ | 1141 KEYWORD("private", Token::FUTURE_STRICT_RESERVED_WORD) \ |
| 1143 KEYWORD("package", Token::FUTURE_STRICT_RESERVED_WORD) \ | 1142 KEYWORD("protected", Token::FUTURE_STRICT_RESERVED_WORD) \ |
| 1144 KEYWORD("private", Token::FUTURE_STRICT_RESERVED_WORD) \ | 1143 KEYWORD("public", Token::FUTURE_STRICT_RESERVED_WORD) \ |
| 1145 KEYWORD("protected", Token::FUTURE_STRICT_RESERVED_WORD) \ | 1144 KEYWORD_GROUP('r') \ |
| 1146 KEYWORD("public", Token::FUTURE_STRICT_RESERVED_WORD) \ | 1145 KEYWORD("return", Token::RETURN) \ |
| 1147 KEYWORD_GROUP('r') \ | 1146 KEYWORD_GROUP('s') \ |
| 1148 KEYWORD("return", Token::RETURN) \ | 1147 KEYWORD("static", Token::STATIC) \ |
| 1149 KEYWORD_GROUP('s') \ | 1148 KEYWORD("super", Token::SUPER) \ |
| 1150 KEYWORD("static", harmony_classes ? Token::STATIC \ | 1149 KEYWORD("switch", Token::SWITCH) \ |
| 1151 : Token::FUTURE_STRICT_RESERVED_WORD) \ | 1150 KEYWORD_GROUP('t') \ |
| 1152 KEYWORD("super", \ | 1151 KEYWORD("this", Token::THIS) \ |
| 1153 harmony_classes ? Token::SUPER : Token::FUTURE_RESERVED_WORD) \ | 1152 KEYWORD("throw", Token::THROW) \ |
| 1154 KEYWORD("switch", Token::SWITCH) \ | 1153 KEYWORD("true", Token::TRUE_LITERAL) \ |
| 1155 KEYWORD_GROUP('t') \ | 1154 KEYWORD("try", Token::TRY) \ |
| 1156 KEYWORD("this", Token::THIS) \ | 1155 KEYWORD("typeof", Token::TYPEOF) \ |
| 1157 KEYWORD("throw", Token::THROW) \ | 1156 KEYWORD_GROUP('v') \ |
| 1158 KEYWORD("true", Token::TRUE_LITERAL) \ | 1157 KEYWORD("var", Token::VAR) \ |
| 1159 KEYWORD("try", Token::TRY) \ | 1158 KEYWORD("void", Token::VOID) \ |
| 1160 KEYWORD("typeof", Token::TYPEOF) \ | 1159 KEYWORD_GROUP('w') \ |
| 1161 KEYWORD_GROUP('v') \ | 1160 KEYWORD("while", Token::WHILE) \ |
| 1162 KEYWORD("var", Token::VAR) \ | 1161 KEYWORD("with", Token::WITH) \ |
| 1163 KEYWORD("void", Token::VOID) \ | 1162 KEYWORD_GROUP('y') \ |
| 1164 KEYWORD_GROUP('w') \ | |
| 1165 KEYWORD("while", Token::WHILE) \ | |
| 1166 KEYWORD("with", Token::WITH) \ | |
| 1167 KEYWORD_GROUP('y') \ | |
| 1168 KEYWORD("yield", Token::YIELD) | 1163 KEYWORD("yield", Token::YIELD) |
| 1169 | 1164 |
| 1170 | 1165 |
| 1171 static Token::Value KeywordOrIdentifierToken(const uint8_t* input, | 1166 static Token::Value KeywordOrIdentifierToken(const uint8_t* input, |
| 1172 int input_length, | 1167 int input_length, |
| 1173 bool harmony_modules, | 1168 bool harmony_modules) { |
| 1174 bool harmony_classes) { | |
| 1175 DCHECK(input_length >= 1); | 1169 DCHECK(input_length >= 1); |
| 1176 const int kMinLength = 2; | 1170 const int kMinLength = 2; |
| 1177 const int kMaxLength = 10; | 1171 const int kMaxLength = 10; |
| 1178 if (input_length < kMinLength || input_length > kMaxLength) { | 1172 if (input_length < kMinLength || input_length > kMaxLength) { |
| 1179 return Token::IDENTIFIER; | 1173 return Token::IDENTIFIER; |
| 1180 } | 1174 } |
| 1181 switch (input[0]) { | 1175 switch (input[0]) { |
| 1182 default: | 1176 default: |
| 1183 #define KEYWORD_GROUP_CASE(ch) \ | 1177 #define KEYWORD_GROUP_CASE(ch) \ |
| 1184 break; \ | 1178 break; \ |
| (...skipping 27 matching lines...) Expand all Loading... |
| 1212 bool Scanner::IdentifierIsFutureStrictReserved( | 1206 bool Scanner::IdentifierIsFutureStrictReserved( |
| 1213 const AstRawString* string) const { | 1207 const AstRawString* string) const { |
| 1214 // Keywords are always 1-byte strings. | 1208 // Keywords are always 1-byte strings. |
| 1215 if (!string->is_one_byte()) return false; | 1209 if (!string->is_one_byte()) return false; |
| 1216 if (string->IsOneByteEqualTo("let") || string->IsOneByteEqualTo("static") || | 1210 if (string->IsOneByteEqualTo("let") || string->IsOneByteEqualTo("static") || |
| 1217 string->IsOneByteEqualTo("yield")) { | 1211 string->IsOneByteEqualTo("yield")) { |
| 1218 return true; | 1212 return true; |
| 1219 } | 1213 } |
| 1220 return Token::FUTURE_STRICT_RESERVED_WORD == | 1214 return Token::FUTURE_STRICT_RESERVED_WORD == |
| 1221 KeywordOrIdentifierToken(string->raw_data(), string->length(), | 1215 KeywordOrIdentifierToken(string->raw_data(), string->length(), |
| 1222 harmony_modules_, harmony_classes_); | 1216 harmony_modules_); |
| 1223 } | 1217 } |
| 1224 | 1218 |
| 1225 | 1219 |
| 1226 Token::Value Scanner::ScanIdentifierOrKeyword() { | 1220 Token::Value Scanner::ScanIdentifierOrKeyword() { |
| 1227 DCHECK(unicode_cache_->IsIdentifierStart(c0_)); | 1221 DCHECK(unicode_cache_->IsIdentifierStart(c0_)); |
| 1228 LiteralScope literal(this); | 1222 LiteralScope literal(this); |
| 1229 if (IsInRange(c0_, 'a', 'z')) { | 1223 if (IsInRange(c0_, 'a', 'z')) { |
| 1230 do { | 1224 do { |
| 1231 uc32 first_char = c0_; | 1225 uc32 first_char = c0_; |
| 1232 Advance<false, false>(); | 1226 Advance<false, false>(); |
| (...skipping 13 matching lines...) Expand all Loading... |
| 1246 } | 1240 } |
| 1247 if (c0_ <= kMaxAscii && c0_ != '\\') { | 1241 if (c0_ <= kMaxAscii && c0_ != '\\') { |
| 1248 literal.Complete(); | 1242 literal.Complete(); |
| 1249 return Token::IDENTIFIER; | 1243 return Token::IDENTIFIER; |
| 1250 } | 1244 } |
| 1251 } else if (c0_ <= kMaxAscii && c0_ != '\\') { | 1245 } else if (c0_ <= kMaxAscii && c0_ != '\\') { |
| 1252 // Only a-z+: could be a keyword or identifier. | 1246 // Only a-z+: could be a keyword or identifier. |
| 1253 literal.Complete(); | 1247 literal.Complete(); |
| 1254 Vector<const uint8_t> chars = next_.literal_chars->one_byte_literal(); | 1248 Vector<const uint8_t> chars = next_.literal_chars->one_byte_literal(); |
| 1255 return KeywordOrIdentifierToken(chars.start(), chars.length(), | 1249 return KeywordOrIdentifierToken(chars.start(), chars.length(), |
| 1256 harmony_modules_, harmony_classes_); | 1250 harmony_modules_); |
| 1257 } | 1251 } |
| 1258 | 1252 |
| 1259 HandleLeadSurrogate(); | 1253 HandleLeadSurrogate(); |
| 1260 } else if (IsInRange(c0_, 'A', 'Z') || c0_ == '_' || c0_ == '$') { | 1254 } else if (IsInRange(c0_, 'A', 'Z') || c0_ == '_' || c0_ == '$') { |
| 1261 do { | 1255 do { |
| 1262 uc32 first_char = c0_; | 1256 uc32 first_char = c0_; |
| 1263 Advance<false, false>(); | 1257 Advance<false, false>(); |
| 1264 AddLiteralChar(first_char); | 1258 AddLiteralChar(first_char); |
| 1265 } while (IsAsciiIdentifier(c0_)); | 1259 } while (IsAsciiIdentifier(c0_)); |
| 1266 | 1260 |
| (...skipping 29 matching lines...) Expand all Loading... |
| 1296 continue; | 1290 continue; |
| 1297 } | 1291 } |
| 1298 // Fallthrough if no longer able to complete keyword. | 1292 // Fallthrough if no longer able to complete keyword. |
| 1299 return ScanIdentifierSuffix(&literal); | 1293 return ScanIdentifierSuffix(&literal); |
| 1300 } | 1294 } |
| 1301 | 1295 |
| 1302 literal.Complete(); | 1296 literal.Complete(); |
| 1303 | 1297 |
| 1304 if (next_.literal_chars->is_one_byte()) { | 1298 if (next_.literal_chars->is_one_byte()) { |
| 1305 Vector<const uint8_t> chars = next_.literal_chars->one_byte_literal(); | 1299 Vector<const uint8_t> chars = next_.literal_chars->one_byte_literal(); |
| 1306 return KeywordOrIdentifierToken(chars.start(), | 1300 return KeywordOrIdentifierToken(chars.start(), chars.length(), |
| 1307 chars.length(), | 1301 harmony_modules_); |
| 1308 harmony_modules_, | |
| 1309 harmony_classes_); | |
| 1310 } | 1302 } |
| 1311 return Token::IDENTIFIER; | 1303 return Token::IDENTIFIER; |
| 1312 } | 1304 } |
| 1313 | 1305 |
| 1314 | 1306 |
| 1315 Token::Value Scanner::ScanIdentifierSuffix(LiteralScope* literal) { | 1307 Token::Value Scanner::ScanIdentifierSuffix(LiteralScope* literal) { |
| 1316 // Scan the rest of the identifier characters. | 1308 // Scan the rest of the identifier characters. |
| 1317 while (c0_ >= 0 && unicode_cache_->IsIdentifierPart(c0_)) { | 1309 while (c0_ >= 0 && unicode_cache_->IsIdentifierPart(c0_)) { |
| 1318 if (c0_ == '\\') { | 1310 if (c0_ == '\\') { |
| 1319 uc32 c = ScanIdentifierUnicodeEscape(); | 1311 uc32 c = ScanIdentifierUnicodeEscape(); |
| (...skipping 305 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1625 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); | 1617 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); |
| 1626 } | 1618 } |
| 1627 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); | 1619 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); |
| 1628 | 1620 |
| 1629 backing_store_.AddBlock(bytes); | 1621 backing_store_.AddBlock(bytes); |
| 1630 return backing_store_.EndSequence().start(); | 1622 return backing_store_.EndSequence().start(); |
| 1631 } | 1623 } |
| 1632 | 1624 |
| 1633 } // namespace internal | 1625 } // namespace internal |
| 1634 } // namespace v8 | 1626 } // namespace v8 |
| OLD | NEW |