OLD | NEW |
1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. |
6 | 6 |
7 #include <stdint.h> | 7 #include <stdint.h> |
8 | 8 |
9 #include <cmath> | 9 #include <cmath> |
10 | 10 |
(...skipping 17 matching lines...) Expand all Loading... |
28 return isolate->factory()->InternalizeTwoByteString(two_byte_literal()); | 28 return isolate->factory()->InternalizeTwoByteString(two_byte_literal()); |
29 } | 29 } |
30 | 30 |
31 | 31 |
32 // ---------------------------------------------------------------------------- | 32 // ---------------------------------------------------------------------------- |
33 // Scanner | 33 // Scanner |
34 | 34 |
35 Scanner::Scanner(UnicodeCache* unicode_cache) | 35 Scanner::Scanner(UnicodeCache* unicode_cache) |
36 : unicode_cache_(unicode_cache), | 36 : unicode_cache_(unicode_cache), |
37 octal_pos_(Location::invalid()), | 37 octal_pos_(Location::invalid()), |
38 harmony_scoping_(false), | |
39 harmony_modules_(false), | 38 harmony_modules_(false), |
40 harmony_numeric_literals_(false), | 39 harmony_numeric_literals_(false), |
41 harmony_classes_(false), | 40 harmony_classes_(false), |
42 harmony_templates_(false), | 41 harmony_templates_(false), |
43 harmony_unicode_(false) {} | 42 harmony_unicode_(false) {} |
44 | 43 |
45 | 44 |
46 void Scanner::Initialize(Utf16CharacterStream* source) { | 45 void Scanner::Initialize(Utf16CharacterStream* source) { |
47 source_ = source; | 46 source_ = source; |
48 // Need to capture identifiers in order to recognize "get" and "set" | 47 // Need to capture identifiers in order to recognize "get" and "set" |
(...skipping 1033 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1082 Advance<capture_raw>(); | 1081 Advance<capture_raw>(); |
1083 return cp; | 1082 return cp; |
1084 } | 1083 } |
1085 return ScanHexNumber<capture_raw>(4); | 1084 return ScanHexNumber<capture_raw>(4); |
1086 } | 1085 } |
1087 | 1086 |
1088 | 1087 |
1089 // ---------------------------------------------------------------------------- | 1088 // ---------------------------------------------------------------------------- |
1090 // Keyword Matcher | 1089 // Keyword Matcher |
1091 | 1090 |
1092 #define KEYWORDS(KEYWORD_GROUP, KEYWORD) \ | 1091 #define KEYWORDS(KEYWORD_GROUP, KEYWORD) \ |
1093 KEYWORD_GROUP('b') \ | 1092 KEYWORD_GROUP('b') \ |
1094 KEYWORD("break", Token::BREAK) \ | 1093 KEYWORD("break", Token::BREAK) \ |
1095 KEYWORD_GROUP('c') \ | 1094 KEYWORD_GROUP('c') \ |
1096 KEYWORD("case", Token::CASE) \ | 1095 KEYWORD("case", Token::CASE) \ |
1097 KEYWORD("catch", Token::CATCH) \ | 1096 KEYWORD("catch", Token::CATCH) \ |
1098 KEYWORD("class", \ | 1097 KEYWORD("class", \ |
1099 harmony_classes ? Token::CLASS : Token::FUTURE_RESERVED_WORD) \ | 1098 harmony_classes ? Token::CLASS : Token::FUTURE_RESERVED_WORD) \ |
1100 KEYWORD("const", Token::CONST) \ | 1099 KEYWORD("const", Token::CONST) \ |
1101 KEYWORD("continue", Token::CONTINUE) \ | 1100 KEYWORD("continue", Token::CONTINUE) \ |
1102 KEYWORD_GROUP('d') \ | 1101 KEYWORD_GROUP('d') \ |
1103 KEYWORD("debugger", Token::DEBUGGER) \ | 1102 KEYWORD("debugger", Token::DEBUGGER) \ |
1104 KEYWORD("default", Token::DEFAULT) \ | 1103 KEYWORD("default", Token::DEFAULT) \ |
1105 KEYWORD("delete", Token::DELETE) \ | 1104 KEYWORD("delete", Token::DELETE) \ |
1106 KEYWORD("do", Token::DO) \ | 1105 KEYWORD("do", Token::DO) \ |
1107 KEYWORD_GROUP('e') \ | 1106 KEYWORD_GROUP('e') \ |
1108 KEYWORD("else", Token::ELSE) \ | 1107 KEYWORD("else", Token::ELSE) \ |
1109 KEYWORD("enum", Token::FUTURE_RESERVED_WORD) \ | 1108 KEYWORD("enum", Token::FUTURE_RESERVED_WORD) \ |
1110 KEYWORD("export", \ | 1109 KEYWORD("export", \ |
1111 harmony_modules ? Token::EXPORT : Token::FUTURE_RESERVED_WORD) \ | 1110 harmony_modules ? Token::EXPORT : Token::FUTURE_RESERVED_WORD) \ |
1112 KEYWORD("extends", \ | 1111 KEYWORD("extends", \ |
1113 harmony_classes ? Token::EXTENDS : Token::FUTURE_RESERVED_WORD) \ | 1112 harmony_classes ? Token::EXTENDS : Token::FUTURE_RESERVED_WORD) \ |
1114 KEYWORD_GROUP('f') \ | 1113 KEYWORD_GROUP('f') \ |
1115 KEYWORD("false", Token::FALSE_LITERAL) \ | 1114 KEYWORD("false", Token::FALSE_LITERAL) \ |
1116 KEYWORD("finally", Token::FINALLY) \ | 1115 KEYWORD("finally", Token::FINALLY) \ |
1117 KEYWORD("for", Token::FOR) \ | 1116 KEYWORD("for", Token::FOR) \ |
1118 KEYWORD("function", Token::FUNCTION) \ | 1117 KEYWORD("function", Token::FUNCTION) \ |
1119 KEYWORD_GROUP('i') \ | 1118 KEYWORD_GROUP('i') \ |
1120 KEYWORD("if", Token::IF) \ | 1119 KEYWORD("if", Token::IF) \ |
1121 KEYWORD("implements", Token::FUTURE_STRICT_RESERVED_WORD) \ | 1120 KEYWORD("implements", Token::FUTURE_STRICT_RESERVED_WORD) \ |
1122 KEYWORD("import", \ | 1121 KEYWORD("import", \ |
1123 harmony_modules ? Token::IMPORT : Token::FUTURE_RESERVED_WORD) \ | 1122 harmony_modules ? Token::IMPORT : Token::FUTURE_RESERVED_WORD) \ |
1124 KEYWORD("in", Token::IN) \ | 1123 KEYWORD("in", Token::IN) \ |
1125 KEYWORD("instanceof", Token::INSTANCEOF) \ | 1124 KEYWORD("instanceof", Token::INSTANCEOF) \ |
1126 KEYWORD("interface", Token::FUTURE_STRICT_RESERVED_WORD) \ | 1125 KEYWORD("interface", Token::FUTURE_STRICT_RESERVED_WORD) \ |
1127 KEYWORD_GROUP('l') \ | 1126 KEYWORD_GROUP('l') \ |
1128 KEYWORD("let", \ | 1127 KEYWORD("let", Token::LET) \ |
1129 harmony_scoping ? Token::LET : Token::FUTURE_STRICT_RESERVED_WORD) \ | 1128 KEYWORD_GROUP('n') \ |
1130 KEYWORD_GROUP('n') \ | 1129 KEYWORD("new", Token::NEW) \ |
1131 KEYWORD("new", Token::NEW) \ | 1130 KEYWORD("null", Token::NULL_LITERAL) \ |
1132 KEYWORD("null", Token::NULL_LITERAL) \ | 1131 KEYWORD_GROUP('p') \ |
1133 KEYWORD_GROUP('p') \ | 1132 KEYWORD("package", Token::FUTURE_STRICT_RESERVED_WORD) \ |
1134 KEYWORD("package", Token::FUTURE_STRICT_RESERVED_WORD) \ | 1133 KEYWORD("private", Token::FUTURE_STRICT_RESERVED_WORD) \ |
1135 KEYWORD("private", Token::FUTURE_STRICT_RESERVED_WORD) \ | 1134 KEYWORD("protected", Token::FUTURE_STRICT_RESERVED_WORD) \ |
1136 KEYWORD("protected", Token::FUTURE_STRICT_RESERVED_WORD) \ | 1135 KEYWORD("public", Token::FUTURE_STRICT_RESERVED_WORD) \ |
1137 KEYWORD("public", Token::FUTURE_STRICT_RESERVED_WORD) \ | 1136 KEYWORD_GROUP('r') \ |
1138 KEYWORD_GROUP('r') \ | 1137 KEYWORD("return", Token::RETURN) \ |
1139 KEYWORD("return", Token::RETURN) \ | 1138 KEYWORD_GROUP('s') \ |
1140 KEYWORD_GROUP('s') \ | 1139 KEYWORD("static", harmony_classes ? Token::STATIC \ |
1141 KEYWORD("static", harmony_classes ? Token::STATIC \ | 1140 : Token::FUTURE_STRICT_RESERVED_WORD) \ |
1142 : Token::FUTURE_STRICT_RESERVED_WORD) \ | 1141 KEYWORD("super", \ |
1143 KEYWORD("super", \ | 1142 harmony_classes ? Token::SUPER : Token::FUTURE_RESERVED_WORD) \ |
1144 harmony_classes ? Token::SUPER : Token::FUTURE_RESERVED_WORD) \ | 1143 KEYWORD("switch", Token::SWITCH) \ |
1145 KEYWORD("switch", Token::SWITCH) \ | 1144 KEYWORD_GROUP('t') \ |
1146 KEYWORD_GROUP('t') \ | 1145 KEYWORD("this", Token::THIS) \ |
1147 KEYWORD("this", Token::THIS) \ | 1146 KEYWORD("throw", Token::THROW) \ |
1148 KEYWORD("throw", Token::THROW) \ | 1147 KEYWORD("true", Token::TRUE_LITERAL) \ |
1149 KEYWORD("true", Token::TRUE_LITERAL) \ | 1148 KEYWORD("try", Token::TRY) \ |
1150 KEYWORD("try", Token::TRY) \ | 1149 KEYWORD("typeof", Token::TYPEOF) \ |
1151 KEYWORD("typeof", Token::TYPEOF) \ | 1150 KEYWORD_GROUP('v') \ |
1152 KEYWORD_GROUP('v') \ | 1151 KEYWORD("var", Token::VAR) \ |
1153 KEYWORD("var", Token::VAR) \ | 1152 KEYWORD("void", Token::VOID) \ |
1154 KEYWORD("void", Token::VOID) \ | 1153 KEYWORD_GROUP('w') \ |
1155 KEYWORD_GROUP('w') \ | 1154 KEYWORD("while", Token::WHILE) \ |
1156 KEYWORD("while", Token::WHILE) \ | 1155 KEYWORD("with", Token::WITH) \ |
1157 KEYWORD("with", Token::WITH) \ | 1156 KEYWORD_GROUP('y') \ |
1158 KEYWORD_GROUP('y') \ | |
1159 KEYWORD("yield", Token::YIELD) | 1157 KEYWORD("yield", Token::YIELD) |
1160 | 1158 |
1161 | 1159 |
1162 static Token::Value KeywordOrIdentifierToken(const uint8_t* input, | 1160 static Token::Value KeywordOrIdentifierToken(const uint8_t* input, |
1163 int input_length, | 1161 int input_length, |
1164 bool harmony_scoping, | |
1165 bool harmony_modules, | 1162 bool harmony_modules, |
1166 bool harmony_classes) { | 1163 bool harmony_classes) { |
1167 DCHECK(input_length >= 1); | 1164 DCHECK(input_length >= 1); |
1168 const int kMinLength = 2; | 1165 const int kMinLength = 2; |
1169 const int kMaxLength = 10; | 1166 const int kMaxLength = 10; |
1170 if (input_length < kMinLength || input_length > kMaxLength) { | 1167 if (input_length < kMinLength || input_length > kMaxLength) { |
1171 return Token::IDENTIFIER; | 1168 return Token::IDENTIFIER; |
1172 } | 1169 } |
1173 switch (input[0]) { | 1170 switch (input[0]) { |
1174 default: | 1171 default: |
(...skipping 29 matching lines...) Expand all Loading... |
1204 bool Scanner::IdentifierIsFutureStrictReserved( | 1201 bool Scanner::IdentifierIsFutureStrictReserved( |
1205 const AstRawString* string) const { | 1202 const AstRawString* string) const { |
1206 // Keywords are always 1-byte strings. | 1203 // Keywords are always 1-byte strings. |
1207 if (!string->is_one_byte()) return false; | 1204 if (!string->is_one_byte()) return false; |
1208 if (string->IsOneByteEqualTo("let") || string->IsOneByteEqualTo("static") || | 1205 if (string->IsOneByteEqualTo("let") || string->IsOneByteEqualTo("static") || |
1209 string->IsOneByteEqualTo("yield")) { | 1206 string->IsOneByteEqualTo("yield")) { |
1210 return true; | 1207 return true; |
1211 } | 1208 } |
1212 return Token::FUTURE_STRICT_RESERVED_WORD == | 1209 return Token::FUTURE_STRICT_RESERVED_WORD == |
1213 KeywordOrIdentifierToken(string->raw_data(), string->length(), | 1210 KeywordOrIdentifierToken(string->raw_data(), string->length(), |
1214 harmony_scoping_, harmony_modules_, | 1211 harmony_modules_, harmony_classes_); |
1215 harmony_classes_); | |
1216 } | 1212 } |
1217 | 1213 |
1218 | 1214 |
1219 Token::Value Scanner::ScanIdentifierOrKeyword() { | 1215 Token::Value Scanner::ScanIdentifierOrKeyword() { |
1220 DCHECK(unicode_cache_->IsIdentifierStart(c0_)); | 1216 DCHECK(unicode_cache_->IsIdentifierStart(c0_)); |
1221 LiteralScope literal(this); | 1217 LiteralScope literal(this); |
1222 if (IsInRange(c0_, 'a', 'z')) { | 1218 if (IsInRange(c0_, 'a', 'z')) { |
1223 do { | 1219 do { |
1224 uc32 first_char = c0_; | 1220 uc32 first_char = c0_; |
1225 Advance<false, false>(); | 1221 Advance<false, false>(); |
(...skipping 13 matching lines...) Expand all Loading... |
1239 } | 1235 } |
1240 if (c0_ <= kMaxAscii && c0_ != '\\') { | 1236 if (c0_ <= kMaxAscii && c0_ != '\\') { |
1241 literal.Complete(); | 1237 literal.Complete(); |
1242 return Token::IDENTIFIER; | 1238 return Token::IDENTIFIER; |
1243 } | 1239 } |
1244 } else if (c0_ <= kMaxAscii && c0_ != '\\') { | 1240 } else if (c0_ <= kMaxAscii && c0_ != '\\') { |
1245 // Only a-z+: could be a keyword or identifier. | 1241 // Only a-z+: could be a keyword or identifier. |
1246 literal.Complete(); | 1242 literal.Complete(); |
1247 Vector<const uint8_t> chars = next_.literal_chars->one_byte_literal(); | 1243 Vector<const uint8_t> chars = next_.literal_chars->one_byte_literal(); |
1248 return KeywordOrIdentifierToken(chars.start(), chars.length(), | 1244 return KeywordOrIdentifierToken(chars.start(), chars.length(), |
1249 harmony_scoping_, harmony_modules_, | 1245 harmony_modules_, harmony_classes_); |
1250 harmony_classes_); | |
1251 } | 1246 } |
1252 | 1247 |
1253 HandleLeadSurrogate(); | 1248 HandleLeadSurrogate(); |
1254 } else if (IsInRange(c0_, 'A', 'Z') || c0_ == '_' || c0_ == '$') { | 1249 } else if (IsInRange(c0_, 'A', 'Z') || c0_ == '_' || c0_ == '$') { |
1255 do { | 1250 do { |
1256 uc32 first_char = c0_; | 1251 uc32 first_char = c0_; |
1257 Advance<false, false>(); | 1252 Advance<false, false>(); |
1258 AddLiteralChar(first_char); | 1253 AddLiteralChar(first_char); |
1259 } while (IsAsciiIdentifier(c0_)); | 1254 } while (IsAsciiIdentifier(c0_)); |
1260 | 1255 |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1292 // Fallthrough if no longer able to complete keyword. | 1287 // Fallthrough if no longer able to complete keyword. |
1293 return ScanIdentifierSuffix(&literal); | 1288 return ScanIdentifierSuffix(&literal); |
1294 } | 1289 } |
1295 | 1290 |
1296 literal.Complete(); | 1291 literal.Complete(); |
1297 | 1292 |
1298 if (next_.literal_chars->is_one_byte()) { | 1293 if (next_.literal_chars->is_one_byte()) { |
1299 Vector<const uint8_t> chars = next_.literal_chars->one_byte_literal(); | 1294 Vector<const uint8_t> chars = next_.literal_chars->one_byte_literal(); |
1300 return KeywordOrIdentifierToken(chars.start(), | 1295 return KeywordOrIdentifierToken(chars.start(), |
1301 chars.length(), | 1296 chars.length(), |
1302 harmony_scoping_, | |
1303 harmony_modules_, | 1297 harmony_modules_, |
1304 harmony_classes_); | 1298 harmony_classes_); |
1305 } | 1299 } |
1306 return Token::IDENTIFIER; | 1300 return Token::IDENTIFIER; |
1307 } | 1301 } |
1308 | 1302 |
1309 | 1303 |
1310 Token::Value Scanner::ScanIdentifierSuffix(LiteralScope* literal) { | 1304 Token::Value Scanner::ScanIdentifierSuffix(LiteralScope* literal) { |
1311 // Scan the rest of the identifier characters. | 1305 // Scan the rest of the identifier characters. |
1312 while (c0_ >= 0 && unicode_cache_->IsIdentifierPart(c0_)) { | 1306 while (c0_ >= 0 && unicode_cache_->IsIdentifierPart(c0_)) { |
(...skipping 256 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1569 } | 1563 } |
1570 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); | 1564 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); |
1571 } | 1565 } |
1572 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); | 1566 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); |
1573 | 1567 |
1574 backing_store_.AddBlock(bytes); | 1568 backing_store_.AddBlock(bytes); |
1575 return backing_store_.EndSequence().start(); | 1569 return backing_store_.EndSequence().start(); |
1576 } | 1570 } |
1577 | 1571 |
1578 } } // namespace v8::internal | 1572 } } // namespace v8::internal |
OLD | NEW |