| OLD | NEW | 
|---|
| 1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. | 
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be | 
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. | 
| 4 | 4 | 
| 5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. | 
| 6 | 6 | 
| 7 #include <stdint.h> | 7 #include <stdint.h> | 
| 8 | 8 | 
| 9 #include <cmath> | 9 #include <cmath> | 
| 10 | 10 | 
| (...skipping 23 matching lines...) Expand all  Loading... | 
| 34 void Utf16CharacterStream::ResetToBookmark() { UNREACHABLE(); } | 34 void Utf16CharacterStream::ResetToBookmark() { UNREACHABLE(); } | 
| 35 | 35 | 
| 36 | 36 | 
| 37 // ---------------------------------------------------------------------------- | 37 // ---------------------------------------------------------------------------- | 
| 38 // Scanner | 38 // Scanner | 
| 39 | 39 | 
| 40 Scanner::Scanner(UnicodeCache* unicode_cache) | 40 Scanner::Scanner(UnicodeCache* unicode_cache) | 
| 41     : unicode_cache_(unicode_cache), | 41     : unicode_cache_(unicode_cache), | 
| 42       bookmark_c0_(kNoBookmark), | 42       bookmark_c0_(kNoBookmark), | 
| 43       octal_pos_(Location::invalid()), | 43       octal_pos_(Location::invalid()), | 
| 44       harmony_modules_(false), |  | 
| 45       harmony_unicode_(false) { | 44       harmony_unicode_(false) { | 
| 46   bookmark_current_.literal_chars = &bookmark_current_literal_; | 45   bookmark_current_.literal_chars = &bookmark_current_literal_; | 
| 47   bookmark_current_.raw_literal_chars = &bookmark_current_raw_literal_; | 46   bookmark_current_.raw_literal_chars = &bookmark_current_raw_literal_; | 
| 48   bookmark_next_.literal_chars = &bookmark_next_literal_; | 47   bookmark_next_.literal_chars = &bookmark_next_literal_; | 
| 49   bookmark_next_.raw_literal_chars = &bookmark_next_raw_literal_; | 48   bookmark_next_.raw_literal_chars = &bookmark_next_raw_literal_; | 
| 50 } | 49 } | 
| 51 | 50 | 
| 52 | 51 | 
| 53 void Scanner::Initialize(Utf16CharacterStream* source) { | 52 void Scanner::Initialize(Utf16CharacterStream* source) { | 
| 54   source_ = source; | 53   source_ = source; | 
| (...skipping 1036 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
| 1091     Advance<capture_raw>(); | 1090     Advance<capture_raw>(); | 
| 1092     return cp; | 1091     return cp; | 
| 1093   } | 1092   } | 
| 1094   return ScanHexNumber<capture_raw>(4); | 1093   return ScanHexNumber<capture_raw>(4); | 
| 1095 } | 1094 } | 
| 1096 | 1095 | 
| 1097 | 1096 | 
| 1098 // ---------------------------------------------------------------------------- | 1097 // ---------------------------------------------------------------------------- | 
| 1099 // Keyword Matcher | 1098 // Keyword Matcher | 
| 1100 | 1099 | 
| 1101 #define KEYWORDS(KEYWORD_GROUP, KEYWORD)                                 \ | 1100 #define KEYWORDS(KEYWORD_GROUP, KEYWORD)                    \ | 
| 1102   KEYWORD_GROUP('b')                                                     \ | 1101   KEYWORD_GROUP('b')                                        \ | 
| 1103   KEYWORD("break", Token::BREAK)                                         \ | 1102   KEYWORD("break", Token::BREAK)                            \ | 
| 1104   KEYWORD_GROUP('c')                                                     \ | 1103   KEYWORD_GROUP('c')                                        \ | 
| 1105   KEYWORD("case", Token::CASE)                                           \ | 1104   KEYWORD("case", Token::CASE)                              \ | 
| 1106   KEYWORD("catch", Token::CATCH)                                         \ | 1105   KEYWORD("catch", Token::CATCH)                            \ | 
| 1107   KEYWORD("class", Token::CLASS)                                         \ | 1106   KEYWORD("class", Token::CLASS)                            \ | 
| 1108   KEYWORD("const", Token::CONST)                                         \ | 1107   KEYWORD("const", Token::CONST)                            \ | 
| 1109   KEYWORD("continue", Token::CONTINUE)                                   \ | 1108   KEYWORD("continue", Token::CONTINUE)                      \ | 
| 1110   KEYWORD_GROUP('d')                                                     \ | 1109   KEYWORD_GROUP('d')                                        \ | 
| 1111   KEYWORD("debugger", Token::DEBUGGER)                                   \ | 1110   KEYWORD("debugger", Token::DEBUGGER)                      \ | 
| 1112   KEYWORD("default", Token::DEFAULT)                                     \ | 1111   KEYWORD("default", Token::DEFAULT)                        \ | 
| 1113   KEYWORD("delete", Token::DELETE)                                       \ | 1112   KEYWORD("delete", Token::DELETE)                          \ | 
| 1114   KEYWORD("do", Token::DO)                                               \ | 1113   KEYWORD("do", Token::DO)                                  \ | 
| 1115   KEYWORD_GROUP('e')                                                     \ | 1114   KEYWORD_GROUP('e')                                        \ | 
| 1116   KEYWORD("else", Token::ELSE)                                           \ | 1115   KEYWORD("else", Token::ELSE)                              \ | 
| 1117   KEYWORD("enum", Token::FUTURE_RESERVED_WORD)                           \ | 1116   KEYWORD("enum", Token::FUTURE_RESERVED_WORD)              \ | 
| 1118   KEYWORD("export",                                                      \ | 1117   KEYWORD("export", Token::EXPORT)                          \ | 
| 1119           harmony_modules ? Token::EXPORT : Token::FUTURE_RESERVED_WORD) \ | 1118   KEYWORD("extends", Token::EXTENDS)                        \ | 
| 1120   KEYWORD("extends", Token::EXTENDS)                                     \ | 1119   KEYWORD_GROUP('f')                                        \ | 
| 1121   KEYWORD_GROUP('f')                                                     \ | 1120   KEYWORD("false", Token::FALSE_LITERAL)                    \ | 
| 1122   KEYWORD("false", Token::FALSE_LITERAL)                                 \ | 1121   KEYWORD("finally", Token::FINALLY)                        \ | 
| 1123   KEYWORD("finally", Token::FINALLY)                                     \ | 1122   KEYWORD("for", Token::FOR)                                \ | 
| 1124   KEYWORD("for", Token::FOR)                                             \ | 1123   KEYWORD("function", Token::FUNCTION)                      \ | 
| 1125   KEYWORD("function", Token::FUNCTION)                                   \ | 1124   KEYWORD_GROUP('i')                                        \ | 
| 1126   KEYWORD_GROUP('i')                                                     \ | 1125   KEYWORD("if", Token::IF)                                  \ | 
| 1127   KEYWORD("if", Token::IF)                                               \ | 1126   KEYWORD("implements", Token::FUTURE_STRICT_RESERVED_WORD) \ | 
| 1128   KEYWORD("implements", Token::FUTURE_STRICT_RESERVED_WORD)              \ | 1127   KEYWORD("import", Token::IMPORT)                          \ | 
| 1129   KEYWORD("import",                                                      \ | 1128   KEYWORD("in", Token::IN)                                  \ | 
| 1130           harmony_modules ? Token::IMPORT : Token::FUTURE_RESERVED_WORD) \ | 1129   KEYWORD("instanceof", Token::INSTANCEOF)                  \ | 
| 1131   KEYWORD("in", Token::IN)                                               \ | 1130   KEYWORD("interface", Token::FUTURE_STRICT_RESERVED_WORD)  \ | 
| 1132   KEYWORD("instanceof", Token::INSTANCEOF)                               \ | 1131   KEYWORD_GROUP('l')                                        \ | 
| 1133   KEYWORD("interface", Token::FUTURE_STRICT_RESERVED_WORD)               \ | 1132   KEYWORD("let", Token::LET)                                \ | 
| 1134   KEYWORD_GROUP('l')                                                     \ | 1133   KEYWORD_GROUP('n')                                        \ | 
| 1135   KEYWORD("let", Token::LET)                                             \ | 1134   KEYWORD("new", Token::NEW)                                \ | 
| 1136   KEYWORD_GROUP('n')                                                     \ | 1135   KEYWORD("null", Token::NULL_LITERAL)                      \ | 
| 1137   KEYWORD("new", Token::NEW)                                             \ | 1136   KEYWORD_GROUP('p')                                        \ | 
| 1138   KEYWORD("null", Token::NULL_LITERAL)                                   \ | 1137   KEYWORD("package", Token::FUTURE_STRICT_RESERVED_WORD)    \ | 
| 1139   KEYWORD_GROUP('p')                                                     \ | 1138   KEYWORD("private", Token::FUTURE_STRICT_RESERVED_WORD)    \ | 
| 1140   KEYWORD("package", Token::FUTURE_STRICT_RESERVED_WORD)                 \ | 1139   KEYWORD("protected", Token::FUTURE_STRICT_RESERVED_WORD)  \ | 
| 1141   KEYWORD("private", Token::FUTURE_STRICT_RESERVED_WORD)                 \ | 1140   KEYWORD("public", Token::FUTURE_STRICT_RESERVED_WORD)     \ | 
| 1142   KEYWORD("protected", Token::FUTURE_STRICT_RESERVED_WORD)               \ | 1141   KEYWORD_GROUP('r')                                        \ | 
| 1143   KEYWORD("public", Token::FUTURE_STRICT_RESERVED_WORD)                  \ | 1142   KEYWORD("return", Token::RETURN)                          \ | 
| 1144   KEYWORD_GROUP('r')                                                     \ | 1143   KEYWORD_GROUP('s')                                        \ | 
| 1145   KEYWORD("return", Token::RETURN)                                       \ | 1144   KEYWORD("static", Token::STATIC)                          \ | 
| 1146   KEYWORD_GROUP('s')                                                     \ | 1145   KEYWORD("super", Token::SUPER)                            \ | 
| 1147   KEYWORD("static", Token::STATIC)                                       \ | 1146   KEYWORD("switch", Token::SWITCH)                          \ | 
| 1148   KEYWORD("super", Token::SUPER)                                         \ | 1147   KEYWORD_GROUP('t')                                        \ | 
| 1149   KEYWORD("switch", Token::SWITCH)                                       \ | 1148   KEYWORD("this", Token::THIS)                              \ | 
| 1150   KEYWORD_GROUP('t')                                                     \ | 1149   KEYWORD("throw", Token::THROW)                            \ | 
| 1151   KEYWORD("this", Token::THIS)                                           \ | 1150   KEYWORD("true", Token::TRUE_LITERAL)                      \ | 
| 1152   KEYWORD("throw", Token::THROW)                                         \ | 1151   KEYWORD("try", Token::TRY)                                \ | 
| 1153   KEYWORD("true", Token::TRUE_LITERAL)                                   \ | 1152   KEYWORD("typeof", Token::TYPEOF)                          \ | 
| 1154   KEYWORD("try", Token::TRY)                                             \ | 1153   KEYWORD_GROUP('v')                                        \ | 
| 1155   KEYWORD("typeof", Token::TYPEOF)                                       \ | 1154   KEYWORD("var", Token::VAR)                                \ | 
| 1156   KEYWORD_GROUP('v')                                                     \ | 1155   KEYWORD("void", Token::VOID)                              \ | 
| 1157   KEYWORD("var", Token::VAR)                                             \ | 1156   KEYWORD_GROUP('w')                                        \ | 
| 1158   KEYWORD("void", Token::VOID)                                           \ | 1157   KEYWORD("while", Token::WHILE)                            \ | 
| 1159   KEYWORD_GROUP('w')                                                     \ | 1158   KEYWORD("with", Token::WITH)                              \ | 
| 1160   KEYWORD("while", Token::WHILE)                                         \ | 1159   KEYWORD_GROUP('y')                                        \ | 
| 1161   KEYWORD("with", Token::WITH)                                           \ |  | 
| 1162   KEYWORD_GROUP('y')                                                     \ |  | 
| 1163   KEYWORD("yield", Token::YIELD) | 1160   KEYWORD("yield", Token::YIELD) | 
| 1164 | 1161 | 
| 1165 | 1162 | 
| 1166 static Token::Value KeywordOrIdentifierToken(const uint8_t* input, | 1163 static Token::Value KeywordOrIdentifierToken(const uint8_t* input, | 
| 1167                                              int input_length, | 1164                                              int input_length) { | 
| 1168                                              bool harmony_modules) { |  | 
| 1169   DCHECK(input_length >= 1); | 1165   DCHECK(input_length >= 1); | 
| 1170   const int kMinLength = 2; | 1166   const int kMinLength = 2; | 
| 1171   const int kMaxLength = 10; | 1167   const int kMaxLength = 10; | 
| 1172   if (input_length < kMinLength || input_length > kMaxLength) { | 1168   if (input_length < kMinLength || input_length > kMaxLength) { | 
| 1173     return Token::IDENTIFIER; | 1169     return Token::IDENTIFIER; | 
| 1174   } | 1170   } | 
| 1175   switch (input[0]) { | 1171   switch (input[0]) { | 
| 1176     default: | 1172     default: | 
| 1177 #define KEYWORD_GROUP_CASE(ch)                                \ | 1173 #define KEYWORD_GROUP_CASE(ch)                                \ | 
| 1178       break;                                                  \ | 1174       break;                                                  \ | 
| (...skipping 26 matching lines...) Expand all  Loading... | 
| 1205 | 1201 | 
| 1206 bool Scanner::IdentifierIsFutureStrictReserved( | 1202 bool Scanner::IdentifierIsFutureStrictReserved( | 
| 1207     const AstRawString* string) const { | 1203     const AstRawString* string) const { | 
| 1208   // Keywords are always 1-byte strings. | 1204   // Keywords are always 1-byte strings. | 
| 1209   if (!string->is_one_byte()) return false; | 1205   if (!string->is_one_byte()) return false; | 
| 1210   if (string->IsOneByteEqualTo("let") || string->IsOneByteEqualTo("static") || | 1206   if (string->IsOneByteEqualTo("let") || string->IsOneByteEqualTo("static") || | 
| 1211       string->IsOneByteEqualTo("yield")) { | 1207       string->IsOneByteEqualTo("yield")) { | 
| 1212     return true; | 1208     return true; | 
| 1213   } | 1209   } | 
| 1214   return Token::FUTURE_STRICT_RESERVED_WORD == | 1210   return Token::FUTURE_STRICT_RESERVED_WORD == | 
| 1215          KeywordOrIdentifierToken(string->raw_data(), string->length(), | 1211          KeywordOrIdentifierToken(string->raw_data(), string->length()); | 
| 1216                                   harmony_modules_); |  | 
| 1217 } | 1212 } | 
| 1218 | 1213 | 
| 1219 | 1214 | 
| 1220 Token::Value Scanner::ScanIdentifierOrKeyword() { | 1215 Token::Value Scanner::ScanIdentifierOrKeyword() { | 
| 1221   DCHECK(unicode_cache_->IsIdentifierStart(c0_)); | 1216   DCHECK(unicode_cache_->IsIdentifierStart(c0_)); | 
| 1222   LiteralScope literal(this); | 1217   LiteralScope literal(this); | 
| 1223   if (IsInRange(c0_, 'a', 'z')) { | 1218   if (IsInRange(c0_, 'a', 'z')) { | 
| 1224     do { | 1219     do { | 
| 1225       uc32 first_char = c0_; | 1220       uc32 first_char = c0_; | 
| 1226       Advance<false, false>(); | 1221       Advance<false, false>(); | 
| (...skipping 12 matching lines...) Expand all  Loading... | 
| 1239         AddLiteralChar(first_char); | 1234         AddLiteralChar(first_char); | 
| 1240       } | 1235       } | 
| 1241       if (c0_ <= kMaxAscii && c0_ != '\\') { | 1236       if (c0_ <= kMaxAscii && c0_ != '\\') { | 
| 1242         literal.Complete(); | 1237         literal.Complete(); | 
| 1243         return Token::IDENTIFIER; | 1238         return Token::IDENTIFIER; | 
| 1244       } | 1239       } | 
| 1245     } else if (c0_ <= kMaxAscii && c0_ != '\\') { | 1240     } else if (c0_ <= kMaxAscii && c0_ != '\\') { | 
| 1246       // Only a-z+: could be a keyword or identifier. | 1241       // Only a-z+: could be a keyword or identifier. | 
| 1247       literal.Complete(); | 1242       literal.Complete(); | 
| 1248       Vector<const uint8_t> chars = next_.literal_chars->one_byte_literal(); | 1243       Vector<const uint8_t> chars = next_.literal_chars->one_byte_literal(); | 
| 1249       return KeywordOrIdentifierToken(chars.start(), chars.length(), | 1244       return KeywordOrIdentifierToken(chars.start(), chars.length()); | 
| 1250                                       harmony_modules_); |  | 
| 1251     } | 1245     } | 
| 1252 | 1246 | 
| 1253     HandleLeadSurrogate(); | 1247     HandleLeadSurrogate(); | 
| 1254   } else if (IsInRange(c0_, 'A', 'Z') || c0_ == '_' || c0_ == '$') { | 1248   } else if (IsInRange(c0_, 'A', 'Z') || c0_ == '_' || c0_ == '$') { | 
| 1255     do { | 1249     do { | 
| 1256       uc32 first_char = c0_; | 1250       uc32 first_char = c0_; | 
| 1257       Advance<false, false>(); | 1251       Advance<false, false>(); | 
| 1258       AddLiteralChar(first_char); | 1252       AddLiteralChar(first_char); | 
| 1259     } while (IsAsciiIdentifier(c0_)); | 1253     } while (IsAsciiIdentifier(c0_)); | 
| 1260 | 1254 | 
| (...skipping 29 matching lines...) Expand all  Loading... | 
| 1290       continue; | 1284       continue; | 
| 1291     } | 1285     } | 
| 1292     // Fallthrough if no longer able to complete keyword. | 1286     // Fallthrough if no longer able to complete keyword. | 
| 1293     return ScanIdentifierSuffix(&literal); | 1287     return ScanIdentifierSuffix(&literal); | 
| 1294   } | 1288   } | 
| 1295 | 1289 | 
| 1296   literal.Complete(); | 1290   literal.Complete(); | 
| 1297 | 1291 | 
| 1298   if (next_.literal_chars->is_one_byte()) { | 1292   if (next_.literal_chars->is_one_byte()) { | 
| 1299     Vector<const uint8_t> chars = next_.literal_chars->one_byte_literal(); | 1293     Vector<const uint8_t> chars = next_.literal_chars->one_byte_literal(); | 
| 1300     return KeywordOrIdentifierToken(chars.start(), chars.length(), | 1294     return KeywordOrIdentifierToken(chars.start(), chars.length()); | 
| 1301                                     harmony_modules_); |  | 
| 1302   } | 1295   } | 
| 1303   return Token::IDENTIFIER; | 1296   return Token::IDENTIFIER; | 
| 1304 } | 1297 } | 
| 1305 | 1298 | 
| 1306 | 1299 | 
| 1307 Token::Value Scanner::ScanIdentifierSuffix(LiteralScope* literal) { | 1300 Token::Value Scanner::ScanIdentifierSuffix(LiteralScope* literal) { | 
| 1308   // Scan the rest of the identifier characters. | 1301   // Scan the rest of the identifier characters. | 
| 1309   while (c0_ >= 0 && unicode_cache_->IsIdentifierPart(c0_)) { | 1302   while (c0_ >= 0 && unicode_cache_->IsIdentifierPart(c0_)) { | 
| 1310     if (c0_ == '\\') { | 1303     if (c0_ == '\\') { | 
| 1311       uc32 c = ScanIdentifierUnicodeEscape(); | 1304       uc32 c = ScanIdentifierUnicodeEscape(); | 
| (...skipping 312 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
| 1624     backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); | 1617     backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); | 
| 1625   } | 1618   } | 
| 1626   backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); | 1619   backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); | 
| 1627 | 1620 | 
| 1628   backing_store_.AddBlock(bytes); | 1621   backing_store_.AddBlock(bytes); | 
| 1629   return backing_store_.EndSequence().start(); | 1622   return backing_store_.EndSequence().start(); | 
| 1630 } | 1623 } | 
| 1631 | 1624 | 
| 1632 }  // namespace internal | 1625 }  // namespace internal | 
| 1633 }  // namespace v8 | 1626 }  // namespace v8 | 
| OLD | NEW | 
|---|