Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(990)

Side by Side Diff: src/scanner.cc

Issue 1262913003: [es6] Remove Scanner and Parser flags for harmony_modules (Closed) Base URL: https://chromium.googlesource.com/v8/v8.git@master
Patch Set: Add flag-setting to cctests Created 5 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « src/scanner.h ('k') | test/cctest/test-parsing.cc » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright 2011 the V8 project authors. All rights reserved. 1 // Copyright 2011 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be 2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file. 3 // found in the LICENSE file.
4 4
5 // Features shared by parsing and pre-parsing scanners. 5 // Features shared by parsing and pre-parsing scanners.
6 6
7 #include <stdint.h> 7 #include <stdint.h>
8 8
9 #include <cmath> 9 #include <cmath>
10 10
(...skipping 23 matching lines...) Expand all
34 void Utf16CharacterStream::ResetToBookmark() { UNREACHABLE(); } 34 void Utf16CharacterStream::ResetToBookmark() { UNREACHABLE(); }
35 35
36 36
37 // ---------------------------------------------------------------------------- 37 // ----------------------------------------------------------------------------
38 // Scanner 38 // Scanner
39 39
40 Scanner::Scanner(UnicodeCache* unicode_cache) 40 Scanner::Scanner(UnicodeCache* unicode_cache)
41 : unicode_cache_(unicode_cache), 41 : unicode_cache_(unicode_cache),
42 bookmark_c0_(kNoBookmark), 42 bookmark_c0_(kNoBookmark),
43 octal_pos_(Location::invalid()), 43 octal_pos_(Location::invalid()),
44 harmony_modules_(false),
45 harmony_unicode_(false) { 44 harmony_unicode_(false) {
46 bookmark_current_.literal_chars = &bookmark_current_literal_; 45 bookmark_current_.literal_chars = &bookmark_current_literal_;
47 bookmark_current_.raw_literal_chars = &bookmark_current_raw_literal_; 46 bookmark_current_.raw_literal_chars = &bookmark_current_raw_literal_;
48 bookmark_next_.literal_chars = &bookmark_next_literal_; 47 bookmark_next_.literal_chars = &bookmark_next_literal_;
49 bookmark_next_.raw_literal_chars = &bookmark_next_raw_literal_; 48 bookmark_next_.raw_literal_chars = &bookmark_next_raw_literal_;
50 } 49 }
51 50
52 51
53 void Scanner::Initialize(Utf16CharacterStream* source) { 52 void Scanner::Initialize(Utf16CharacterStream* source) {
54 source_ = source; 53 source_ = source;
(...skipping 1036 matching lines...) Expand 10 before | Expand all | Expand 10 after
1091 Advance<capture_raw>(); 1090 Advance<capture_raw>();
1092 return cp; 1091 return cp;
1093 } 1092 }
1094 return ScanHexNumber<capture_raw>(4); 1093 return ScanHexNumber<capture_raw>(4);
1095 } 1094 }
1096 1095
1097 1096
1098 // ---------------------------------------------------------------------------- 1097 // ----------------------------------------------------------------------------
1099 // Keyword Matcher 1098 // Keyword Matcher
1100 1099
1101 #define KEYWORDS(KEYWORD_GROUP, KEYWORD) \ 1100 #define KEYWORDS(KEYWORD_GROUP, KEYWORD) \
1102 KEYWORD_GROUP('b') \ 1101 KEYWORD_GROUP('b') \
1103 KEYWORD("break", Token::BREAK) \ 1102 KEYWORD("break", Token::BREAK) \
1104 KEYWORD_GROUP('c') \ 1103 KEYWORD_GROUP('c') \
1105 KEYWORD("case", Token::CASE) \ 1104 KEYWORD("case", Token::CASE) \
1106 KEYWORD("catch", Token::CATCH) \ 1105 KEYWORD("catch", Token::CATCH) \
1107 KEYWORD("class", Token::CLASS) \ 1106 KEYWORD("class", Token::CLASS) \
1108 KEYWORD("const", Token::CONST) \ 1107 KEYWORD("const", Token::CONST) \
1109 KEYWORD("continue", Token::CONTINUE) \ 1108 KEYWORD("continue", Token::CONTINUE) \
1110 KEYWORD_GROUP('d') \ 1109 KEYWORD_GROUP('d') \
1111 KEYWORD("debugger", Token::DEBUGGER) \ 1110 KEYWORD("debugger", Token::DEBUGGER) \
1112 KEYWORD("default", Token::DEFAULT) \ 1111 KEYWORD("default", Token::DEFAULT) \
1113 KEYWORD("delete", Token::DELETE) \ 1112 KEYWORD("delete", Token::DELETE) \
1114 KEYWORD("do", Token::DO) \ 1113 KEYWORD("do", Token::DO) \
1115 KEYWORD_GROUP('e') \ 1114 KEYWORD_GROUP('e') \
1116 KEYWORD("else", Token::ELSE) \ 1115 KEYWORD("else", Token::ELSE) \
1117 KEYWORD("enum", Token::FUTURE_RESERVED_WORD) \ 1116 KEYWORD("enum", Token::FUTURE_RESERVED_WORD) \
1118 KEYWORD("export", \ 1117 KEYWORD("export", Token::EXPORT) \
1119 harmony_modules ? Token::EXPORT : Token::FUTURE_RESERVED_WORD) \ 1118 KEYWORD("extends", Token::EXTENDS) \
1120 KEYWORD("extends", Token::EXTENDS) \ 1119 KEYWORD_GROUP('f') \
1121 KEYWORD_GROUP('f') \ 1120 KEYWORD("false", Token::FALSE_LITERAL) \
1122 KEYWORD("false", Token::FALSE_LITERAL) \ 1121 KEYWORD("finally", Token::FINALLY) \
1123 KEYWORD("finally", Token::FINALLY) \ 1122 KEYWORD("for", Token::FOR) \
1124 KEYWORD("for", Token::FOR) \ 1123 KEYWORD("function", Token::FUNCTION) \
1125 KEYWORD("function", Token::FUNCTION) \ 1124 KEYWORD_GROUP('i') \
1126 KEYWORD_GROUP('i') \ 1125 KEYWORD("if", Token::IF) \
1127 KEYWORD("if", Token::IF) \ 1126 KEYWORD("implements", Token::FUTURE_STRICT_RESERVED_WORD) \
1128 KEYWORD("implements", Token::FUTURE_STRICT_RESERVED_WORD) \ 1127 KEYWORD("import", Token::IMPORT) \
1129 KEYWORD("import", \ 1128 KEYWORD("in", Token::IN) \
1130 harmony_modules ? Token::IMPORT : Token::FUTURE_RESERVED_WORD) \ 1129 KEYWORD("instanceof", Token::INSTANCEOF) \
1131 KEYWORD("in", Token::IN) \ 1130 KEYWORD("interface", Token::FUTURE_STRICT_RESERVED_WORD) \
1132 KEYWORD("instanceof", Token::INSTANCEOF) \ 1131 KEYWORD_GROUP('l') \
1133 KEYWORD("interface", Token::FUTURE_STRICT_RESERVED_WORD) \ 1132 KEYWORD("let", Token::LET) \
1134 KEYWORD_GROUP('l') \ 1133 KEYWORD_GROUP('n') \
1135 KEYWORD("let", Token::LET) \ 1134 KEYWORD("new", Token::NEW) \
1136 KEYWORD_GROUP('n') \ 1135 KEYWORD("null", Token::NULL_LITERAL) \
1137 KEYWORD("new", Token::NEW) \ 1136 KEYWORD_GROUP('p') \
1138 KEYWORD("null", Token::NULL_LITERAL) \ 1137 KEYWORD("package", Token::FUTURE_STRICT_RESERVED_WORD) \
1139 KEYWORD_GROUP('p') \ 1138 KEYWORD("private", Token::FUTURE_STRICT_RESERVED_WORD) \
1140 KEYWORD("package", Token::FUTURE_STRICT_RESERVED_WORD) \ 1139 KEYWORD("protected", Token::FUTURE_STRICT_RESERVED_WORD) \
1141 KEYWORD("private", Token::FUTURE_STRICT_RESERVED_WORD) \ 1140 KEYWORD("public", Token::FUTURE_STRICT_RESERVED_WORD) \
1142 KEYWORD("protected", Token::FUTURE_STRICT_RESERVED_WORD) \ 1141 KEYWORD_GROUP('r') \
1143 KEYWORD("public", Token::FUTURE_STRICT_RESERVED_WORD) \ 1142 KEYWORD("return", Token::RETURN) \
1144 KEYWORD_GROUP('r') \ 1143 KEYWORD_GROUP('s') \
1145 KEYWORD("return", Token::RETURN) \ 1144 KEYWORD("static", Token::STATIC) \
1146 KEYWORD_GROUP('s') \ 1145 KEYWORD("super", Token::SUPER) \
1147 KEYWORD("static", Token::STATIC) \ 1146 KEYWORD("switch", Token::SWITCH) \
1148 KEYWORD("super", Token::SUPER) \ 1147 KEYWORD_GROUP('t') \
1149 KEYWORD("switch", Token::SWITCH) \ 1148 KEYWORD("this", Token::THIS) \
1150 KEYWORD_GROUP('t') \ 1149 KEYWORD("throw", Token::THROW) \
1151 KEYWORD("this", Token::THIS) \ 1150 KEYWORD("true", Token::TRUE_LITERAL) \
1152 KEYWORD("throw", Token::THROW) \ 1151 KEYWORD("try", Token::TRY) \
1153 KEYWORD("true", Token::TRUE_LITERAL) \ 1152 KEYWORD("typeof", Token::TYPEOF) \
1154 KEYWORD("try", Token::TRY) \ 1153 KEYWORD_GROUP('v') \
1155 KEYWORD("typeof", Token::TYPEOF) \ 1154 KEYWORD("var", Token::VAR) \
1156 KEYWORD_GROUP('v') \ 1155 KEYWORD("void", Token::VOID) \
1157 KEYWORD("var", Token::VAR) \ 1156 KEYWORD_GROUP('w') \
1158 KEYWORD("void", Token::VOID) \ 1157 KEYWORD("while", Token::WHILE) \
1159 KEYWORD_GROUP('w') \ 1158 KEYWORD("with", Token::WITH) \
1160 KEYWORD("while", Token::WHILE) \ 1159 KEYWORD_GROUP('y') \
1161 KEYWORD("with", Token::WITH) \
1162 KEYWORD_GROUP('y') \
1163 KEYWORD("yield", Token::YIELD) 1160 KEYWORD("yield", Token::YIELD)
1164 1161
1165 1162
1166 static Token::Value KeywordOrIdentifierToken(const uint8_t* input, 1163 static Token::Value KeywordOrIdentifierToken(const uint8_t* input,
1167 int input_length, 1164 int input_length) {
1168 bool harmony_modules) {
1169 DCHECK(input_length >= 1); 1165 DCHECK(input_length >= 1);
1170 const int kMinLength = 2; 1166 const int kMinLength = 2;
1171 const int kMaxLength = 10; 1167 const int kMaxLength = 10;
1172 if (input_length < kMinLength || input_length > kMaxLength) { 1168 if (input_length < kMinLength || input_length > kMaxLength) {
1173 return Token::IDENTIFIER; 1169 return Token::IDENTIFIER;
1174 } 1170 }
1175 switch (input[0]) { 1171 switch (input[0]) {
1176 default: 1172 default:
1177 #define KEYWORD_GROUP_CASE(ch) \ 1173 #define KEYWORD_GROUP_CASE(ch) \
1178 break; \ 1174 break; \
(...skipping 26 matching lines...) Expand all
1205 1201
1206 bool Scanner::IdentifierIsFutureStrictReserved( 1202 bool Scanner::IdentifierIsFutureStrictReserved(
1207 const AstRawString* string) const { 1203 const AstRawString* string) const {
1208 // Keywords are always 1-byte strings. 1204 // Keywords are always 1-byte strings.
1209 if (!string->is_one_byte()) return false; 1205 if (!string->is_one_byte()) return false;
1210 if (string->IsOneByteEqualTo("let") || string->IsOneByteEqualTo("static") || 1206 if (string->IsOneByteEqualTo("let") || string->IsOneByteEqualTo("static") ||
1211 string->IsOneByteEqualTo("yield")) { 1207 string->IsOneByteEqualTo("yield")) {
1212 return true; 1208 return true;
1213 } 1209 }
1214 return Token::FUTURE_STRICT_RESERVED_WORD == 1210 return Token::FUTURE_STRICT_RESERVED_WORD ==
1215 KeywordOrIdentifierToken(string->raw_data(), string->length(), 1211 KeywordOrIdentifierToken(string->raw_data(), string->length());
1216 harmony_modules_);
1217 } 1212 }
1218 1213
1219 1214
1220 Token::Value Scanner::ScanIdentifierOrKeyword() { 1215 Token::Value Scanner::ScanIdentifierOrKeyword() {
1221 DCHECK(unicode_cache_->IsIdentifierStart(c0_)); 1216 DCHECK(unicode_cache_->IsIdentifierStart(c0_));
1222 LiteralScope literal(this); 1217 LiteralScope literal(this);
1223 if (IsInRange(c0_, 'a', 'z')) { 1218 if (IsInRange(c0_, 'a', 'z')) {
1224 do { 1219 do {
1225 uc32 first_char = c0_; 1220 uc32 first_char = c0_;
1226 Advance<false, false>(); 1221 Advance<false, false>();
(...skipping 12 matching lines...) Expand all
1239 AddLiteralChar(first_char); 1234 AddLiteralChar(first_char);
1240 } 1235 }
1241 if (c0_ <= kMaxAscii && c0_ != '\\') { 1236 if (c0_ <= kMaxAscii && c0_ != '\\') {
1242 literal.Complete(); 1237 literal.Complete();
1243 return Token::IDENTIFIER; 1238 return Token::IDENTIFIER;
1244 } 1239 }
1245 } else if (c0_ <= kMaxAscii && c0_ != '\\') { 1240 } else if (c0_ <= kMaxAscii && c0_ != '\\') {
1246 // Only a-z+: could be a keyword or identifier. 1241 // Only a-z+: could be a keyword or identifier.
1247 literal.Complete(); 1242 literal.Complete();
1248 Vector<const uint8_t> chars = next_.literal_chars->one_byte_literal(); 1243 Vector<const uint8_t> chars = next_.literal_chars->one_byte_literal();
1249 return KeywordOrIdentifierToken(chars.start(), chars.length(), 1244 return KeywordOrIdentifierToken(chars.start(), chars.length());
1250 harmony_modules_);
1251 } 1245 }
1252 1246
1253 HandleLeadSurrogate(); 1247 HandleLeadSurrogate();
1254 } else if (IsInRange(c0_, 'A', 'Z') || c0_ == '_' || c0_ == '$') { 1248 } else if (IsInRange(c0_, 'A', 'Z') || c0_ == '_' || c0_ == '$') {
1255 do { 1249 do {
1256 uc32 first_char = c0_; 1250 uc32 first_char = c0_;
1257 Advance<false, false>(); 1251 Advance<false, false>();
1258 AddLiteralChar(first_char); 1252 AddLiteralChar(first_char);
1259 } while (IsAsciiIdentifier(c0_)); 1253 } while (IsAsciiIdentifier(c0_));
1260 1254
(...skipping 29 matching lines...) Expand all
1290 continue; 1284 continue;
1291 } 1285 }
1292 // Fallthrough if no longer able to complete keyword. 1286 // Fallthrough if no longer able to complete keyword.
1293 return ScanIdentifierSuffix(&literal); 1287 return ScanIdentifierSuffix(&literal);
1294 } 1288 }
1295 1289
1296 literal.Complete(); 1290 literal.Complete();
1297 1291
1298 if (next_.literal_chars->is_one_byte()) { 1292 if (next_.literal_chars->is_one_byte()) {
1299 Vector<const uint8_t> chars = next_.literal_chars->one_byte_literal(); 1293 Vector<const uint8_t> chars = next_.literal_chars->one_byte_literal();
1300 return KeywordOrIdentifierToken(chars.start(), chars.length(), 1294 return KeywordOrIdentifierToken(chars.start(), chars.length());
1301 harmony_modules_);
1302 } 1295 }
1303 return Token::IDENTIFIER; 1296 return Token::IDENTIFIER;
1304 } 1297 }
1305 1298
1306 1299
1307 Token::Value Scanner::ScanIdentifierSuffix(LiteralScope* literal) { 1300 Token::Value Scanner::ScanIdentifierSuffix(LiteralScope* literal) {
1308 // Scan the rest of the identifier characters. 1301 // Scan the rest of the identifier characters.
1309 while (c0_ >= 0 && unicode_cache_->IsIdentifierPart(c0_)) { 1302 while (c0_ >= 0 && unicode_cache_->IsIdentifierPart(c0_)) {
1310 if (c0_ == '\\') { 1303 if (c0_ == '\\') {
1311 uc32 c = ScanIdentifierUnicodeEscape(); 1304 uc32 c = ScanIdentifierUnicodeEscape();
(...skipping 312 matching lines...) Expand 10 before | Expand all | Expand 10 after
1624 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); 1617 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u));
1625 } 1618 }
1626 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); 1619 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f));
1627 1620
1628 backing_store_.AddBlock(bytes); 1621 backing_store_.AddBlock(bytes);
1629 return backing_store_.EndSequence().start(); 1622 return backing_store_.EndSequence().start();
1630 } 1623 }
1631 1624
1632 } // namespace internal 1625 } // namespace internal
1633 } // namespace v8 1626 } // namespace v8
OLDNEW
« no previous file with comments | « src/scanner.h ('k') | test/cctest/test-parsing.cc » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698