OLD | NEW |
1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. |
6 | 6 |
7 #include "src/parsing/scanner.h" | 7 #include "src/parsing/scanner.h" |
8 | 8 |
9 #include <stdint.h> | 9 #include <stdint.h> |
10 | 10 |
(...skipping 231 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
242 | 242 |
243 Token::Value Scanner::Next() { | 243 Token::Value Scanner::Next() { |
244 if (next_.token == Token::EOS) { | 244 if (next_.token == Token::EOS) { |
245 next_.location.beg_pos = current_.location.beg_pos; | 245 next_.location.beg_pos = current_.location.beg_pos; |
246 next_.location.end_pos = current_.location.end_pos; | 246 next_.location.end_pos = current_.location.end_pos; |
247 } | 247 } |
248 current_ = next_; | 248 current_ = next_; |
249 if (V8_UNLIKELY(next_next_.token != Token::UNINITIALIZED)) { | 249 if (V8_UNLIKELY(next_next_.token != Token::UNINITIALIZED)) { |
250 next_ = next_next_; | 250 next_ = next_next_; |
251 next_next_.token = Token::UNINITIALIZED; | 251 next_next_.token = Token::UNINITIALIZED; |
| 252 has_line_terminator_before_next_ = has_line_terminator_after_next_; |
252 return current_.token; | 253 return current_.token; |
253 } | 254 } |
254 has_line_terminator_before_next_ = false; | 255 has_line_terminator_before_next_ = false; |
255 has_multiline_comment_before_next_ = false; | 256 has_multiline_comment_before_next_ = false; |
256 if (static_cast<unsigned>(c0_) <= 0x7f) { | 257 if (static_cast<unsigned>(c0_) <= 0x7f) { |
257 Token::Value token = static_cast<Token::Value>(one_char_tokens[c0_]); | 258 Token::Value token = static_cast<Token::Value>(one_char_tokens[c0_]); |
258 if (token != Token::ILLEGAL) { | 259 if (token != Token::ILLEGAL) { |
259 int pos = source_pos(); | 260 int pos = source_pos(); |
260 next_.token = token; | 261 next_.token = token; |
261 next_.location.beg_pos = pos; | 262 next_.location.beg_pos = pos; |
262 next_.location.end_pos = pos + 1; | 263 next_.location.end_pos = pos + 1; |
263 Advance(); | 264 Advance(); |
264 return current_.token; | 265 return current_.token; |
265 } | 266 } |
266 } | 267 } |
267 Scan(); | 268 Scan(); |
268 return current_.token; | 269 return current_.token; |
269 } | 270 } |
270 | 271 |
271 | 272 |
272 Token::Value Scanner::PeekAhead() { | 273 Token::Value Scanner::PeekAhead() { |
273 if (next_next_.token != Token::UNINITIALIZED) { | 274 if (next_next_.token != Token::UNINITIALIZED) { |
274 return next_next_.token; | 275 return next_next_.token; |
275 } | 276 } |
276 TokenDesc prev = current_; | 277 TokenDesc prev = current_; |
| 278 bool has_line_terminator_before_next = |
| 279 has_line_terminator_before_next_ || has_multiline_comment_before_next_; |
277 Next(); | 280 Next(); |
| 281 has_line_terminator_after_next_ = |
| 282 has_line_terminator_before_next_ || has_multiline_comment_before_next_; |
| 283 has_line_terminator_before_next_ = has_line_terminator_before_next; |
278 Token::Value ret = next_.token; | 284 Token::Value ret = next_.token; |
279 next_next_ = next_; | 285 next_next_ = next_; |
280 next_ = current_; | 286 next_ = current_; |
281 current_ = prev; | 287 current_ = prev; |
282 return ret; | 288 return ret; |
283 } | 289 } |
284 | 290 |
285 | 291 |
286 // TODO(yangguo): check whether this is actually necessary. | 292 // TODO(yangguo): check whether this is actually necessary. |
287 static inline bool IsLittleEndianByteOrderMark(uc32 c) { | 293 static inline bool IsLittleEndianByteOrderMark(uc32 c) { |
(...skipping 841 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1129 const bool unicode = true; | 1135 const bool unicode = true; |
1130 return ScanHexNumber<capture_raw, unicode>(4); | 1136 return ScanHexNumber<capture_raw, unicode>(4); |
1131 } | 1137 } |
1132 | 1138 |
1133 | 1139 |
1134 // ---------------------------------------------------------------------------- | 1140 // ---------------------------------------------------------------------------- |
1135 // Keyword Matcher | 1141 // Keyword Matcher |
1136 | 1142 |
1137 #define KEYWORDS(KEYWORD_GROUP, KEYWORD) \ | 1143 #define KEYWORDS(KEYWORD_GROUP, KEYWORD) \ |
1138 KEYWORD_GROUP('a') \ | 1144 KEYWORD_GROUP('a') \ |
| 1145 KEYWORD("async", Token::ASYNC) \ |
1139 KEYWORD("await", Token::AWAIT) \ | 1146 KEYWORD("await", Token::AWAIT) \ |
1140 KEYWORD_GROUP('b') \ | 1147 KEYWORD_GROUP('b') \ |
1141 KEYWORD("break", Token::BREAK) \ | 1148 KEYWORD("break", Token::BREAK) \ |
1142 KEYWORD_GROUP('c') \ | 1149 KEYWORD_GROUP('c') \ |
1143 KEYWORD("case", Token::CASE) \ | 1150 KEYWORD("case", Token::CASE) \ |
1144 KEYWORD("catch", Token::CATCH) \ | 1151 KEYWORD("catch", Token::CATCH) \ |
1145 KEYWORD("class", Token::CLASS) \ | 1152 KEYWORD("class", Token::CLASS) \ |
1146 KEYWORD("const", Token::CONST) \ | 1153 KEYWORD("const", Token::CONST) \ |
1147 KEYWORD("continue", Token::CONTINUE) \ | 1154 KEYWORD("continue", Token::CONTINUE) \ |
1148 KEYWORD_GROUP('d') \ | 1155 KEYWORD_GROUP('d') \ |
(...skipping 538 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1687 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); | 1694 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); |
1688 } | 1695 } |
1689 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); | 1696 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); |
1690 | 1697 |
1691 backing_store_.AddBlock(bytes); | 1698 backing_store_.AddBlock(bytes); |
1692 return backing_store_.EndSequence().start(); | 1699 return backing_store_.EndSequence().start(); |
1693 } | 1700 } |
1694 | 1701 |
1695 } // namespace internal | 1702 } // namespace internal |
1696 } // namespace v8 | 1703 } // namespace v8 |
OLD | NEW |