OLD | NEW |
1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. |
6 | 6 |
7 #include "src/parsing/scanner.h" | 7 #include "src/parsing/scanner.h" |
8 | 8 |
9 #include <stdint.h> | 9 #include <stdint.h> |
10 | 10 |
(...skipping 231 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
242 | 242 |
243 Token::Value Scanner::Next() { | 243 Token::Value Scanner::Next() { |
244 if (next_.token == Token::EOS) { | 244 if (next_.token == Token::EOS) { |
245 next_.location.beg_pos = current_.location.beg_pos; | 245 next_.location.beg_pos = current_.location.beg_pos; |
246 next_.location.end_pos = current_.location.end_pos; | 246 next_.location.end_pos = current_.location.end_pos; |
247 } | 247 } |
248 current_ = next_; | 248 current_ = next_; |
249 if (V8_UNLIKELY(next_next_.token != Token::UNINITIALIZED)) { | 249 if (V8_UNLIKELY(next_next_.token != Token::UNINITIALIZED)) { |
250 next_ = next_next_; | 250 next_ = next_next_; |
251 next_next_.token = Token::UNINITIALIZED; | 251 next_next_.token = Token::UNINITIALIZED; |
| 252 has_line_terminator_before_next_ = has_line_terminator_before_next2_; |
| 253 has_multiline_comment_before_next_ = has_multiline_comment_before_next2_; |
252 return current_.token; | 254 return current_.token; |
253 } | 255 } |
254 has_line_terminator_before_next_ = false; | 256 has_line_terminator_before_next_ = false; |
255 has_multiline_comment_before_next_ = false; | 257 has_multiline_comment_before_next_ = false; |
256 if (static_cast<unsigned>(c0_) <= 0x7f) { | 258 if (static_cast<unsigned>(c0_) <= 0x7f) { |
257 Token::Value token = static_cast<Token::Value>(one_char_tokens[c0_]); | 259 Token::Value token = static_cast<Token::Value>(one_char_tokens[c0_]); |
258 if (token != Token::ILLEGAL) { | 260 if (token != Token::ILLEGAL) { |
259 int pos = source_pos(); | 261 int pos = source_pos(); |
260 next_.token = token; | 262 next_.token = token; |
261 next_.location.beg_pos = pos; | 263 next_.location.beg_pos = pos; |
262 next_.location.end_pos = pos + 1; | 264 next_.location.end_pos = pos + 1; |
263 Advance(); | 265 Advance(); |
264 return current_.token; | 266 return current_.token; |
265 } | 267 } |
266 } | 268 } |
267 Scan(); | 269 Scan(); |
268 return current_.token; | 270 return current_.token; |
269 } | 271 } |
270 | 272 |
271 | 273 |
272 Token::Value Scanner::PeekAhead() { | 274 Token::Value Scanner::PeekAhead() { |
273 if (next_next_.token != Token::UNINITIALIZED) { | 275 if (next_next_.token != Token::UNINITIALIZED) { |
274 return next_next_.token; | 276 return next_next_.token; |
275 } | 277 } |
276 TokenDesc prev = current_; | 278 TokenDesc prev = current_; |
| 279 bool has_line_terminator_before_next = has_line_terminator_before_next_; |
| 280 bool has_multiline_comment_before_next = has_multiline_comment_before_next_; |
277 Next(); | 281 Next(); |
| 282 has_line_terminator_before_next2_ = has_line_terminator_before_next_; |
| 283 has_multiline_comment_before_next2_ = has_multiline_comment_before_next_; |
| 284 has_line_terminator_before_next_ = has_line_terminator_before_next; |
| 285 has_multiline_comment_before_next_ = has_multiline_comment_before_next; |
278 Token::Value ret = next_.token; | 286 Token::Value ret = next_.token; |
279 next_next_ = next_; | 287 next_next_ = next_; |
280 next_ = current_; | 288 next_ = current_; |
281 current_ = prev; | 289 current_ = prev; |
282 return ret; | 290 return ret; |
283 } | 291 } |
284 | 292 |
285 | 293 |
286 // TODO(yangguo): check whether this is actually necessary. | 294 // TODO(yangguo): check whether this is actually necessary. |
287 static inline bool IsLittleEndianByteOrderMark(uc32 c) { | 295 static inline bool IsLittleEndianByteOrderMark(uc32 c) { |
(...skipping 1173 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1461 } | 1469 } |
1462 | 1470 |
1463 | 1471 |
1464 const AstRawString* Scanner::NextSymbol(AstValueFactory* ast_value_factory) { | 1472 const AstRawString* Scanner::NextSymbol(AstValueFactory* ast_value_factory) { |
1465 if (is_next_literal_one_byte()) { | 1473 if (is_next_literal_one_byte()) { |
1466 return ast_value_factory->GetOneByteString(next_literal_one_byte_string()); | 1474 return ast_value_factory->GetOneByteString(next_literal_one_byte_string()); |
1467 } | 1475 } |
1468 return ast_value_factory->GetTwoByteString(next_literal_two_byte_string()); | 1476 return ast_value_factory->GetTwoByteString(next_literal_two_byte_string()); |
1469 } | 1477 } |
1470 | 1478 |
| 1479 const AstRawString* Scanner::NextNextSymbol( |
| 1480 AstValueFactory* ast_value_factory) { |
| 1481 DCHECK(next_next_.token != Token::UNINITIALIZED); |
| 1482 LiteralBuffer* literal = next_next_.literal_chars; |
| 1483 if (literal->is_one_byte()) { |
| 1484 return ast_value_factory->GetOneByteString(literal->one_byte_literal()); |
| 1485 } |
| 1486 return ast_value_factory->GetTwoByteString(literal->two_byte_literal()); |
| 1487 } |
1471 | 1488 |
1472 const AstRawString* Scanner::CurrentRawSymbol( | 1489 const AstRawString* Scanner::CurrentRawSymbol( |
1473 AstValueFactory* ast_value_factory) { | 1490 AstValueFactory* ast_value_factory) { |
1474 if (is_raw_literal_one_byte()) { | 1491 if (is_raw_literal_one_byte()) { |
1475 return ast_value_factory->GetOneByteString(raw_literal_one_byte_string()); | 1492 return ast_value_factory->GetOneByteString(raw_literal_one_byte_string()); |
1476 } | 1493 } |
1477 return ast_value_factory->GetTwoByteString(raw_literal_two_byte_string()); | 1494 return ast_value_factory->GetTwoByteString(raw_literal_two_byte_string()); |
1478 } | 1495 } |
1479 | 1496 |
1480 | 1497 |
(...skipping 205 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1686 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); | 1703 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); |
1687 } | 1704 } |
1688 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); | 1705 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); |
1689 | 1706 |
1690 backing_store_.AddBlock(bytes); | 1707 backing_store_.AddBlock(bytes); |
1691 return backing_store_.EndSequence().start(); | 1708 return backing_store_.EndSequence().start(); |
1692 } | 1709 } |
1693 | 1710 |
1694 } // namespace internal | 1711 } // namespace internal |
1695 } // namespace v8 | 1712 } // namespace v8 |
OLD | NEW |