OLD | NEW |
1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. |
6 | 6 |
7 #ifndef V8_SCANNER_H_ | 7 #ifndef V8_SCANNER_H_ |
8 #define V8_SCANNER_H_ | 8 #define V8_SCANNER_H_ |
9 | 9 |
10 #include "src/allocation.h" | 10 #include "src/allocation.h" |
(...skipping 381 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
392 | 392 |
393 // -1 is outside of the range of any real source code. | 393 // -1 is outside of the range of any real source code. |
394 static const int kNoOctalLocation = -1; | 394 static const int kNoOctalLocation = -1; |
395 | 395 |
396 explicit Scanner(UnicodeCache* scanner_contants); | 396 explicit Scanner(UnicodeCache* scanner_contants); |
397 | 397 |
398 void Initialize(Utf16CharacterStream* source); | 398 void Initialize(Utf16CharacterStream* source); |
399 | 399 |
400 // Returns the next token and advances input. | 400 // Returns the next token and advances input. |
401 Token::Value Next(); | 401 Token::Value Next(); |
| 402 // Returns the token following peek() |
| 403 Token::Value PeekAhead(); |
402 // Returns the current token again. | 404 // Returns the current token again. |
403 Token::Value current_token() { return current_.token; } | 405 Token::Value current_token() { return current_.token; } |
404 // Returns the location information for the current token | 406 // Returns the location information for the current token |
405 // (the token last returned by Next()). | 407 // (the token last returned by Next()). |
406 Location location() const { return current_.location; } | 408 Location location() const { return current_.location; } |
407 | 409 |
408 // Similar functions for the upcoming token. | 410 // Similar functions for the upcoming token. |
409 | 411 |
410 // One token look-ahead (past the token returned by Next()). | 412 // One token look-ahead (past the token returned by Next()). |
411 Token::Value peek() const { return next_.token; } | 413 Token::Value peek() const { return next_.token; } |
(...skipping 108 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
520 uc32 ScanOctalEscape(uc32 c, int length); | 522 uc32 ScanOctalEscape(uc32 c, int length); |
521 | 523 |
522 // Call this after setting source_ to the input. | 524 // Call this after setting source_ to the input. |
523 void Init() { | 525 void Init() { |
524 // Set c0_ (one character ahead) | 526 // Set c0_ (one character ahead) |
525 STATIC_ASSERT(kCharacterLookaheadBufferSize == 1); | 527 STATIC_ASSERT(kCharacterLookaheadBufferSize == 1); |
526 Advance(); | 528 Advance(); |
527 // Initialize current_ to not refer to a literal. | 529 // Initialize current_ to not refer to a literal. |
528 current_.literal_chars = NULL; | 530 current_.literal_chars = NULL; |
529 current_.raw_literal_chars = NULL; | 531 current_.raw_literal_chars = NULL; |
| 532 next_next_.token = Token::UNINITIALIZED; |
530 } | 533 } |
531 | 534 |
532 // Support BookmarkScope functionality. | 535 // Support BookmarkScope functionality. |
533 bool SetBookmark(); | 536 bool SetBookmark(); |
534 void ResetToBookmark(); | 537 void ResetToBookmark(); |
535 bool BookmarkHasBeenSet(); | 538 bool BookmarkHasBeenSet(); |
536 bool BookmarkHasBeenReset(); | 539 bool BookmarkHasBeenReset(); |
537 void DropBookmark(); | 540 void DropBookmark(); |
538 static void CopyTokenDesc(TokenDesc* to, TokenDesc* from); | 541 static void CopyTokenDesc(TokenDesc* to, TokenDesc* from); |
539 | 542 |
540 // Literal buffer support | 543 // Literal buffer support |
541 inline void StartLiteral() { | 544 inline void StartLiteral() { |
542 LiteralBuffer* free_buffer = (current_.literal_chars == &literal_buffer1_) ? | 545 LiteralBuffer* free_buffer = |
543 &literal_buffer2_ : &literal_buffer1_; | 546 (current_.literal_chars == &literal_buffer0_) |
| 547 ? &literal_buffer1_ |
| 548 : (current_.literal_chars == &literal_buffer1_) ? &literal_buffer2_ |
| 549 : &literal_buffer0_; |
544 free_buffer->Reset(); | 550 free_buffer->Reset(); |
545 next_.literal_chars = free_buffer; | 551 next_.literal_chars = free_buffer; |
546 } | 552 } |
547 | 553 |
548 inline void StartRawLiteral() { | 554 inline void StartRawLiteral() { |
549 LiteralBuffer* free_buffer = | 555 LiteralBuffer* free_buffer = |
550 (current_.raw_literal_chars == &raw_literal_buffer1_) ? | 556 (current_.raw_literal_chars == &raw_literal_buffer0_) |
551 &raw_literal_buffer2_ : &raw_literal_buffer1_; | 557 ? &raw_literal_buffer1_ |
| 558 : (current_.raw_literal_chars == &raw_literal_buffer1_) |
| 559 ? &raw_literal_buffer2_ |
| 560 : &raw_literal_buffer0_; |
552 free_buffer->Reset(); | 561 free_buffer->Reset(); |
553 next_.raw_literal_chars = free_buffer; | 562 next_.raw_literal_chars = free_buffer; |
554 } | 563 } |
555 | 564 |
556 INLINE(void AddLiteralChar(uc32 c)) { | 565 INLINE(void AddLiteralChar(uc32 c)) { |
557 DCHECK_NOT_NULL(next_.literal_chars); | 566 DCHECK_NOT_NULL(next_.literal_chars); |
558 next_.literal_chars->AddChar(c); | 567 next_.literal_chars->AddChar(c); |
559 } | 568 } |
560 | 569 |
561 INLINE(void AddRawLiteralChar(uc32 c)) { | 570 INLINE(void AddRawLiteralChar(uc32 c)) { |
(...skipping 156 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
718 Token::Value ScanTemplateSpan(); | 727 Token::Value ScanTemplateSpan(); |
719 | 728 |
720 // Return the current source position. | 729 // Return the current source position. |
721 int source_pos() { | 730 int source_pos() { |
722 return static_cast<int>(source_->pos()) - kCharacterLookaheadBufferSize; | 731 return static_cast<int>(source_->pos()) - kCharacterLookaheadBufferSize; |
723 } | 732 } |
724 | 733 |
725 UnicodeCache* unicode_cache_; | 734 UnicodeCache* unicode_cache_; |
726 | 735 |
727 // Buffers collecting literal strings, numbers, etc. | 736 // Buffers collecting literal strings, numbers, etc. |
| 737 LiteralBuffer literal_buffer0_; |
728 LiteralBuffer literal_buffer1_; | 738 LiteralBuffer literal_buffer1_; |
729 LiteralBuffer literal_buffer2_; | 739 LiteralBuffer literal_buffer2_; |
730 | 740 |
731 // Values parsed from magic comments. | 741 // Values parsed from magic comments. |
732 LiteralBuffer source_url_; | 742 LiteralBuffer source_url_; |
733 LiteralBuffer source_mapping_url_; | 743 LiteralBuffer source_mapping_url_; |
734 | 744 |
735 // Buffer to store raw string values | 745 // Buffer to store raw string values |
| 746 LiteralBuffer raw_literal_buffer0_; |
736 LiteralBuffer raw_literal_buffer1_; | 747 LiteralBuffer raw_literal_buffer1_; |
737 LiteralBuffer raw_literal_buffer2_; | 748 LiteralBuffer raw_literal_buffer2_; |
738 | 749 |
739 TokenDesc current_; // desc for current token (as returned by Next()) | 750 TokenDesc current_; // desc for current token (as returned by Next()) |
740 TokenDesc next_; // desc for next token (one token look-ahead) | 751 TokenDesc next_; // desc for next token (one token look-ahead) |
| 752 TokenDesc next_next_; // desc for the token after next (push-back) |
741 | 753 |
742 // Variables for Scanner::BookmarkScope and the *Bookmark implementation. | 754 // Variables for Scanner::BookmarkScope and the *Bookmark implementation. |
743 // These variables contain the scanner state when a bookmark is set. | 755 // These variables contain the scanner state when a bookmark is set. |
744 // | 756 // |
745 // We will use bookmark_c0_ as a 'control' variable, where: | 757 // We will use bookmark_c0_ as a 'control' variable, where: |
746 // - bookmark_c0_ >= 0: A bookmark has been set and this contains c0_. | 758 // - bookmark_c0_ >= 0: A bookmark has been set and this contains c0_. |
747 // - bookmark_c0_ == -1: No bookmark has been set. | 759 // - bookmark_c0_ == -1: No bookmark has been set. |
748 // - bookmark_c0_ == -2: The bookmark has been applied (ResetToBookmark). | 760 // - bookmark_c0_ == -2: The bookmark has been applied (ResetToBookmark). |
749 // | 761 // |
750 // Which state is being bookmarked? The parser state is distributed over | 762 // Which state is being bookmarked? The parser state is distributed over |
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
785 // inside multiline comments. | 797 // inside multiline comments. |
786 bool has_line_terminator_before_next_; | 798 bool has_line_terminator_before_next_; |
787 // Whether there is a multi-line comment that contains a | 799 // Whether there is a multi-line comment that contains a |
788 // line-terminator after the current token, and before the next. | 800 // line-terminator after the current token, and before the next. |
789 bool has_multiline_comment_before_next_; | 801 bool has_multiline_comment_before_next_; |
790 }; | 802 }; |
791 | 803 |
792 } } // namespace v8::internal | 804 } } // namespace v8::internal |
793 | 805 |
794 #endif // V8_SCANNER_H_ | 806 #endif // V8_SCANNER_H_ |
OLD | NEW |