OLD | NEW |
1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. |
6 | 6 |
7 #ifndef V8_SCANNER_H_ | 7 #ifndef V8_SCANNER_H_ |
8 #define V8_SCANNER_H_ | 8 #define V8_SCANNER_H_ |
9 | 9 |
10 #include "src/allocation.h" | 10 #include "src/allocation.h" |
(...skipping 343 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
354 | 354 |
355 // -1 is outside of the range of any real source code. | 355 // -1 is outside of the range of any real source code. |
356 static const int kNoOctalLocation = -1; | 356 static const int kNoOctalLocation = -1; |
357 | 357 |
358 explicit Scanner(UnicodeCache* scanner_contants); | 358 explicit Scanner(UnicodeCache* scanner_contants); |
359 | 359 |
360 void Initialize(Utf16CharacterStream* source); | 360 void Initialize(Utf16CharacterStream* source); |
361 | 361 |
362 // Returns the next token and advances input. | 362 // Returns the next token and advances input. |
363 Token::Value Next(); | 363 Token::Value Next(); |
| 364 // Returns the token following peek() |
| 365 Token::Value PeekAhead(); |
364 // Returns the current token again. | 366 // Returns the current token again. |
365 Token::Value current_token() { return current_.token; } | 367 Token::Value current_token() { return current_.token; } |
366 // Returns the location information for the current token | 368 // Returns the location information for the current token |
367 // (the token last returned by Next()). | 369 // (the token last returned by Next()). |
368 Location location() const { return current_.location; } | 370 Location location() const { return current_.location; } |
369 | 371 |
370 // Similar functions for the upcoming token. | 372 // Similar functions for the upcoming token. |
371 | 373 |
372 // One token look-ahead (past the token returned by Next()). | 374 // One token look-ahead (past the token returned by Next()). |
373 Token::Value peek() const { return next_.token; } | 375 Token::Value peek() const { return next_.token; } |
(...skipping 108 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
482 uc32 ScanOctalEscape(uc32 c, int length); | 484 uc32 ScanOctalEscape(uc32 c, int length); |
483 | 485 |
484 // Call this after setting source_ to the input. | 486 // Call this after setting source_ to the input. |
485 void Init() { | 487 void Init() { |
486 // Set c0_ (one character ahead) | 488 // Set c0_ (one character ahead) |
487 STATIC_ASSERT(kCharacterLookaheadBufferSize == 1); | 489 STATIC_ASSERT(kCharacterLookaheadBufferSize == 1); |
488 Advance(); | 490 Advance(); |
489 // Initialize current_ to not refer to a literal. | 491 // Initialize current_ to not refer to a literal. |
490 current_.literal_chars = NULL; | 492 current_.literal_chars = NULL; |
491 current_.raw_literal_chars = NULL; | 493 current_.raw_literal_chars = NULL; |
| 494 next_next_.token = Token::UNINITIALIZED; |
492 } | 495 } |
493 | 496 |
494 // Support BookmarkScope functionality. | 497 // Support BookmarkScope functionality. |
495 bool SetBookmark(); | 498 bool SetBookmark(); |
496 void ResetToBookmark(); | 499 void ResetToBookmark(); |
497 bool BookmarkHasBeenSet(); | 500 bool BookmarkHasBeenSet(); |
498 bool BookmarkHasBeenReset(); | 501 bool BookmarkHasBeenReset(); |
499 void DropBookmark(); | 502 void DropBookmark(); |
500 static void CopyTokenDesc(TokenDesc* to, TokenDesc* from); | 503 static void CopyTokenDesc(TokenDesc* to, TokenDesc* from); |
501 | 504 |
502 // Literal buffer support | 505 // Literal buffer support |
503 inline void StartLiteral() { | 506 inline void StartLiteral() { |
504 LiteralBuffer* free_buffer = (current_.literal_chars == &literal_buffer1_) ? | 507 LiteralBuffer* free_buffer = |
505 &literal_buffer2_ : &literal_buffer1_; | 508 (current_.literal_chars == &literal_buffer0_) |
| 509 ? &literal_buffer1_ |
| 510 : (current_.literal_chars == &literal_buffer1_) ? &literal_buffer2_ |
| 511 : &literal_buffer0_; |
506 free_buffer->Reset(); | 512 free_buffer->Reset(); |
507 next_.literal_chars = free_buffer; | 513 next_.literal_chars = free_buffer; |
508 } | 514 } |
509 | 515 |
510 inline void StartRawLiteral() { | 516 inline void StartRawLiteral() { |
511 LiteralBuffer* free_buffer = | 517 LiteralBuffer* free_buffer = |
512 (current_.raw_literal_chars == &raw_literal_buffer1_) ? | 518 (current_.raw_literal_chars == &raw_literal_buffer0_) |
513 &raw_literal_buffer2_ : &raw_literal_buffer1_; | 519 ? &raw_literal_buffer1_ |
| 520 : (current_.raw_literal_chars == &raw_literal_buffer1_) |
| 521 ? &raw_literal_buffer2_ |
| 522 : &raw_literal_buffer0_; |
514 free_buffer->Reset(); | 523 free_buffer->Reset(); |
515 next_.raw_literal_chars = free_buffer; | 524 next_.raw_literal_chars = free_buffer; |
516 } | 525 } |
517 | 526 |
518 INLINE(void AddLiteralChar(uc32 c)) { | 527 INLINE(void AddLiteralChar(uc32 c)) { |
519 DCHECK_NOT_NULL(next_.literal_chars); | 528 DCHECK_NOT_NULL(next_.literal_chars); |
520 next_.literal_chars->AddChar(c); | 529 next_.literal_chars->AddChar(c); |
521 } | 530 } |
522 | 531 |
523 INLINE(void AddRawLiteralChar(uc32 c)) { | 532 INLINE(void AddRawLiteralChar(uc32 c)) { |
(...skipping 156 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
680 Token::Value ScanTemplateSpan(); | 689 Token::Value ScanTemplateSpan(); |
681 | 690 |
682 // Return the current source position. | 691 // Return the current source position. |
683 int source_pos() { | 692 int source_pos() { |
684 return static_cast<int>(source_->pos()) - kCharacterLookaheadBufferSize; | 693 return static_cast<int>(source_->pos()) - kCharacterLookaheadBufferSize; |
685 } | 694 } |
686 | 695 |
687 UnicodeCache* unicode_cache_; | 696 UnicodeCache* unicode_cache_; |
688 | 697 |
689 // Buffers collecting literal strings, numbers, etc. | 698 // Buffers collecting literal strings, numbers, etc. |
| 699 LiteralBuffer literal_buffer0_; |
690 LiteralBuffer literal_buffer1_; | 700 LiteralBuffer literal_buffer1_; |
691 LiteralBuffer literal_buffer2_; | 701 LiteralBuffer literal_buffer2_; |
692 | 702 |
693 // Values parsed from magic comments. | 703 // Values parsed from magic comments. |
694 LiteralBuffer source_url_; | 704 LiteralBuffer source_url_; |
695 LiteralBuffer source_mapping_url_; | 705 LiteralBuffer source_mapping_url_; |
696 | 706 |
697 // Buffer to store raw string values | 707 // Buffer to store raw string values |
| 708 LiteralBuffer raw_literal_buffer0_; |
698 LiteralBuffer raw_literal_buffer1_; | 709 LiteralBuffer raw_literal_buffer1_; |
699 LiteralBuffer raw_literal_buffer2_; | 710 LiteralBuffer raw_literal_buffer2_; |
700 | 711 |
701 TokenDesc current_; // desc for current token (as returned by Next()) | 712 TokenDesc current_; // desc for current token (as returned by Next()) |
702 TokenDesc next_; // desc for next token (one token look-ahead) | 713 TokenDesc next_; // desc for next token (one token look-ahead) |
| 714 TokenDesc next_next_; // desc for the token after next (after PeakAhead()) |
703 | 715 |
704 // Variables for Scanner::BookmarkScope and the *Bookmark implementation. | 716 // Variables for Scanner::BookmarkScope and the *Bookmark implementation. |
705 // These variables contain the scanner state when a bookmark is set. | 717 // These variables contain the scanner state when a bookmark is set. |
706 // | 718 // |
707 // We will use bookmark_c0_ as a 'control' variable, where: | 719 // We will use bookmark_c0_ as a 'control' variable, where: |
708 // - bookmark_c0_ >= 0: A bookmark has been set and this contains c0_. | 720 // - bookmark_c0_ >= 0: A bookmark has been set and this contains c0_. |
709 // - bookmark_c0_ == -1: No bookmark has been set. | 721 // - bookmark_c0_ == -1: No bookmark has been set. |
710 // - bookmark_c0_ == -2: The bookmark has been applied (ResetToBookmark). | 722 // - bookmark_c0_ == -2: The bookmark has been applied (ResetToBookmark). |
711 // | 723 // |
712 // Which state is being bookmarked? The parser state is distributed over | 724 // Which state is being bookmarked? The parser state is distributed over |
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
747 // inside multiline comments. | 759 // inside multiline comments. |
748 bool has_line_terminator_before_next_; | 760 bool has_line_terminator_before_next_; |
749 // Whether there is a multi-line comment that contains a | 761 // Whether there is a multi-line comment that contains a |
750 // line-terminator after the current token, and before the next. | 762 // line-terminator after the current token, and before the next. |
751 bool has_multiline_comment_before_next_; | 763 bool has_multiline_comment_before_next_; |
752 }; | 764 }; |
753 | 765 |
754 } } // namespace v8::internal | 766 } } // namespace v8::internal |
755 | 767 |
756 #endif // V8_SCANNER_H_ | 768 #endif // V8_SCANNER_H_ |
OLD | NEW |