OLD | NEW |
1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. |
6 | 6 |
7 #ifndef V8_SCANNER_H_ | 7 #ifndef V8_SCANNER_H_ |
8 #define V8_SCANNER_H_ | 8 #define V8_SCANNER_H_ |
9 | 9 |
10 #include "src/allocation.h" | 10 #include "src/allocation.h" |
(...skipping 71 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
82 } | 82 } |
83 return SlowSeekForward(code_unit_count); | 83 return SlowSeekForward(code_unit_count); |
84 } | 84 } |
85 | 85 |
86 // Pushes back the most recently read UTF-16 code unit (or negative | 86 // Pushes back the most recently read UTF-16 code unit (or negative |
87 // value if at end of input), i.e., the value returned by the most recent | 87 // value if at end of input), i.e., the value returned by the most recent |
88 // call to Advance. | 88 // call to Advance. |
89 // Must not be used right after calling SeekForward. | 89 // Must not be used right after calling SeekForward. |
90 virtual void PushBack(int32_t code_unit) = 0; | 90 virtual void PushBack(int32_t code_unit) = 0; |
91 | 91 |
| 92 virtual bool SetBookmark(); |
| 93 virtual void ResetToBookmark(); |
| 94 |
92 protected: | 95 protected: |
93 static const uc32 kEndOfInput = -1; | 96 static const uc32 kEndOfInput = -1; |
94 | 97 |
95 // Ensures that the buffer_cursor_ points to the code_unit at | 98 // Ensures that the buffer_cursor_ points to the code_unit at |
96 // position pos_ of the input, if possible. If the position | 99 // position pos_ of the input, if possible. If the position |
97 // is at or after the end of the input, return false. If there | 100 // is at or after the end of the input, return false. If there |
98 // are more code_units available, return true. | 101 // are more code_units available, return true. |
99 virtual bool ReadBlock() = 0; | 102 virtual bool ReadBlock() = 0; |
100 virtual size_t SlowSeekForward(size_t code_unit_count) = 0; | 103 virtual size_t SlowSeekForward(size_t code_unit_count) = 0; |
101 | 104 |
(...skipping 159 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
261 position_ -= delta * (is_one_byte_ ? kOneByteSize : kUC16Size); | 264 position_ -= delta * (is_one_byte_ ? kOneByteSize : kUC16Size); |
262 } | 265 } |
263 | 266 |
264 void Reset() { | 267 void Reset() { |
265 position_ = 0; | 268 position_ = 0; |
266 is_one_byte_ = true; | 269 is_one_byte_ = true; |
267 } | 270 } |
268 | 271 |
269 Handle<String> Internalize(Isolate* isolate) const; | 272 Handle<String> Internalize(Isolate* isolate) const; |
270 | 273 |
| 274 void CopyFrom(const LiteralBuffer* other) { |
| 275 if (other == nullptr) { |
| 276 Reset(); |
| 277 } else { |
| 278 is_one_byte_ = other->is_one_byte_; |
| 279 position_ = other->position_; |
| 280 backing_store_.Dispose(); |
| 281 backing_store_ = other->backing_store_.Clone(); |
| 282 } |
| 283 } |
| 284 |
271 private: | 285 private: |
272 static const int kInitialCapacity = 16; | 286 static const int kInitialCapacity = 16; |
273 static const int kGrowthFactory = 4; | 287 static const int kGrowthFactory = 4; |
274 static const int kMinConversionSlack = 256; | 288 static const int kMinConversionSlack = 256; |
275 static const int kMaxGrowth = 1 * MB; | 289 static const int kMaxGrowth = 1 * MB; |
276 inline int NewCapacity(int min_capacity) { | 290 inline int NewCapacity(int min_capacity) { |
277 int capacity = Max(min_capacity, backing_store_.length()); | 291 int capacity = Max(min_capacity, backing_store_.length()); |
278 int new_capacity = Min(capacity * kGrowthFactory, capacity + kMaxGrowth); | 292 int new_capacity = Min(capacity * kGrowthFactory, capacity + kMaxGrowth); |
279 return new_capacity; | 293 return new_capacity; |
280 } | 294 } |
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
335 } | 349 } |
336 void Complete() { | 350 void Complete() { |
337 complete_ = true; | 351 complete_ = true; |
338 } | 352 } |
339 | 353 |
340 private: | 354 private: |
341 Scanner* scanner_; | 355 Scanner* scanner_; |
342 bool complete_; | 356 bool complete_; |
343 }; | 357 }; |
344 | 358 |
| 359 // Scoped helper for a re-settable bookmark. |
| 360 class BookmarkScope { |
| 361 public: |
| 362 explicit BookmarkScope(Scanner* scanner) : scanner_(scanner) { |
| 363 DCHECK_NOT_NULL(scanner_); |
| 364 } |
| 365 ~BookmarkScope() { scanner_->DropBookmark(); } |
| 366 |
| 367 bool Set() { return scanner_->SetBookmark(); } |
| 368 void Reset() { scanner_->ResetToBookmark(); } |
| 369 bool HasBeenSet() { return scanner_->BookmarkHasBeenSet(); } |
| 370 bool HasBeenReset() { return scanner_->BookmarkHasBeenReset(); } |
| 371 |
| 372 private: |
| 373 Scanner* scanner_; |
| 374 |
| 375 DISALLOW_COPY_AND_ASSIGN(BookmarkScope); |
| 376 }; |
| 377 |
345 // Representation of an interval of source positions. | 378 // Representation of an interval of source positions. |
346 struct Location { | 379 struct Location { |
347 Location(int b, int e) : beg_pos(b), end_pos(e) { } | 380 Location(int b, int e) : beg_pos(b), end_pos(e) { } |
348 Location() : beg_pos(0), end_pos(0) { } | 381 Location() : beg_pos(0), end_pos(0) { } |
349 | 382 |
350 bool IsValid() const { | 383 bool IsValid() const { |
351 return beg_pos >= 0 && end_pos >= beg_pos; | 384 return beg_pos >= 0 && end_pos >= beg_pos; |
352 } | 385 } |
353 | 386 |
354 static Location invalid() { return Location(-1, -1); } | 387 static Location invalid() { return Location(-1, -1); } |
(...skipping 148 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
503 // Call this after setting source_ to the input. | 536 // Call this after setting source_ to the input. |
504 void Init() { | 537 void Init() { |
505 // Set c0_ (one character ahead) | 538 // Set c0_ (one character ahead) |
506 STATIC_ASSERT(kCharacterLookaheadBufferSize == 1); | 539 STATIC_ASSERT(kCharacterLookaheadBufferSize == 1); |
507 Advance(); | 540 Advance(); |
508 // Initialize current_ to not refer to a literal. | 541 // Initialize current_ to not refer to a literal. |
509 current_.literal_chars = NULL; | 542 current_.literal_chars = NULL; |
510 current_.raw_literal_chars = NULL; | 543 current_.raw_literal_chars = NULL; |
511 } | 544 } |
512 | 545 |
| 546 // Support BookmarkScope functionality. |
| 547 bool SetBookmark(); |
| 548 void ResetToBookmark(); |
| 549 bool BookmarkHasBeenSet(); |
| 550 bool BookmarkHasBeenReset(); |
| 551 void DropBookmark(); |
| 552 static void CopyTokenDesc(TokenDesc* to, TokenDesc* from); |
| 553 |
513 // Literal buffer support | 554 // Literal buffer support |
514 inline void StartLiteral() { | 555 inline void StartLiteral() { |
515 LiteralBuffer* free_buffer = (current_.literal_chars == &literal_buffer1_) ? | 556 LiteralBuffer* free_buffer = (current_.literal_chars == &literal_buffer1_) ? |
516 &literal_buffer2_ : &literal_buffer1_; | 557 &literal_buffer2_ : &literal_buffer1_; |
517 free_buffer->Reset(); | 558 free_buffer->Reset(); |
518 next_.literal_chars = free_buffer; | 559 next_.literal_chars = free_buffer; |
519 } | 560 } |
520 | 561 |
521 inline void StartRawLiteral() { | 562 inline void StartRawLiteral() { |
522 LiteralBuffer* free_buffer = | 563 LiteralBuffer* free_buffer = |
(...skipping 182 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
705 LiteralBuffer source_url_; | 746 LiteralBuffer source_url_; |
706 LiteralBuffer source_mapping_url_; | 747 LiteralBuffer source_mapping_url_; |
707 | 748 |
708 // Buffer to store raw string values | 749 // Buffer to store raw string values |
709 LiteralBuffer raw_literal_buffer1_; | 750 LiteralBuffer raw_literal_buffer1_; |
710 LiteralBuffer raw_literal_buffer2_; | 751 LiteralBuffer raw_literal_buffer2_; |
711 | 752 |
712 TokenDesc current_; // desc for current token (as returned by Next()) | 753 TokenDesc current_; // desc for current token (as returned by Next()) |
713 TokenDesc next_; // desc for next token (one token look-ahead) | 754 TokenDesc next_; // desc for next token (one token look-ahead) |
714 | 755 |
| 756 // Variables for Scanner::BookmarkScope and the *Bookmark implementation. |
| 757 // These variables contain the scanner state when a bookmark is set. |
| 758 // |
| 759 // We will use bookmark_c0_ as a 'control' variable, where: |
| 760 // - bookmark_c0_ >= 0: A bookmark has been set and this contains c0_. |
| 761 // - bookmark_c0_ == -1: No bookmark has been set. |
| 762 // - bookmark_c0_ == -2: The bookmark has been applied (ResetToBookmark). |
| 763 // |
| 764 // Which state is being bookmarked? The parser state is distributed over |
| 765 // several variables, roughly like this: |
| 766 // ... 1234 + 5678 ..... [character stream] |
| 767 // [current_] [next_] c0_ | [scanner state] |
| 768 // So when the scanner is logically at the beginning of an expression |
| 769 // like "1234 + 4567", then: |
| 770 // - current_ contains "1234" |
| 771 // - next_ contains "+" |
| 772 // - c0_ contains ' ' (the space between "+" and "5678", |
| 773 // - the source_ character stream points to the beginning of "5678". |
| 774 // To be able to restore this state, we will keep copies of current_, next_, |
| 775 // and c0_; we'll ask the stream to bookmark itself, and we'll copy the |
| 776 // contents of current_'s and next_'s literal buffers to bookmark_*_literal_. |
| 777 static const uc32 kNoBookmark = -1; |
| 778 static const uc32 kBookmarkWasApplied = -2; |
| 779 uc32 bookmark_c0_; |
| 780 TokenDesc bookmark_current_; |
| 781 TokenDesc bookmark_next_; |
| 782 LiteralBuffer bookmark_current_literal_; |
| 783 LiteralBuffer bookmark_current_raw_literal_; |
| 784 LiteralBuffer bookmark_next_literal_; |
| 785 LiteralBuffer bookmark_next_raw_literal_; |
| 786 |
715 // Input stream. Must be initialized to an Utf16CharacterStream. | 787 // Input stream. Must be initialized to an Utf16CharacterStream. |
716 Utf16CharacterStream* source_; | 788 Utf16CharacterStream* source_; |
717 | 789 |
718 | 790 |
719 // Start position of the octal literal last scanned. | 791 // Start position of the octal literal last scanned. |
720 Location octal_pos_; | 792 Location octal_pos_; |
721 | 793 |
722 // One Unicode character look-ahead; c0_ < 0 at the end of the input. | 794 // One Unicode character look-ahead; c0_ < 0 at the end of the input. |
723 uc32 c0_; | 795 uc32 c0_; |
724 | 796 |
725 // Whether there is a line terminator whitespace character after | 797 // Whether there is a line terminator whitespace character after |
726 // the current token, and before the next. Does not count newlines | 798 // the current token, and before the next. Does not count newlines |
727 // inside multiline comments. | 799 // inside multiline comments. |
728 bool has_line_terminator_before_next_; | 800 bool has_line_terminator_before_next_; |
729 // Whether there is a multi-line comment that contains a | 801 // Whether there is a multi-line comment that contains a |
730 // line-terminator after the current token, and before the next. | 802 // line-terminator after the current token, and before the next. |
731 bool has_multiline_comment_before_next_; | 803 bool has_multiline_comment_before_next_; |
732 // Whether we scan 'module', 'import', 'export' as keywords. | 804 // Whether we scan 'module', 'import', 'export' as keywords. |
733 bool harmony_modules_; | 805 bool harmony_modules_; |
734 // Whether we scan 'class', 'extends', 'static' and 'super' as keywords. | 806 // Whether we scan 'class', 'extends', 'static' and 'super' as keywords. |
735 bool harmony_classes_; | 807 bool harmony_classes_; |
736 // Whether we allow \u{xxxxx}. | 808 // Whether we allow \u{xxxxx}. |
737 bool harmony_unicode_; | 809 bool harmony_unicode_; |
738 }; | 810 }; |
739 | 811 |
740 } } // namespace v8::internal | 812 } } // namespace v8::internal |
741 | 813 |
742 #endif // V8_SCANNER_H_ | 814 #endif // V8_SCANNER_H_ |
OLD | NEW |