OLD | NEW |
1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. |
6 | 6 |
7 #ifndef V8_SCANNER_H_ | 7 #ifndef V8_SCANNER_H_ |
8 #define V8_SCANNER_H_ | 8 #define V8_SCANNER_H_ |
9 | 9 |
10 #include "src/allocation.h" | 10 #include "src/allocation.h" |
(...skipping 49 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
60 // parser problem! The scanner treats the final kEndOfInput as | 60 // parser problem! The scanner treats the final kEndOfInput as |
61 // a code unit with a position, and does math relative to that | 61 // a code unit with a position, and does math relative to that |
62 // position. | 62 // position. |
63 pos_++; | 63 pos_++; |
64 | 64 |
65 return kEndOfInput; | 65 return kEndOfInput; |
66 } | 66 } |
67 | 67 |
68 // Return the current position in the code unit stream. | 68 // Return the current position in the code unit stream. |
69 // Starts at zero. | 69 // Starts at zero. |
70 inline unsigned pos() const { return pos_; } | 70 inline size_t pos() const { return pos_; } |
71 | 71 |
72 // Skips forward past the next code_unit_count UTF-16 code units | 72 // Skips forward past the next code_unit_count UTF-16 code units |
73 // in the input, or until the end of input if that comes sooner. | 73 // in the input, or until the end of input if that comes sooner. |
74 // Returns the number of code units actually skipped. If less | 74 // Returns the number of code units actually skipped. If less |
75 // than code_unit_count, | 75 // than code_unit_count, |
76 inline unsigned SeekForward(unsigned code_unit_count) { | 76 inline size_t SeekForward(size_t code_unit_count) { |
77 unsigned buffered_chars = | 77 size_t buffered_chars = buffer_end_ - buffer_cursor_; |
78 static_cast<unsigned>(buffer_end_ - buffer_cursor_); | |
79 if (code_unit_count <= buffered_chars) { | 78 if (code_unit_count <= buffered_chars) { |
80 buffer_cursor_ += code_unit_count; | 79 buffer_cursor_ += code_unit_count; |
81 pos_ += code_unit_count; | 80 pos_ += code_unit_count; |
82 return code_unit_count; | 81 return code_unit_count; |
83 } | 82 } |
84 return SlowSeekForward(code_unit_count); | 83 return SlowSeekForward(code_unit_count); |
85 } | 84 } |
86 | 85 |
87 // Pushes back the most recently read UTF-16 code unit (or negative | 86 // Pushes back the most recently read UTF-16 code unit (or negative |
88 // value if at end of input), i.e., the value returned by the most recent | 87 // value if at end of input), i.e., the value returned by the most recent |
89 // call to Advance. | 88 // call to Advance. |
90 // Must not be used right after calling SeekForward. | 89 // Must not be used right after calling SeekForward. |
91 virtual void PushBack(int32_t code_unit) = 0; | 90 virtual void PushBack(int32_t code_unit) = 0; |
92 | 91 |
93 protected: | 92 protected: |
94 static const uc32 kEndOfInput = -1; | 93 static const uc32 kEndOfInput = -1; |
95 | 94 |
96 // Ensures that the buffer_cursor_ points to the code_unit at | 95 // Ensures that the buffer_cursor_ points to the code_unit at |
97 // position pos_ of the input, if possible. If the position | 96 // position pos_ of the input, if possible. If the position |
98 // is at or after the end of the input, return false. If there | 97 // is at or after the end of the input, return false. If there |
99 // are more code_units available, return true. | 98 // are more code_units available, return true. |
100 virtual bool ReadBlock() = 0; | 99 virtual bool ReadBlock() = 0; |
101 virtual unsigned SlowSeekForward(unsigned code_unit_count) = 0; | 100 virtual size_t SlowSeekForward(size_t code_unit_count) = 0; |
102 | 101 |
103 const uint16_t* buffer_cursor_; | 102 const uint16_t* buffer_cursor_; |
104 const uint16_t* buffer_end_; | 103 const uint16_t* buffer_end_; |
105 unsigned pos_; | 104 size_t pos_; |
106 }; | 105 }; |
107 | 106 |
108 | 107 |
109 // --------------------------------------------------------------------- | 108 // --------------------------------------------------------------------- |
110 // Caching predicates used by scanners. | 109 // Caching predicates used by scanners. |
111 | 110 |
112 class UnicodeCache { | 111 class UnicodeCache { |
113 public: | 112 public: |
114 UnicodeCache() {} | 113 UnicodeCache() {} |
115 typedef unibrow::Utf8Decoder<512> Utf8Decoder; | 114 typedef unibrow::Utf8Decoder<512> Utf8Decoder; |
(...skipping 574 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
690 // If the escape sequence cannot be decoded the result is kBadChar. | 689 // If the escape sequence cannot be decoded the result is kBadChar. |
691 uc32 ScanIdentifierUnicodeEscape(); | 690 uc32 ScanIdentifierUnicodeEscape(); |
692 // Helper for the above functions. | 691 // Helper for the above functions. |
693 template <bool capture_raw> | 692 template <bool capture_raw> |
694 uc32 ScanUnicodeEscape(); | 693 uc32 ScanUnicodeEscape(); |
695 | 694 |
696 Token::Value ScanTemplateSpan(); | 695 Token::Value ScanTemplateSpan(); |
697 | 696 |
698 // Return the current source position. | 697 // Return the current source position. |
699 int source_pos() { | 698 int source_pos() { |
700 return source_->pos() - kCharacterLookaheadBufferSize; | 699 return static_cast<int>(source_->pos()) - kCharacterLookaheadBufferSize; |
701 } | 700 } |
702 | 701 |
703 UnicodeCache* unicode_cache_; | 702 UnicodeCache* unicode_cache_; |
704 | 703 |
705 // Buffers collecting literal strings, numbers, etc. | 704 // Buffers collecting literal strings, numbers, etc. |
706 LiteralBuffer literal_buffer1_; | 705 LiteralBuffer literal_buffer1_; |
707 LiteralBuffer literal_buffer2_; | 706 LiteralBuffer literal_buffer2_; |
708 | 707 |
709 // Values parsed from magic comments. | 708 // Values parsed from magic comments. |
710 LiteralBuffer source_url_; | 709 LiteralBuffer source_url_; |
(...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
743 bool harmony_classes_; | 742 bool harmony_classes_; |
744 // Whether we scan TEMPLATE_SPAN and TEMPLATE_TAIL | 743 // Whether we scan TEMPLATE_SPAN and TEMPLATE_TAIL |
745 bool harmony_templates_; | 744 bool harmony_templates_; |
746 // Whether we allow \u{xxxxx}. | 745 // Whether we allow \u{xxxxx}. |
747 bool harmony_unicode_; | 746 bool harmony_unicode_; |
748 }; | 747 }; |
749 | 748 |
750 } } // namespace v8::internal | 749 } } // namespace v8::internal |
751 | 750 |
752 #endif // V8_SCANNER_H_ | 751 #endif // V8_SCANNER_H_ |
OLD | NEW |