OLD | NEW |
1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. |
6 | 6 |
7 #ifndef V8_PARSING_SCANNER_H_ | 7 #ifndef V8_PARSING_SCANNER_H_ |
8 #define V8_PARSING_SCANNER_H_ | 8 #define V8_PARSING_SCANNER_H_ |
9 | 9 |
10 #include "src/allocation.h" | 10 #include "src/allocation.h" |
(...skipping 28 matching lines...) Expand all Loading... |
39 } | 39 } |
40 | 40 |
41 | 41 |
42 // --------------------------------------------------------------------- | 42 // --------------------------------------------------------------------- |
43 // Buffered stream of UTF-16 code units, using an internal UTF-16 buffer. | 43 // Buffered stream of UTF-16 code units, using an internal UTF-16 buffer. |
44 // A code unit is a 16 bit value representing either a 16 bit code point | 44 // A code unit is a 16 bit value representing either a 16 bit code point |
45 // or one part of a surrogate pair that make a single 21 bit code point. | 45 // or one part of a surrogate pair that make a single 21 bit code point. |
46 | 46 |
47 class Utf16CharacterStream { | 47 class Utf16CharacterStream { |
48 public: | 48 public: |
49 Utf16CharacterStream() : pos_(0) { } | 49 Utf16CharacterStream() : pos_(0) {} |
50 virtual ~Utf16CharacterStream() { } | 50 virtual ~Utf16CharacterStream() {} |
51 | 51 |
52 // Returns and advances past the next UTF-16 code unit in the input | 52 // Returns and advances past the next UTF-16 code unit in the input |
53 // stream. If there are no more code units, it returns a negative | 53 // stream. If there are no more code units, it returns a negative |
54 // value. | 54 // value. |
55 inline uc32 Advance() { | 55 inline uc32 Advance() { |
56 if (buffer_cursor_ < buffer_end_ || ReadBlock()) { | 56 if (buffer_cursor_ < buffer_end_ || ReadBlock()) { |
57 pos_++; | 57 pos_++; |
58 return static_cast<uc32>(*(buffer_cursor_++)); | 58 return static_cast<uc32>(*(buffer_cursor_++)); |
59 } | 59 } |
60 // Note: currently the following increment is necessary to avoid a | 60 // Note: currently the following increment is necessary to avoid a |
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
108 size_t pos_; | 108 size_t pos_; |
109 }; | 109 }; |
110 | 110 |
111 | 111 |
112 // --------------------------------------------------------------------- | 112 // --------------------------------------------------------------------- |
113 // DuplicateFinder discovers duplicate symbols. | 113 // DuplicateFinder discovers duplicate symbols. |
114 | 114 |
115 class DuplicateFinder { | 115 class DuplicateFinder { |
116 public: | 116 public: |
117 explicit DuplicateFinder(UnicodeCache* constants) | 117 explicit DuplicateFinder(UnicodeCache* constants) |
118 : unicode_constants_(constants), | 118 : unicode_constants_(constants), backing_store_(16), map_(&Match) {} |
119 backing_store_(16), | |
120 map_(&Match) { } | |
121 | 119 |
122 int AddOneByteSymbol(Vector<const uint8_t> key, int value); | 120 int AddOneByteSymbol(Vector<const uint8_t> key, int value); |
123 int AddTwoByteSymbol(Vector<const uint16_t> key, int value); | 121 int AddTwoByteSymbol(Vector<const uint16_t> key, int value); |
124 // Add a a number literal by converting it (if necessary) | 122 // Add a a number literal by converting it (if necessary) |
125 // to the string that ToString(ToNumber(literal)) would generate. | 123 // to the string that ToString(ToNumber(literal)) would generate. |
126 // and then adding that string with AddOneByteSymbol. | 124 // and then adding that string with AddOneByteSymbol. |
127 // This string is the actual value used as key in an object literal, | 125 // This string is the actual value used as key in an object literal, |
128 // and the one that must be different from the other keys. | 126 // and the one that must be different from the other keys. |
129 int AddNumber(Vector<const uint8_t> key, int value); | 127 int AddNumber(Vector<const uint8_t> key, int value); |
130 | 128 |
(...skipping 26 matching lines...) Expand all Loading... |
157 // Buffer used for string->number->canonical string conversions. | 155 // Buffer used for string->number->canonical string conversions. |
158 char number_buffer_[kBufferSize]; | 156 char number_buffer_[kBufferSize]; |
159 }; | 157 }; |
160 | 158 |
161 | 159 |
162 // ---------------------------------------------------------------------------- | 160 // ---------------------------------------------------------------------------- |
163 // LiteralBuffer - Collector of chars of literals. | 161 // LiteralBuffer - Collector of chars of literals. |
164 | 162 |
165 class LiteralBuffer { | 163 class LiteralBuffer { |
166 public: | 164 public: |
167 LiteralBuffer() : is_one_byte_(true), position_(0), backing_store_() { } | 165 LiteralBuffer() : is_one_byte_(true), position_(0), backing_store_() {} |
168 | 166 |
169 ~LiteralBuffer() { backing_store_.Dispose(); } | 167 ~LiteralBuffer() { backing_store_.Dispose(); } |
170 | 168 |
171 INLINE(void AddChar(uint32_t code_unit)) { | 169 INLINE(void AddChar(uint32_t code_unit)) { |
172 if (position_ >= backing_store_.length()) ExpandBuffer(); | 170 if (position_ >= backing_store_.length()) ExpandBuffer(); |
173 if (is_one_byte_) { | 171 if (is_one_byte_) { |
174 if (code_unit <= unibrow::Latin1::kMaxChar) { | 172 if (code_unit <= unibrow::Latin1::kMaxChar) { |
175 backing_store_[position_] = static_cast<byte>(code_unit); | 173 backing_store_[position_] = static_cast<byte>(code_unit); |
176 position_ += kOneByteSize; | 174 position_ += kOneByteSize; |
177 return; | 175 return; |
(...skipping 11 matching lines...) Expand all Loading... |
189 *reinterpret_cast<uint16_t*>(&backing_store_[position_]) = | 187 *reinterpret_cast<uint16_t*>(&backing_store_[position_]) = |
190 unibrow::Utf16::TrailSurrogate(code_unit); | 188 unibrow::Utf16::TrailSurrogate(code_unit); |
191 position_ += kUC16Size; | 189 position_ += kUC16Size; |
192 } | 190 } |
193 } | 191 } |
194 | 192 |
195 bool is_one_byte() const { return is_one_byte_; } | 193 bool is_one_byte() const { return is_one_byte_; } |
196 | 194 |
197 bool is_contextual_keyword(Vector<const char> keyword) const { | 195 bool is_contextual_keyword(Vector<const char> keyword) const { |
198 return is_one_byte() && keyword.length() == position_ && | 196 return is_one_byte() && keyword.length() == position_ && |
199 (memcmp(keyword.start(), backing_store_.start(), position_) == 0); | 197 (memcmp(keyword.start(), backing_store_.start(), position_) == 0); |
200 } | 198 } |
201 | 199 |
202 Vector<const uint16_t> two_byte_literal() const { | 200 Vector<const uint16_t> two_byte_literal() const { |
203 DCHECK(!is_one_byte_); | 201 DCHECK(!is_one_byte_); |
204 DCHECK((position_ & 0x1) == 0); | 202 DCHECK((position_ & 0x1) == 0); |
205 return Vector<const uint16_t>( | 203 return Vector<const uint16_t>( |
206 reinterpret_cast<const uint16_t*>(backing_store_.start()), | 204 reinterpret_cast<const uint16_t*>(backing_store_.start()), |
207 position_ >> 1); | 205 position_ >> 1); |
208 } | 206 } |
209 | 207 |
210 Vector<const uint8_t> one_byte_literal() const { | 208 Vector<const uint8_t> one_byte_literal() const { |
211 DCHECK(is_one_byte_); | 209 DCHECK(is_one_byte_); |
212 return Vector<const uint8_t>( | 210 return Vector<const uint8_t>( |
213 reinterpret_cast<const uint8_t*>(backing_store_.start()), | 211 reinterpret_cast<const uint8_t*>(backing_store_.start()), position_); |
214 position_); | |
215 } | 212 } |
216 | 213 |
217 int length() const { | 214 int length() const { return is_one_byte_ ? position_ : (position_ >> 1); } |
218 return is_one_byte_ ? position_ : (position_ >> 1); | |
219 } | |
220 | 215 |
221 void ReduceLength(int delta) { | 216 void ReduceLength(int delta) { |
222 position_ -= delta * (is_one_byte_ ? kOneByteSize : kUC16Size); | 217 position_ -= delta * (is_one_byte_ ? kOneByteSize : kUC16Size); |
223 } | 218 } |
224 | 219 |
225 void Reset() { | 220 void Reset() { |
226 position_ = 0; | 221 position_ = 0; |
227 is_one_byte_ = true; | 222 is_one_byte_ = true; |
228 } | 223 } |
229 | 224 |
(...skipping 65 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
295 | 290 |
296 class Scanner { | 291 class Scanner { |
297 public: | 292 public: |
298 // Scoped helper for literal recording. Automatically drops the literal | 293 // Scoped helper for literal recording. Automatically drops the literal |
299 // if aborting the scanning before it's complete. | 294 // if aborting the scanning before it's complete. |
300 class LiteralScope { | 295 class LiteralScope { |
301 public: | 296 public: |
302 explicit LiteralScope(Scanner* self) : scanner_(self), complete_(false) { | 297 explicit LiteralScope(Scanner* self) : scanner_(self), complete_(false) { |
303 scanner_->StartLiteral(); | 298 scanner_->StartLiteral(); |
304 } | 299 } |
305 ~LiteralScope() { | 300 ~LiteralScope() { |
306 if (!complete_) scanner_->DropLiteral(); | 301 if (!complete_) scanner_->DropLiteral(); |
307 } | |
308 void Complete() { | |
309 complete_ = true; | |
310 } | 302 } |
| 303 void Complete() { complete_ = true; } |
311 | 304 |
312 private: | 305 private: |
313 Scanner* scanner_; | 306 Scanner* scanner_; |
314 bool complete_; | 307 bool complete_; |
315 }; | 308 }; |
316 | 309 |
317 // Scoped helper for a re-settable bookmark. | 310 // Scoped helper for a re-settable bookmark. |
318 class BookmarkScope { | 311 class BookmarkScope { |
319 public: | 312 public: |
320 explicit BookmarkScope(Scanner* scanner) : scanner_(scanner) { | 313 explicit BookmarkScope(Scanner* scanner) : scanner_(scanner) { |
321 DCHECK_NOT_NULL(scanner_); | 314 DCHECK_NOT_NULL(scanner_); |
322 } | 315 } |
323 ~BookmarkScope() { scanner_->DropBookmark(); } | 316 ~BookmarkScope() { scanner_->DropBookmark(); } |
324 | 317 |
325 bool Set() { return scanner_->SetBookmark(); } | 318 bool Set() { return scanner_->SetBookmark(); } |
326 void Reset() { scanner_->ResetToBookmark(); } | 319 void Reset() { scanner_->ResetToBookmark(); } |
327 bool HasBeenSet() { return scanner_->BookmarkHasBeenSet(); } | 320 bool HasBeenSet() { return scanner_->BookmarkHasBeenSet(); } |
328 bool HasBeenReset() { return scanner_->BookmarkHasBeenReset(); } | 321 bool HasBeenReset() { return scanner_->BookmarkHasBeenReset(); } |
329 | 322 |
330 private: | 323 private: |
331 Scanner* scanner_; | 324 Scanner* scanner_; |
332 | 325 |
333 DISALLOW_COPY_AND_ASSIGN(BookmarkScope); | 326 DISALLOW_COPY_AND_ASSIGN(BookmarkScope); |
334 }; | 327 }; |
335 | 328 |
336 // Representation of an interval of source positions. | 329 // Representation of an interval of source positions. |
337 struct Location { | 330 struct Location { |
338 Location(int b, int e) : beg_pos(b), end_pos(e) { } | 331 Location(int b, int e) : beg_pos(b), end_pos(e) {} |
339 Location() : beg_pos(0), end_pos(0) { } | 332 Location() : beg_pos(0), end_pos(0) {} |
340 | 333 |
341 bool IsValid() const { | 334 bool IsValid() const { return beg_pos >= 0 && end_pos >= beg_pos; } |
342 return beg_pos >= 0 && end_pos >= beg_pos; | |
343 } | |
344 | 335 |
345 static Location invalid() { return Location(-1, -1); } | 336 static Location invalid() { return Location(-1, -1); } |
346 | 337 |
347 int beg_pos; | 338 int beg_pos; |
348 int end_pos; | 339 int end_pos; |
349 }; | 340 }; |
350 | 341 |
351 // -1 is outside of the range of any real source code. | 342 // -1 is outside of the range of any real source code. |
352 static const int kNoOctalLocation = -1; | 343 static const int kNoOctalLocation = -1; |
353 | 344 |
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
387 return next_.literal_chars->is_contextual_keyword(keyword); | 378 return next_.literal_chars->is_contextual_keyword(keyword); |
388 } | 379 } |
389 | 380 |
390 const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory); | 381 const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory); |
391 const AstRawString* NextSymbol(AstValueFactory* ast_value_factory); | 382 const AstRawString* NextSymbol(AstValueFactory* ast_value_factory); |
392 const AstRawString* CurrentRawSymbol(AstValueFactory* ast_value_factory); | 383 const AstRawString* CurrentRawSymbol(AstValueFactory* ast_value_factory); |
393 | 384 |
394 double DoubleValue(); | 385 double DoubleValue(); |
395 bool ContainsDot(); | 386 bool ContainsDot(); |
396 bool LiteralMatches(const char* data, int length, bool allow_escapes = true) { | 387 bool LiteralMatches(const char* data, int length, bool allow_escapes = true) { |
397 if (is_literal_one_byte() && | 388 if (is_literal_one_byte() && literal_length() == length && |
398 literal_length() == length && | |
399 (allow_escapes || !literal_contains_escapes())) { | 389 (allow_escapes || !literal_contains_escapes())) { |
400 const char* token = | 390 const char* token = |
401 reinterpret_cast<const char*>(literal_one_byte_string().start()); | 391 reinterpret_cast<const char*>(literal_one_byte_string().start()); |
402 return !strncmp(token, data, length); | 392 return !strncmp(token, data, length); |
403 } | 393 } |
404 return false; | 394 return false; |
405 } | 395 } |
406 inline bool UnescapedLiteralMatches(const char* data, int length) { | 396 inline bool UnescapedLiteralMatches(const char* data, int length) { |
407 return LiteralMatches(data, length, false); | 397 return LiteralMatches(data, length, false); |
408 } | 398 } |
409 | 399 |
410 void IsGetOrSet(bool* is_get, bool* is_set) { | 400 void IsGetOrSet(bool* is_get, bool* is_set) { |
411 if (is_literal_one_byte() && | 401 if (is_literal_one_byte() && literal_length() == 3 && |
412 literal_length() == 3 && | |
413 !literal_contains_escapes()) { | 402 !literal_contains_escapes()) { |
414 const char* token = | 403 const char* token = |
415 reinterpret_cast<const char*>(literal_one_byte_string().start()); | 404 reinterpret_cast<const char*>(literal_one_byte_string().start()); |
416 *is_get = strncmp(token, "get", 3) == 0; | 405 *is_get = strncmp(token, "get", 3) == 0; |
417 *is_set = !*is_get && strncmp(token, "set", 3) == 0; | 406 *is_set = !*is_get && strncmp(token, "set", 3) == 0; |
418 } | 407 } |
419 } | 408 } |
420 | 409 |
421 int FindSymbol(DuplicateFinder* finder, int value); | 410 int FindSymbol(DuplicateFinder* finder, int value); |
422 | 411 |
(...skipping 339 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
762 bool has_line_terminator_before_next_; | 751 bool has_line_terminator_before_next_; |
763 // Whether there is a multi-line comment that contains a | 752 // Whether there is a multi-line comment that contains a |
764 // line-terminator after the current token, and before the next. | 753 // line-terminator after the current token, and before the next. |
765 bool has_multiline_comment_before_next_; | 754 bool has_multiline_comment_before_next_; |
766 }; | 755 }; |
767 | 756 |
768 } // namespace internal | 757 } // namespace internal |
769 } // namespace v8 | 758 } // namespace v8 |
770 | 759 |
771 #endif // V8_PARSING_SCANNER_H_ | 760 #endif // V8_PARSING_SCANNER_H_ |
OLD | NEW |