| OLD | NEW |
| 1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. |
| 6 | 6 |
| 7 #ifndef V8_PARSING_SCANNER_H_ | 7 #ifndef V8_PARSING_SCANNER_H_ |
| 8 #define V8_PARSING_SCANNER_H_ | 8 #define V8_PARSING_SCANNER_H_ |
| 9 | 9 |
| 10 #include "src/allocation.h" | 10 #include "src/allocation.h" |
| (...skipping 28 matching lines...) Expand all Loading... |
| 39 } | 39 } |
| 40 | 40 |
| 41 | 41 |
| 42 // --------------------------------------------------------------------- | 42 // --------------------------------------------------------------------- |
| 43 // Buffered stream of UTF-16 code units, using an internal UTF-16 buffer. | 43 // Buffered stream of UTF-16 code units, using an internal UTF-16 buffer. |
| 44 // A code unit is a 16 bit value representing either a 16 bit code point | 44 // A code unit is a 16 bit value representing either a 16 bit code point |
| 45 // or one part of a surrogate pair that make a single 21 bit code point. | 45 // or one part of a surrogate pair that make a single 21 bit code point. |
| 46 | 46 |
| 47 class Utf16CharacterStream { | 47 class Utf16CharacterStream { |
| 48 public: | 48 public: |
| 49 Utf16CharacterStream() : pos_(0) { } | 49 Utf16CharacterStream() : pos_(0) {} |
| 50 virtual ~Utf16CharacterStream() { } | 50 virtual ~Utf16CharacterStream() {} |
| 51 | 51 |
| 52 // Returns and advances past the next UTF-16 code unit in the input | 52 // Returns and advances past the next UTF-16 code unit in the input |
| 53 // stream. If there are no more code units, it returns a negative | 53 // stream. If there are no more code units, it returns a negative |
| 54 // value. | 54 // value. |
| 55 inline uc32 Advance() { | 55 inline uc32 Advance() { |
| 56 if (buffer_cursor_ < buffer_end_ || ReadBlock()) { | 56 if (buffer_cursor_ < buffer_end_ || ReadBlock()) { |
| 57 pos_++; | 57 pos_++; |
| 58 return static_cast<uc32>(*(buffer_cursor_++)); | 58 return static_cast<uc32>(*(buffer_cursor_++)); |
| 59 } | 59 } |
| 60 // Note: currently the following increment is necessary to avoid a | 60 // Note: currently the following increment is necessary to avoid a |
| (...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 108 size_t pos_; | 108 size_t pos_; |
| 109 }; | 109 }; |
| 110 | 110 |
| 111 | 111 |
| 112 // --------------------------------------------------------------------- | 112 // --------------------------------------------------------------------- |
| 113 // DuplicateFinder discovers duplicate symbols. | 113 // DuplicateFinder discovers duplicate symbols. |
| 114 | 114 |
| 115 class DuplicateFinder { | 115 class DuplicateFinder { |
| 116 public: | 116 public: |
| 117 explicit DuplicateFinder(UnicodeCache* constants) | 117 explicit DuplicateFinder(UnicodeCache* constants) |
| 118 : unicode_constants_(constants), | 118 : unicode_constants_(constants), backing_store_(16), map_(&Match) {} |
| 119 backing_store_(16), | |
| 120 map_(&Match) { } | |
| 121 | 119 |
| 122 int AddOneByteSymbol(Vector<const uint8_t> key, int value); | 120 int AddOneByteSymbol(Vector<const uint8_t> key, int value); |
| 123 int AddTwoByteSymbol(Vector<const uint16_t> key, int value); | 121 int AddTwoByteSymbol(Vector<const uint16_t> key, int value); |
| 124 // Add a a number literal by converting it (if necessary) | 122 // Add a a number literal by converting it (if necessary) |
| 125 // to the string that ToString(ToNumber(literal)) would generate. | 123 // to the string that ToString(ToNumber(literal)) would generate. |
| 126 // and then adding that string with AddOneByteSymbol. | 124 // and then adding that string with AddOneByteSymbol. |
| 127 // This string is the actual value used as key in an object literal, | 125 // This string is the actual value used as key in an object literal, |
| 128 // and the one that must be different from the other keys. | 126 // and the one that must be different from the other keys. |
| 129 int AddNumber(Vector<const uint8_t> key, int value); | 127 int AddNumber(Vector<const uint8_t> key, int value); |
| 130 | 128 |
| (...skipping 26 matching lines...) Expand all Loading... |
| 157 // Buffer used for string->number->canonical string conversions. | 155 // Buffer used for string->number->canonical string conversions. |
| 158 char number_buffer_[kBufferSize]; | 156 char number_buffer_[kBufferSize]; |
| 159 }; | 157 }; |
| 160 | 158 |
| 161 | 159 |
| 162 // ---------------------------------------------------------------------------- | 160 // ---------------------------------------------------------------------------- |
| 163 // LiteralBuffer - Collector of chars of literals. | 161 // LiteralBuffer - Collector of chars of literals. |
| 164 | 162 |
| 165 class LiteralBuffer { | 163 class LiteralBuffer { |
| 166 public: | 164 public: |
| 167 LiteralBuffer() : is_one_byte_(true), position_(0), backing_store_() { } | 165 LiteralBuffer() : is_one_byte_(true), position_(0), backing_store_() {} |
| 168 | 166 |
| 169 ~LiteralBuffer() { backing_store_.Dispose(); } | 167 ~LiteralBuffer() { backing_store_.Dispose(); } |
| 170 | 168 |
| 171 INLINE(void AddChar(uint32_t code_unit)) { | 169 INLINE(void AddChar(uint32_t code_unit)) { |
| 172 if (position_ >= backing_store_.length()) ExpandBuffer(); | 170 if (position_ >= backing_store_.length()) ExpandBuffer(); |
| 173 if (is_one_byte_) { | 171 if (is_one_byte_) { |
| 174 if (code_unit <= unibrow::Latin1::kMaxChar) { | 172 if (code_unit <= unibrow::Latin1::kMaxChar) { |
| 175 backing_store_[position_] = static_cast<byte>(code_unit); | 173 backing_store_[position_] = static_cast<byte>(code_unit); |
| 176 position_ += kOneByteSize; | 174 position_ += kOneByteSize; |
| 177 return; | 175 return; |
| (...skipping 11 matching lines...) Expand all Loading... |
| 189 *reinterpret_cast<uint16_t*>(&backing_store_[position_]) = | 187 *reinterpret_cast<uint16_t*>(&backing_store_[position_]) = |
| 190 unibrow::Utf16::TrailSurrogate(code_unit); | 188 unibrow::Utf16::TrailSurrogate(code_unit); |
| 191 position_ += kUC16Size; | 189 position_ += kUC16Size; |
| 192 } | 190 } |
| 193 } | 191 } |
| 194 | 192 |
| 195 bool is_one_byte() const { return is_one_byte_; } | 193 bool is_one_byte() const { return is_one_byte_; } |
| 196 | 194 |
| 197 bool is_contextual_keyword(Vector<const char> keyword) const { | 195 bool is_contextual_keyword(Vector<const char> keyword) const { |
| 198 return is_one_byte() && keyword.length() == position_ && | 196 return is_one_byte() && keyword.length() == position_ && |
| 199 (memcmp(keyword.start(), backing_store_.start(), position_) == 0); | 197 (memcmp(keyword.start(), backing_store_.start(), position_) == 0); |
| 200 } | 198 } |
| 201 | 199 |
| 202 Vector<const uint16_t> two_byte_literal() const { | 200 Vector<const uint16_t> two_byte_literal() const { |
| 203 DCHECK(!is_one_byte_); | 201 DCHECK(!is_one_byte_); |
| 204 DCHECK((position_ & 0x1) == 0); | 202 DCHECK((position_ & 0x1) == 0); |
| 205 return Vector<const uint16_t>( | 203 return Vector<const uint16_t>( |
| 206 reinterpret_cast<const uint16_t*>(backing_store_.start()), | 204 reinterpret_cast<const uint16_t*>(backing_store_.start()), |
| 207 position_ >> 1); | 205 position_ >> 1); |
| 208 } | 206 } |
| 209 | 207 |
| 210 Vector<const uint8_t> one_byte_literal() const { | 208 Vector<const uint8_t> one_byte_literal() const { |
| 211 DCHECK(is_one_byte_); | 209 DCHECK(is_one_byte_); |
| 212 return Vector<const uint8_t>( | 210 return Vector<const uint8_t>( |
| 213 reinterpret_cast<const uint8_t*>(backing_store_.start()), | 211 reinterpret_cast<const uint8_t*>(backing_store_.start()), position_); |
| 214 position_); | |
| 215 } | 212 } |
| 216 | 213 |
| 217 int length() const { | 214 int length() const { return is_one_byte_ ? position_ : (position_ >> 1); } |
| 218 return is_one_byte_ ? position_ : (position_ >> 1); | |
| 219 } | |
| 220 | 215 |
| 221 void ReduceLength(int delta) { | 216 void ReduceLength(int delta) { |
| 222 position_ -= delta * (is_one_byte_ ? kOneByteSize : kUC16Size); | 217 position_ -= delta * (is_one_byte_ ? kOneByteSize : kUC16Size); |
| 223 } | 218 } |
| 224 | 219 |
| 225 void Reset() { | 220 void Reset() { |
| 226 position_ = 0; | 221 position_ = 0; |
| 227 is_one_byte_ = true; | 222 is_one_byte_ = true; |
| 228 } | 223 } |
| 229 | 224 |
| (...skipping 65 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 295 | 290 |
| 296 class Scanner { | 291 class Scanner { |
| 297 public: | 292 public: |
| 298 // Scoped helper for literal recording. Automatically drops the literal | 293 // Scoped helper for literal recording. Automatically drops the literal |
| 299 // if aborting the scanning before it's complete. | 294 // if aborting the scanning before it's complete. |
| 300 class LiteralScope { | 295 class LiteralScope { |
| 301 public: | 296 public: |
| 302 explicit LiteralScope(Scanner* self) : scanner_(self), complete_(false) { | 297 explicit LiteralScope(Scanner* self) : scanner_(self), complete_(false) { |
| 303 scanner_->StartLiteral(); | 298 scanner_->StartLiteral(); |
| 304 } | 299 } |
| 305 ~LiteralScope() { | 300 ~LiteralScope() { |
| 306 if (!complete_) scanner_->DropLiteral(); | 301 if (!complete_) scanner_->DropLiteral(); |
| 307 } | |
| 308 void Complete() { | |
| 309 complete_ = true; | |
| 310 } | 302 } |
| 303 void Complete() { complete_ = true; } |
| 311 | 304 |
| 312 private: | 305 private: |
| 313 Scanner* scanner_; | 306 Scanner* scanner_; |
| 314 bool complete_; | 307 bool complete_; |
| 315 }; | 308 }; |
| 316 | 309 |
| 317 // Scoped helper for a re-settable bookmark. | 310 // Scoped helper for a re-settable bookmark. |
| 318 class BookmarkScope { | 311 class BookmarkScope { |
| 319 public: | 312 public: |
| 320 explicit BookmarkScope(Scanner* scanner) : scanner_(scanner) { | 313 explicit BookmarkScope(Scanner* scanner) : scanner_(scanner) { |
| 321 DCHECK_NOT_NULL(scanner_); | 314 DCHECK_NOT_NULL(scanner_); |
| 322 } | 315 } |
| 323 ~BookmarkScope() { scanner_->DropBookmark(); } | 316 ~BookmarkScope() { scanner_->DropBookmark(); } |
| 324 | 317 |
| 325 bool Set() { return scanner_->SetBookmark(); } | 318 bool Set() { return scanner_->SetBookmark(); } |
| 326 void Reset() { scanner_->ResetToBookmark(); } | 319 void Reset() { scanner_->ResetToBookmark(); } |
| 327 bool HasBeenSet() { return scanner_->BookmarkHasBeenSet(); } | 320 bool HasBeenSet() { return scanner_->BookmarkHasBeenSet(); } |
| 328 bool HasBeenReset() { return scanner_->BookmarkHasBeenReset(); } | 321 bool HasBeenReset() { return scanner_->BookmarkHasBeenReset(); } |
| 329 | 322 |
| 330 private: | 323 private: |
| 331 Scanner* scanner_; | 324 Scanner* scanner_; |
| 332 | 325 |
| 333 DISALLOW_COPY_AND_ASSIGN(BookmarkScope); | 326 DISALLOW_COPY_AND_ASSIGN(BookmarkScope); |
| 334 }; | 327 }; |
| 335 | 328 |
| 336 // Representation of an interval of source positions. | 329 // Representation of an interval of source positions. |
| 337 struct Location { | 330 struct Location { |
| 338 Location(int b, int e) : beg_pos(b), end_pos(e) { } | 331 Location(int b, int e) : beg_pos(b), end_pos(e) {} |
| 339 Location() : beg_pos(0), end_pos(0) { } | 332 Location() : beg_pos(0), end_pos(0) {} |
| 340 | 333 |
| 341 bool IsValid() const { | 334 bool IsValid() const { return beg_pos >= 0 && end_pos >= beg_pos; } |
| 342 return beg_pos >= 0 && end_pos >= beg_pos; | |
| 343 } | |
| 344 | 335 |
| 345 static Location invalid() { return Location(-1, -1); } | 336 static Location invalid() { return Location(-1, -1); } |
| 346 | 337 |
| 347 int beg_pos; | 338 int beg_pos; |
| 348 int end_pos; | 339 int end_pos; |
| 349 }; | 340 }; |
| 350 | 341 |
| 351 // -1 is outside of the range of any real source code. | 342 // -1 is outside of the range of any real source code. |
| 352 static const int kNoOctalLocation = -1; | 343 static const int kNoOctalLocation = -1; |
| 353 | 344 |
| (...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 387 return next_.literal_chars->is_contextual_keyword(keyword); | 378 return next_.literal_chars->is_contextual_keyword(keyword); |
| 388 } | 379 } |
| 389 | 380 |
| 390 const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory); | 381 const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory); |
| 391 const AstRawString* NextSymbol(AstValueFactory* ast_value_factory); | 382 const AstRawString* NextSymbol(AstValueFactory* ast_value_factory); |
| 392 const AstRawString* CurrentRawSymbol(AstValueFactory* ast_value_factory); | 383 const AstRawString* CurrentRawSymbol(AstValueFactory* ast_value_factory); |
| 393 | 384 |
| 394 double DoubleValue(); | 385 double DoubleValue(); |
| 395 bool ContainsDot(); | 386 bool ContainsDot(); |
| 396 bool LiteralMatches(const char* data, int length, bool allow_escapes = true) { | 387 bool LiteralMatches(const char* data, int length, bool allow_escapes = true) { |
| 397 if (is_literal_one_byte() && | 388 if (is_literal_one_byte() && literal_length() == length && |
| 398 literal_length() == length && | |
| 399 (allow_escapes || !literal_contains_escapes())) { | 389 (allow_escapes || !literal_contains_escapes())) { |
| 400 const char* token = | 390 const char* token = |
| 401 reinterpret_cast<const char*>(literal_one_byte_string().start()); | 391 reinterpret_cast<const char*>(literal_one_byte_string().start()); |
| 402 return !strncmp(token, data, length); | 392 return !strncmp(token, data, length); |
| 403 } | 393 } |
| 404 return false; | 394 return false; |
| 405 } | 395 } |
| 406 inline bool UnescapedLiteralMatches(const char* data, int length) { | 396 inline bool UnescapedLiteralMatches(const char* data, int length) { |
| 407 return LiteralMatches(data, length, false); | 397 return LiteralMatches(data, length, false); |
| 408 } | 398 } |
| 409 | 399 |
| 410 void IsGetOrSet(bool* is_get, bool* is_set) { | 400 void IsGetOrSet(bool* is_get, bool* is_set) { |
| 411 if (is_literal_one_byte() && | 401 if (is_literal_one_byte() && literal_length() == 3 && |
| 412 literal_length() == 3 && | |
| 413 !literal_contains_escapes()) { | 402 !literal_contains_escapes()) { |
| 414 const char* token = | 403 const char* token = |
| 415 reinterpret_cast<const char*>(literal_one_byte_string().start()); | 404 reinterpret_cast<const char*>(literal_one_byte_string().start()); |
| 416 *is_get = strncmp(token, "get", 3) == 0; | 405 *is_get = strncmp(token, "get", 3) == 0; |
| 417 *is_set = !*is_get && strncmp(token, "set", 3) == 0; | 406 *is_set = !*is_get && strncmp(token, "set", 3) == 0; |
| 418 } | 407 } |
| 419 } | 408 } |
| 420 | 409 |
| 421 int FindSymbol(DuplicateFinder* finder, int value); | 410 int FindSymbol(DuplicateFinder* finder, int value); |
| 422 | 411 |
| (...skipping 339 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 762 bool has_line_terminator_before_next_; | 751 bool has_line_terminator_before_next_; |
| 763 // Whether there is a multi-line comment that contains a | 752 // Whether there is a multi-line comment that contains a |
| 764 // line-terminator after the current token, and before the next. | 753 // line-terminator after the current token, and before the next. |
| 765 bool has_multiline_comment_before_next_; | 754 bool has_multiline_comment_before_next_; |
| 766 }; | 755 }; |
| 767 | 756 |
| 768 } // namespace internal | 757 } // namespace internal |
| 769 } // namespace v8 | 758 } // namespace v8 |
| 770 | 759 |
| 771 #endif // V8_PARSING_SCANNER_H_ | 760 #endif // V8_PARSING_SCANNER_H_ |
| OLD | NEW |