| OLD | NEW |
| 1 // Copyright 2010 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
| 2 // Redistribution and use in source and binary forms, with or without | 2 // Redistribution and use in source and binary forms, with or without |
| 3 // modification, are permitted provided that the following conditions are | 3 // modification, are permitted provided that the following conditions are |
| 4 // met: | 4 // met: |
| 5 // | 5 // |
| 6 // * Redistributions of source code must retain the above copyright | 6 // * Redistributions of source code must retain the above copyright |
| 7 // notice, this list of conditions and the following disclaimer. | 7 // notice, this list of conditions and the following disclaimer. |
| 8 // * Redistributions in binary form must reproduce the above | 8 // * Redistributions in binary form must reproduce the above |
| 9 // copyright notice, this list of conditions and the following | 9 // copyright notice, this list of conditions and the following |
| 10 // disclaimer in the documentation and/or other materials provided | 10 // disclaimer in the documentation and/or other materials provided |
| 11 // with the distribution. | 11 // with the distribution. |
| (...skipping 100 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 112 // are more characters available, return true. | 112 // are more characters available, return true. |
| 113 virtual bool ReadBlock() = 0; | 113 virtual bool ReadBlock() = 0; |
| 114 virtual unsigned SlowSeekForward(unsigned character_count) = 0; | 114 virtual unsigned SlowSeekForward(unsigned character_count) = 0; |
| 115 | 115 |
| 116 const uc16* buffer_cursor_; | 116 const uc16* buffer_cursor_; |
| 117 const uc16* buffer_end_; | 117 const uc16* buffer_end_; |
| 118 unsigned pos_; | 118 unsigned pos_; |
| 119 }; | 119 }; |
| 120 | 120 |
| 121 | 121 |
| 122 class ScannerConstants { | 122 class UnicodeCache { |
| 123 // --------------------------------------------------------------------- | 123 // --------------------------------------------------------------------- |
| 124 // Constants used by scanners. | 124 // Caching predicates used by scanners. |
| 125 public: | 125 public: |
| 126 ScannerConstants() {} | 126 UnicodeCache() {} |
| 127 typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder; | 127 typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder; |
| 128 | 128 |
| 129 StaticResource<Utf8Decoder>* utf8_decoder() { | 129 StaticResource<Utf8Decoder>* utf8_decoder() { |
| 130 return &utf8_decoder_; | 130 return &utf8_decoder_; |
| 131 } | 131 } |
| 132 | 132 |
| 133 bool IsIdentifierStart(unibrow::uchar c) { return kIsIdentifierStart.get(c); } | 133 bool IsIdentifierStart(unibrow::uchar c) { return kIsIdentifierStart.get(c); } |
| 134 bool IsIdentifierPart(unibrow::uchar c) { return kIsIdentifierPart.get(c); } | 134 bool IsIdentifierPart(unibrow::uchar c) { return kIsIdentifierPart.get(c); } |
| 135 bool IsLineTerminator(unibrow::uchar c) { return kIsLineTerminator.get(c); } | 135 bool IsLineTerminator(unibrow::uchar c) { return kIsLineTerminator.get(c); } |
| 136 bool IsWhiteSpace(unibrow::uchar c) { return kIsWhiteSpace.get(c); } | 136 bool IsWhiteSpace(unibrow::uchar c) { return kIsWhiteSpace.get(c); } |
| 137 | 137 |
| 138 bool IsIdentifier(unibrow::CharacterStream* buffer); | |
| 139 | |
| 140 private: | 138 private: |
| 141 | 139 |
| 142 unibrow::Predicate<IdentifierStart, 128> kIsIdentifierStart; | 140 unibrow::Predicate<IdentifierStart, 128> kIsIdentifierStart; |
| 143 unibrow::Predicate<IdentifierPart, 128> kIsIdentifierPart; | 141 unibrow::Predicate<IdentifierPart, 128> kIsIdentifierPart; |
| 144 unibrow::Predicate<unibrow::LineTerminator, 128> kIsLineTerminator; | 142 unibrow::Predicate<unibrow::LineTerminator, 128> kIsLineTerminator; |
| 145 unibrow::Predicate<unibrow::WhiteSpace, 128> kIsWhiteSpace; | 143 unibrow::Predicate<unibrow::WhiteSpace, 128> kIsWhiteSpace; |
| 146 StaticResource<Utf8Decoder> utf8_decoder_; | 144 StaticResource<Utf8Decoder> utf8_decoder_; |
| 147 | 145 |
| 148 DISALLOW_COPY_AND_ASSIGN(ScannerConstants); | 146 DISALLOW_COPY_AND_ASSIGN(UnicodeCache); |
| 149 }; | 147 }; |
| 150 | 148 |
| 149 |
| 151 // ---------------------------------------------------------------------------- | 150 // ---------------------------------------------------------------------------- |
| 152 // LiteralBuffer - Collector of chars of literals. | 151 // LiteralBuffer - Collector of chars of literals. |
| 153 | 152 |
| 154 class LiteralBuffer { | 153 class LiteralBuffer { |
| 155 public: | 154 public: |
| 156 LiteralBuffer() : is_ascii_(true), position_(0), backing_store_() { } | 155 LiteralBuffer() : is_ascii_(true), position_(0), backing_store_() { } |
| 157 | 156 |
| 158 ~LiteralBuffer() { | 157 ~LiteralBuffer() { |
| 159 if (backing_store_.length() > 0) { | 158 if (backing_store_.length() > 0) { |
| 160 backing_store_.Dispose(); | 159 backing_store_.Dispose(); |
| (...skipping 104 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 265 public: | 264 public: |
| 266 explicit LiteralScope(Scanner* self); | 265 explicit LiteralScope(Scanner* self); |
| 267 ~LiteralScope(); | 266 ~LiteralScope(); |
| 268 void Complete(); | 267 void Complete(); |
| 269 | 268 |
| 270 private: | 269 private: |
| 271 Scanner* scanner_; | 270 Scanner* scanner_; |
| 272 bool complete_; | 271 bool complete_; |
| 273 }; | 272 }; |
| 274 | 273 |
| 275 explicit Scanner(ScannerConstants* scanner_contants); | 274 explicit Scanner(UnicodeCache* scanner_contants); |
| 276 | 275 |
| 277 // Returns the current token again. | 276 // Returns the current token again. |
| 278 Token::Value current_token() { return current_.token; } | 277 Token::Value current_token() { return current_.token; } |
| 279 | 278 |
| 280 // One token look-ahead (past the token returned by Next()). | 279 // One token look-ahead (past the token returned by Next()). |
| 281 Token::Value peek() const { return next_.token; } | 280 Token::Value peek() const { return next_.token; } |
| 282 | 281 |
| 283 struct Location { | 282 struct Location { |
| 284 Location(int b, int e) : beg_pos(b), end_pos(e) { } | 283 Location(int b, int e) : beg_pos(b), end_pos(e) { } |
| 285 Location() : beg_pos(0), end_pos(0) { } | 284 Location() : beg_pos(0), end_pos(0) { } |
| (...skipping 134 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 420 uc32 ScanHexEscape(uc32 c, int length); | 419 uc32 ScanHexEscape(uc32 c, int length); |
| 421 | 420 |
| 422 // Scans octal escape sequence. Also accepts "\0" decimal escape sequence. | 421 // Scans octal escape sequence. Also accepts "\0" decimal escape sequence. |
| 423 uc32 ScanOctalEscape(uc32 c, int length); | 422 uc32 ScanOctalEscape(uc32 c, int length); |
| 424 | 423 |
| 425 // Return the current source position. | 424 // Return the current source position. |
| 426 int source_pos() { | 425 int source_pos() { |
| 427 return source_->pos() - kCharacterLookaheadBufferSize; | 426 return source_->pos() - kCharacterLookaheadBufferSize; |
| 428 } | 427 } |
| 429 | 428 |
| 430 ScannerConstants* scanner_constants_; | 429 UnicodeCache* unicode_cache_; |
| 431 | 430 |
| 432 // Buffers collecting literal strings, numbers, etc. | 431 // Buffers collecting literal strings, numbers, etc. |
| 433 LiteralBuffer literal_buffer1_; | 432 LiteralBuffer literal_buffer1_; |
| 434 LiteralBuffer literal_buffer2_; | 433 LiteralBuffer literal_buffer2_; |
| 435 | 434 |
| 436 TokenDesc current_; // desc for current token (as returned by Next()) | 435 TokenDesc current_; // desc for current token (as returned by Next()) |
| 437 TokenDesc next_; // desc for next token (one token look-ahead) | 436 TokenDesc next_; // desc for next token (one token look-ahead) |
| 438 | 437 |
| 439 // Input stream. Must be initialized to an UC16CharacterStream. | 438 // Input stream. Must be initialized to an UC16CharacterStream. |
| 440 UC16CharacterStream* source_; | 439 UC16CharacterStream* source_; |
| (...skipping 25 matching lines...) Expand all Loading... |
| 466 void Complete() { | 465 void Complete() { |
| 467 scanner_->TerminateLiteral(); | 466 scanner_->TerminateLiteral(); |
| 468 complete_ = true; | 467 complete_ = true; |
| 469 } | 468 } |
| 470 | 469 |
| 471 private: | 470 private: |
| 472 JavaScriptScanner* scanner_; | 471 JavaScriptScanner* scanner_; |
| 473 bool complete_; | 472 bool complete_; |
| 474 }; | 473 }; |
| 475 | 474 |
| 476 explicit JavaScriptScanner(ScannerConstants* scanner_contants); | 475 explicit JavaScriptScanner(UnicodeCache* scanner_contants); |
| 477 | 476 |
| 478 // Returns the next token. | 477 // Returns the next token. |
| 479 Token::Value Next(); | 478 Token::Value Next(); |
| 480 | 479 |
| 481 // Returns true if there was a line terminator before the peek'ed token. | 480 // Returns true if there was a line terminator before the peek'ed token. |
| 482 bool has_line_terminator_before_next() const { | 481 bool has_line_terminator_before_next() const { |
| 483 return has_line_terminator_before_next_; | 482 return has_line_terminator_before_next_; |
| 484 } | 483 } |
| 485 | 484 |
| 486 // Scans the input as a regular expression pattern, previous | 485 // Scans the input as a regular expression pattern, previous |
| (...skipping 168 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 655 // keyword with the current prefix). | 654 // keyword with the current prefix). |
| 656 const char* keyword_; | 655 const char* keyword_; |
| 657 int counter_; | 656 int counter_; |
| 658 Token::Value keyword_token_; | 657 Token::Value keyword_token_; |
| 659 }; | 658 }; |
| 660 | 659 |
| 661 | 660 |
| 662 } } // namespace v8::internal | 661 } } // namespace v8::internal |
| 663 | 662 |
| 664 #endif // V8_SCANNER_BASE_H_ | 663 #endif // V8_SCANNER_BASE_H_ |
| OLD | NEW |