OLD | NEW |
1 // Copyright 2010 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
2 // Redistribution and use in source and binary forms, with or without | 2 // Redistribution and use in source and binary forms, with or without |
3 // modification, are permitted provided that the following conditions are | 3 // modification, are permitted provided that the following conditions are |
4 // met: | 4 // met: |
5 // | 5 // |
6 // * Redistributions of source code must retain the above copyright | 6 // * Redistributions of source code must retain the above copyright |
7 // notice, this list of conditions and the following disclaimer. | 7 // notice, this list of conditions and the following disclaimer. |
8 // * Redistributions in binary form must reproduce the above | 8 // * Redistributions in binary form must reproduce the above |
9 // copyright notice, this list of conditions and the following | 9 // copyright notice, this list of conditions and the following |
10 // disclaimer in the documentation and/or other materials provided | 10 // disclaimer in the documentation and/or other materials provided |
11 // with the distribution. | 11 // with the distribution. |
(...skipping 100 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
112 // are more characters available, return true. | 112 // are more characters available, return true. |
113 virtual bool ReadBlock() = 0; | 113 virtual bool ReadBlock() = 0; |
114 virtual unsigned SlowSeekForward(unsigned character_count) = 0; | 114 virtual unsigned SlowSeekForward(unsigned character_count) = 0; |
115 | 115 |
116 const uc16* buffer_cursor_; | 116 const uc16* buffer_cursor_; |
117 const uc16* buffer_end_; | 117 const uc16* buffer_end_; |
118 unsigned pos_; | 118 unsigned pos_; |
119 }; | 119 }; |
120 | 120 |
121 | 121 |
122 class ScannerConstants { | 122 class UnicodeCache { |
123 // --------------------------------------------------------------------- | 123 // --------------------------------------------------------------------- |
124 // Constants used by scanners. | 124 // Caching predicates used by scanners. |
125 public: | 125 public: |
126 ScannerConstants() {} | 126 UnicodeCache() {} |
127 typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder; | 127 typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder; |
128 | 128 |
129 StaticResource<Utf8Decoder>* utf8_decoder() { | 129 StaticResource<Utf8Decoder>* utf8_decoder() { |
130 return &utf8_decoder_; | 130 return &utf8_decoder_; |
131 } | 131 } |
132 | 132 |
133 bool IsIdentifierStart(unibrow::uchar c) { return kIsIdentifierStart.get(c); } | 133 bool IsIdentifierStart(unibrow::uchar c) { return kIsIdentifierStart.get(c); } |
134 bool IsIdentifierPart(unibrow::uchar c) { return kIsIdentifierPart.get(c); } | 134 bool IsIdentifierPart(unibrow::uchar c) { return kIsIdentifierPart.get(c); } |
135 bool IsLineTerminator(unibrow::uchar c) { return kIsLineTerminator.get(c); } | 135 bool IsLineTerminator(unibrow::uchar c) { return kIsLineTerminator.get(c); } |
136 bool IsWhiteSpace(unibrow::uchar c) { return kIsWhiteSpace.get(c); } | 136 bool IsWhiteSpace(unibrow::uchar c) { return kIsWhiteSpace.get(c); } |
137 | 137 |
138 bool IsIdentifier(unibrow::CharacterStream* buffer); | |
139 | |
140 private: | 138 private: |
141 | 139 |
142 unibrow::Predicate<IdentifierStart, 128> kIsIdentifierStart; | 140 unibrow::Predicate<IdentifierStart, 128> kIsIdentifierStart; |
143 unibrow::Predicate<IdentifierPart, 128> kIsIdentifierPart; | 141 unibrow::Predicate<IdentifierPart, 128> kIsIdentifierPart; |
144 unibrow::Predicate<unibrow::LineTerminator, 128> kIsLineTerminator; | 142 unibrow::Predicate<unibrow::LineTerminator, 128> kIsLineTerminator; |
145 unibrow::Predicate<unibrow::WhiteSpace, 128> kIsWhiteSpace; | 143 unibrow::Predicate<unibrow::WhiteSpace, 128> kIsWhiteSpace; |
146 StaticResource<Utf8Decoder> utf8_decoder_; | 144 StaticResource<Utf8Decoder> utf8_decoder_; |
147 | 145 |
148 DISALLOW_COPY_AND_ASSIGN(ScannerConstants); | 146 DISALLOW_COPY_AND_ASSIGN(UnicodeCache); |
149 }; | 147 }; |
150 | 148 |
| 149 |
151 // ---------------------------------------------------------------------------- | 150 // ---------------------------------------------------------------------------- |
152 // LiteralBuffer - Collector of chars of literals. | 151 // LiteralBuffer - Collector of chars of literals. |
153 | 152 |
154 class LiteralBuffer { | 153 class LiteralBuffer { |
155 public: | 154 public: |
156 LiteralBuffer() : is_ascii_(true), position_(0), backing_store_() { } | 155 LiteralBuffer() : is_ascii_(true), position_(0), backing_store_() { } |
157 | 156 |
158 ~LiteralBuffer() { | 157 ~LiteralBuffer() { |
159 if (backing_store_.length() > 0) { | 158 if (backing_store_.length() > 0) { |
160 backing_store_.Dispose(); | 159 backing_store_.Dispose(); |
(...skipping 104 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
265 public: | 264 public: |
266 explicit LiteralScope(Scanner* self); | 265 explicit LiteralScope(Scanner* self); |
267 ~LiteralScope(); | 266 ~LiteralScope(); |
268 void Complete(); | 267 void Complete(); |
269 | 268 |
270 private: | 269 private: |
271 Scanner* scanner_; | 270 Scanner* scanner_; |
272 bool complete_; | 271 bool complete_; |
273 }; | 272 }; |
274 | 273 |
275 explicit Scanner(ScannerConstants* scanner_contants); | 274 explicit Scanner(UnicodeCache* scanner_contants); |
276 | 275 |
277 // Returns the current token again. | 276 // Returns the current token again. |
278 Token::Value current_token() { return current_.token; } | 277 Token::Value current_token() { return current_.token; } |
279 | 278 |
280 // One token look-ahead (past the token returned by Next()). | 279 // One token look-ahead (past the token returned by Next()). |
281 Token::Value peek() const { return next_.token; } | 280 Token::Value peek() const { return next_.token; } |
282 | 281 |
283 struct Location { | 282 struct Location { |
284 Location(int b, int e) : beg_pos(b), end_pos(e) { } | 283 Location(int b, int e) : beg_pos(b), end_pos(e) { } |
285 Location() : beg_pos(0), end_pos(0) { } | 284 Location() : beg_pos(0), end_pos(0) { } |
(...skipping 134 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
420 uc32 ScanHexEscape(uc32 c, int length); | 419 uc32 ScanHexEscape(uc32 c, int length); |
421 | 420 |
422 // Scans octal escape sequence. Also accepts "\0" decimal escape sequence. | 421 // Scans octal escape sequence. Also accepts "\0" decimal escape sequence. |
423 uc32 ScanOctalEscape(uc32 c, int length); | 422 uc32 ScanOctalEscape(uc32 c, int length); |
424 | 423 |
425 // Return the current source position. | 424 // Return the current source position. |
426 int source_pos() { | 425 int source_pos() { |
427 return source_->pos() - kCharacterLookaheadBufferSize; | 426 return source_->pos() - kCharacterLookaheadBufferSize; |
428 } | 427 } |
429 | 428 |
430 ScannerConstants* scanner_constants_; | 429 UnicodeCache* unicode_cache_; |
431 | 430 |
432 // Buffers collecting literal strings, numbers, etc. | 431 // Buffers collecting literal strings, numbers, etc. |
433 LiteralBuffer literal_buffer1_; | 432 LiteralBuffer literal_buffer1_; |
434 LiteralBuffer literal_buffer2_; | 433 LiteralBuffer literal_buffer2_; |
435 | 434 |
436 TokenDesc current_; // desc for current token (as returned by Next()) | 435 TokenDesc current_; // desc for current token (as returned by Next()) |
437 TokenDesc next_; // desc for next token (one token look-ahead) | 436 TokenDesc next_; // desc for next token (one token look-ahead) |
438 | 437 |
439 // Input stream. Must be initialized to an UC16CharacterStream. | 438 // Input stream. Must be initialized to an UC16CharacterStream. |
440 UC16CharacterStream* source_; | 439 UC16CharacterStream* source_; |
(...skipping 25 matching lines...) Expand all Loading... |
466 void Complete() { | 465 void Complete() { |
467 scanner_->TerminateLiteral(); | 466 scanner_->TerminateLiteral(); |
468 complete_ = true; | 467 complete_ = true; |
469 } | 468 } |
470 | 469 |
471 private: | 470 private: |
472 JavaScriptScanner* scanner_; | 471 JavaScriptScanner* scanner_; |
473 bool complete_; | 472 bool complete_; |
474 }; | 473 }; |
475 | 474 |
476 explicit JavaScriptScanner(ScannerConstants* scanner_contants); | 475 explicit JavaScriptScanner(UnicodeCache* scanner_contants); |
477 | 476 |
478 // Returns the next token. | 477 // Returns the next token. |
479 Token::Value Next(); | 478 Token::Value Next(); |
480 | 479 |
481 // Returns true if there was a line terminator before the peek'ed token. | 480 // Returns true if there was a line terminator before the peek'ed token. |
482 bool has_line_terminator_before_next() const { | 481 bool has_line_terminator_before_next() const { |
483 return has_line_terminator_before_next_; | 482 return has_line_terminator_before_next_; |
484 } | 483 } |
485 | 484 |
486 // Scans the input as a regular expression pattern, previous | 485 // Scans the input as a regular expression pattern, previous |
(...skipping 168 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
655 // keyword with the current prefix). | 654 // keyword with the current prefix). |
656 const char* keyword_; | 655 const char* keyword_; |
657 int counter_; | 656 int counter_; |
658 Token::Value keyword_token_; | 657 Token::Value keyword_token_; |
659 }; | 658 }; |
660 | 659 |
661 | 660 |
662 } } // namespace v8::internal | 661 } } // namespace v8::internal |
663 | 662 |
664 #endif // V8_SCANNER_BASE_H_ | 663 #endif // V8_SCANNER_BASE_H_ |
OLD | NEW |