| OLD | NEW |
| 1 // Copyright 2010 the V8 project authors. All rights reserved. | 1 // Copyright 2010 the V8 project authors. All rights reserved. |
| 2 // Redistribution and use in source and binary forms, with or without | 2 // Redistribution and use in source and binary forms, with or without |
| 3 // modification, are permitted provided that the following conditions are | 3 // modification, are permitted provided that the following conditions are |
| 4 // met: | 4 // met: |
| 5 // | 5 // |
| 6 // * Redistributions of source code must retain the above copyright | 6 // * Redistributions of source code must retain the above copyright |
| 7 // notice, this list of conditions and the following disclaimer. | 7 // notice, this list of conditions and the following disclaimer. |
| 8 // * Redistributions in binary form must reproduce the above | 8 // * Redistributions in binary form must reproduce the above |
| 9 // copyright notice, this list of conditions and the following | 9 // copyright notice, this list of conditions and the following |
| 10 // disclaimer in the documentation and/or other materials provided | 10 // disclaimer in the documentation and/or other materials provided |
| (...skipping 101 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 112 // are more characters available, return true. | 112 // are more characters available, return true. |
| 113 virtual bool ReadBlock() = 0; | 113 virtual bool ReadBlock() = 0; |
| 114 virtual unsigned SlowSeekForward(unsigned character_count) = 0; | 114 virtual unsigned SlowSeekForward(unsigned character_count) = 0; |
| 115 | 115 |
| 116 const uc16* buffer_cursor_; | 116 const uc16* buffer_cursor_; |
| 117 const uc16* buffer_end_; | 117 const uc16* buffer_end_; |
| 118 unsigned pos_; | 118 unsigned pos_; |
| 119 }; | 119 }; |
| 120 | 120 |
| 121 | 121 |
| 122 class ScannerConstants { |
| 122 // --------------------------------------------------------------------- | 123 // --------------------------------------------------------------------- |
| 123 // Constants used by scanners. | 124 // Constants used by scanners. |
| 124 | |
| 125 class ScannerConstants : AllStatic { | |
| 126 public: | 125 public: |
| 127 typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder; | 126 typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder; |
| 128 | 127 |
| 129 static StaticResource<Utf8Decoder>* utf8_decoder() { | 128 StaticResource<Utf8Decoder>* utf8_decoder() { |
| 130 return &utf8_decoder_; | 129 return &utf8_decoder_; |
| 131 } | 130 } |
| 132 | 131 |
| 133 static unibrow::Predicate<IdentifierStart, 128> kIsIdentifierStart; | 132 bool IsIdentifierStart(unibrow::uchar c) { return kIsIdentifierStart.get(c); } |
| 134 static unibrow::Predicate<IdentifierPart, 128> kIsIdentifierPart; | 133 bool IsIdentifierPart(unibrow::uchar c) { return kIsIdentifierPart.get(c); } |
| 135 static unibrow::Predicate<unibrow::LineTerminator, 128> kIsLineTerminator; | 134 bool IsLineTerminator(unibrow::uchar c) { return kIsLineTerminator.get(c); } |
| 136 static unibrow::Predicate<unibrow::WhiteSpace, 128> kIsWhiteSpace; | 135 bool IsWhiteSpace(unibrow::uchar c) { return kIsWhiteSpace.get(c); } |
| 137 | 136 |
| 138 static bool IsIdentifier(unibrow::CharacterStream* buffer); | 137 bool IsIdentifier(unibrow::CharacterStream* buffer); |
| 139 | 138 |
| 140 private: | 139 private: |
| 141 static StaticResource<Utf8Decoder> utf8_decoder_; | 140 ScannerConstants() {} |
| 141 |
| 142 unibrow::Predicate<IdentifierStart, 128> kIsIdentifierStart; |
| 143 unibrow::Predicate<IdentifierPart, 128> kIsIdentifierPart; |
| 144 unibrow::Predicate<unibrow::LineTerminator, 128> kIsLineTerminator; |
| 145 unibrow::Predicate<unibrow::WhiteSpace, 128> kIsWhiteSpace; |
| 146 StaticResource<Utf8Decoder> utf8_decoder_; |
| 147 |
| 148 friend class Isolate; |
| 149 DISALLOW_COPY_AND_ASSIGN(ScannerConstants); |
| 142 }; | 150 }; |
| 143 | 151 |
| 144 // ---------------------------------------------------------------------------- | 152 // ---------------------------------------------------------------------------- |
| 145 // LiteralBuffer - Collector of chars of literals. | 153 // LiteralBuffer - Collector of chars of literals. |
| 146 | 154 |
| 147 class LiteralBuffer { | 155 class LiteralBuffer { |
| 148 public: | 156 public: |
| 149 LiteralBuffer() : is_ascii_(true), position_(0), backing_store_() { } | 157 LiteralBuffer() : is_ascii_(true), position_(0), backing_store_() { } |
| 150 | 158 |
| 151 ~LiteralBuffer() { | 159 ~LiteralBuffer() { |
| (...skipping 79 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 231 backing_store_.Dispose(); | 239 backing_store_.Dispose(); |
| 232 backing_store_ = new_store; | 240 backing_store_ = new_store; |
| 233 } | 241 } |
| 234 position_ = new_content_size; | 242 position_ = new_content_size; |
| 235 is_ascii_ = false; | 243 is_ascii_ = false; |
| 236 } | 244 } |
| 237 | 245 |
| 238 bool is_ascii_; | 246 bool is_ascii_; |
| 239 int position_; | 247 int position_; |
| 240 Vector<byte> backing_store_; | 248 Vector<byte> backing_store_; |
| 249 |
| 250 DISALLOW_COPY_AND_ASSIGN(LiteralBuffer); |
| 241 }; | 251 }; |
| 242 | 252 |
| 243 | 253 |
| 244 // ---------------------------------------------------------------------------- | 254 // ---------------------------------------------------------------------------- |
| 245 // Scanner base-class. | 255 // Scanner base-class. |
| 246 | 256 |
| 247 // Generic functionality used by both JSON and JavaScript scanners. | 257 // Generic functionality used by both JSON and JavaScript scanners. |
| 248 class Scanner { | 258 class Scanner { |
| 249 public: | 259 public: |
| 250 // -1 is outside of the range of any real source code. | 260 // -1 is outside of the range of any real source code. |
| 251 static const int kNoOctalLocation = -1; | 261 static const int kNoOctalLocation = -1; |
| 252 | 262 |
| 253 typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder; | 263 typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder; |
| 254 | 264 |
| 255 class LiteralScope { | 265 class LiteralScope { |
| 256 public: | 266 public: |
| 257 explicit LiteralScope(Scanner* self); | 267 explicit LiteralScope(Scanner* self); |
| 258 ~LiteralScope(); | 268 ~LiteralScope(); |
| 259 void Complete(); | 269 void Complete(); |
| 260 | 270 |
| 261 private: | 271 private: |
| 262 Scanner* scanner_; | 272 Scanner* scanner_; |
| 263 bool complete_; | 273 bool complete_; |
| 264 }; | 274 }; |
| 265 | 275 |
| 266 Scanner(); | 276 explicit Scanner(Isolate* isolate); |
| 267 | 277 |
| 268 // Returns the current token again. | 278 // Returns the current token again. |
| 269 Token::Value current_token() { return current_.token; } | 279 Token::Value current_token() { return current_.token; } |
| 270 | 280 |
| 271 // One token look-ahead (past the token returned by Next()). | 281 // One token look-ahead (past the token returned by Next()). |
| 272 Token::Value peek() const { return next_.token; } | 282 Token::Value peek() const { return next_.token; } |
| 273 | 283 |
| 274 struct Location { | 284 struct Location { |
| 275 Location(int b, int e) : beg_pos(b), end_pos(e) { } | 285 Location(int b, int e) : beg_pos(b), end_pos(e) { } |
| 276 Location() : beg_pos(0), end_pos(0) { } | 286 Location() : beg_pos(0), end_pos(0) { } |
| (...skipping 134 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 411 uc32 ScanHexEscape(uc32 c, int length); | 421 uc32 ScanHexEscape(uc32 c, int length); |
| 412 | 422 |
| 413 // Scans octal escape sequence. Also accepts "\0" decimal escape sequence. | 423 // Scans octal escape sequence. Also accepts "\0" decimal escape sequence. |
| 414 uc32 ScanOctalEscape(uc32 c, int length); | 424 uc32 ScanOctalEscape(uc32 c, int length); |
| 415 | 425 |
| 416 // Return the current source position. | 426 // Return the current source position. |
| 417 int source_pos() { | 427 int source_pos() { |
| 418 return source_->pos() - kCharacterLookaheadBufferSize; | 428 return source_->pos() - kCharacterLookaheadBufferSize; |
| 419 } | 429 } |
| 420 | 430 |
| 431 ScannerConstants* scanner_constants_; |
| 432 |
| 421 // Buffers collecting literal strings, numbers, etc. | 433 // Buffers collecting literal strings, numbers, etc. |
| 422 LiteralBuffer literal_buffer1_; | 434 LiteralBuffer literal_buffer1_; |
| 423 LiteralBuffer literal_buffer2_; | 435 LiteralBuffer literal_buffer2_; |
| 424 | 436 |
| 425 TokenDesc current_; // desc for current token (as returned by Next()) | 437 TokenDesc current_; // desc for current token (as returned by Next()) |
| 426 TokenDesc next_; // desc for next token (one token look-ahead) | 438 TokenDesc next_; // desc for next token (one token look-ahead) |
| 427 | 439 |
| 428 // Input stream. Must be initialized to an UC16CharacterStream. | 440 // Input stream. Must be initialized to an UC16CharacterStream. |
| 429 UC16CharacterStream* source_; | 441 UC16CharacterStream* source_; |
| 430 | 442 |
| (...skipping 24 matching lines...) Expand all Loading... |
| 455 void Complete() { | 467 void Complete() { |
| 456 scanner_->TerminateLiteral(); | 468 scanner_->TerminateLiteral(); |
| 457 complete_ = true; | 469 complete_ = true; |
| 458 } | 470 } |
| 459 | 471 |
| 460 private: | 472 private: |
| 461 JavaScriptScanner* scanner_; | 473 JavaScriptScanner* scanner_; |
| 462 bool complete_; | 474 bool complete_; |
| 463 }; | 475 }; |
| 464 | 476 |
| 465 JavaScriptScanner(); | 477 explicit JavaScriptScanner(Isolate* isolate); |
| 466 | 478 |
| 467 // Returns the next token. | 479 // Returns the next token. |
| 468 Token::Value Next(); | 480 Token::Value Next(); |
| 469 | 481 |
| 470 // Returns true if there was a line terminator before the peek'ed token. | 482 // Returns true if there was a line terminator before the peek'ed token. |
| 471 bool has_line_terminator_before_next() const { | 483 bool has_line_terminator_before_next() const { |
| 472 return has_line_terminator_before_next_; | 484 return has_line_terminator_before_next_; |
| 473 } | 485 } |
| 474 | 486 |
| 475 // Scans the input as a regular expression pattern, previous | 487 // Scans the input as a regular expression pattern, previous |
| (...skipping 168 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 644 // keyword with the current prefix). | 656 // keyword with the current prefix). |
| 645 const char* keyword_; | 657 const char* keyword_; |
| 646 int counter_; | 658 int counter_; |
| 647 Token::Value keyword_token_; | 659 Token::Value keyword_token_; |
| 648 }; | 660 }; |
| 649 | 661 |
| 650 | 662 |
| 651 } } // namespace v8::internal | 663 } } // namespace v8::internal |
| 652 | 664 |
| 653 #endif // V8_SCANNER_BASE_H_ | 665 #endif // V8_SCANNER_BASE_H_ |
| OLD | NEW |