OLD | NEW |
1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. |
6 | 6 |
7 #ifndef V8_PARSING_SCANNER_H_ | 7 #ifndef V8_PARSING_SCANNER_H_ |
8 #define V8_PARSING_SCANNER_H_ | 8 #define V8_PARSING_SCANNER_H_ |
9 | 9 |
10 #include "src/allocation.h" | 10 #include "src/allocation.h" |
(...skipping 201 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
212 // Similar functions for the upcoming token. | 212 // Similar functions for the upcoming token. |
213 | 213 |
214 // One token look-ahead (past the token returned by Next()). | 214 // One token look-ahead (past the token returned by Next()). |
215 Token::Value peek() const { return next_.token; } | 215 Token::Value peek() const { return next_.token; } |
216 | 216 |
217 Location peek_location() const { return next_.location; } | 217 Location peek_location() const { return next_.location; } |
218 | 218 |
219 bool literal_contains_escapes() const { | 219 bool literal_contains_escapes() const { |
220 return LiteralContainsEscapes(current_); | 220 return LiteralContainsEscapes(current_); |
221 } | 221 } |
222 bool next_literal_contains_escapes() const { | |
223 return LiteralContainsEscapes(next_); | |
224 } | |
225 bool is_literal_contextual_keyword(Vector<const char> keyword) { | 222 bool is_literal_contextual_keyword(Vector<const char> keyword) { |
| 223 DCHECK(current_.token == Token::IDENTIFIER || |
| 224 current_.token == Token::ESCAPED_STRICT_RESERVED_WORD); |
226 DCHECK_NOT_NULL(current_.literal_chars); | 225 DCHECK_NOT_NULL(current_.literal_chars); |
227 return current_.literal_chars->is_contextual_keyword(keyword); | 226 return current_.literal_chars->is_contextual_keyword(keyword); |
228 } | 227 } |
229 bool is_next_contextual_keyword(Vector<const char> keyword) { | 228 bool is_next_contextual_keyword(Vector<const char> keyword) { |
230 DCHECK_NOT_NULL(next_.literal_chars); | 229 DCHECK_NOT_NULL(next_.literal_chars); |
231 return next_.literal_chars->is_contextual_keyword(keyword); | 230 return next_.literal_chars->is_contextual_keyword(keyword); |
232 } | 231 } |
233 | 232 |
234 const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory); | 233 const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory); |
235 const AstRawString* NextSymbol(AstValueFactory* ast_value_factory); | 234 const AstRawString* NextSymbol(AstValueFactory* ast_value_factory); |
236 const AstRawString* CurrentRawSymbol(AstValueFactory* ast_value_factory); | 235 const AstRawString* CurrentRawSymbol(AstValueFactory* ast_value_factory); |
237 | 236 |
238 double DoubleValue(); | 237 double DoubleValue(); |
239 bool ContainsDot(); | 238 bool ContainsDot(); |
240 bool LiteralMatches(const char* data, int length, bool allow_escapes = true) { | 239 bool LiteralMatches(const char* data, int length, bool allow_escapes = true) { |
241 if (is_literal_one_byte() && | 240 if (!current_.literal_chars) { |
242 literal_length() == length && | 241 return !strncmp(Token::Name(current_.token), data, length); |
243 (allow_escapes || !literal_contains_escapes())) { | 242 } else if (is_literal_one_byte() && literal_length() == length && |
| 243 (allow_escapes || !literal_contains_escapes())) { |
244 const char* token = | 244 const char* token = |
245 reinterpret_cast<const char*>(literal_one_byte_string().start()); | 245 reinterpret_cast<const char*>(literal_one_byte_string().start()); |
246 return !strncmp(token, data, length); | 246 return !strncmp(token, data, length); |
247 } | 247 } |
248 return false; | 248 return false; |
249 } | 249 } |
250 inline bool UnescapedLiteralMatches(const char* data, int length) { | 250 inline bool UnescapedLiteralMatches(const char* data, int length) { |
251 return LiteralMatches(data, length, false); | 251 return LiteralMatches(data, length, false); |
252 } | 252 } |
253 | 253 |
(...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
292 return has_line_terminator_before_next_ || | 292 return has_line_terminator_before_next_ || |
293 has_multiline_comment_before_next_; | 293 has_multiline_comment_before_next_; |
294 } | 294 } |
295 | 295 |
296 bool HasAnyLineTerminatorAfterNext() { | 296 bool HasAnyLineTerminatorAfterNext() { |
297 Token::Value ensure_next_next = PeekAhead(); | 297 Token::Value ensure_next_next = PeekAhead(); |
298 USE(ensure_next_next); | 298 USE(ensure_next_next); |
299 return has_line_terminator_after_next_; | 299 return has_line_terminator_after_next_; |
300 } | 300 } |
301 | 301 |
302 // Scans the input as a regular expression pattern, previous | 302 // Scans the input as a regular expression pattern, next token must be /(=). |
303 // character(s) must be /(=). Returns true if a pattern is scanned. | 303 // Returns true if a pattern is scanned. |
304 bool ScanRegExpPattern(bool seen_equal); | 304 bool ScanRegExpPattern(); |
305 // Scans the input as regular expression flags. Returns the flags on success. | 305 // Scans the input as regular expression flags. Returns the flags on success. |
306 Maybe<RegExp::Flags> ScanRegExpFlags(); | 306 Maybe<RegExp::Flags> ScanRegExpFlags(); |
307 | 307 |
308 // Scans the input as a template literal | 308 // Scans the input as a template literal |
309 Token::Value ScanTemplateStart(); | 309 Token::Value ScanTemplateStart(); |
310 Token::Value ScanTemplateContinuation(); | 310 Token::Value ScanTemplateContinuation(); |
311 | 311 |
312 Handle<String> SourceUrl(Isolate* isolate) const { | 312 Handle<String> SourceUrl(Isolate* isolate) const { |
313 Handle<String> tmp; | 313 Handle<String> tmp; |
314 if (source_url_.length() > 0) tmp = source_url_.Internalize(isolate); | 314 if (source_url_.length() > 0) tmp = source_url_.Internalize(isolate); |
(...skipping 193 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
508 // Scans octal escape sequence. Also accepts "\0" decimal escape sequence. | 508 // Scans octal escape sequence. Also accepts "\0" decimal escape sequence. |
509 template <bool capture_raw> | 509 template <bool capture_raw> |
510 uc32 ScanOctalEscape(uc32 c, int length); | 510 uc32 ScanOctalEscape(uc32 c, int length); |
511 | 511 |
512 // Call this after setting source_ to the input. | 512 // Call this after setting source_ to the input. |
513 void Init() { | 513 void Init() { |
514 // Set c0_ (one character ahead) | 514 // Set c0_ (one character ahead) |
515 STATIC_ASSERT(kCharacterLookaheadBufferSize == 1); | 515 STATIC_ASSERT(kCharacterLookaheadBufferSize == 1); |
516 Advance(); | 516 Advance(); |
517 // Initialize current_ to not refer to a literal. | 517 // Initialize current_ to not refer to a literal. |
| 518 current_.token = Token::UNINITIALIZED; |
518 current_.literal_chars = NULL; | 519 current_.literal_chars = NULL; |
519 current_.raw_literal_chars = NULL; | 520 current_.raw_literal_chars = NULL; |
| 521 next_.token = Token::UNINITIALIZED; |
| 522 next_.literal_chars = NULL; |
| 523 next_.raw_literal_chars = NULL; |
520 next_next_.token = Token::UNINITIALIZED; | 524 next_next_.token = Token::UNINITIALIZED; |
| 525 next_next_.literal_chars = NULL; |
| 526 next_next_.raw_literal_chars = NULL; |
521 found_html_comment_ = false; | 527 found_html_comment_ = false; |
522 scanner_error_ = MessageTemplate::kNone; | 528 scanner_error_ = MessageTemplate::kNone; |
523 } | 529 } |
524 | 530 |
525 // Support BookmarkScope functionality. | 531 // Support BookmarkScope functionality. |
526 bool SetBookmark(); | 532 bool SetBookmark(); |
527 void ResetToBookmark(); | 533 void ResetToBookmark(); |
528 bool BookmarkHasBeenSet(); | 534 bool BookmarkHasBeenSet(); |
529 bool BookmarkHasBeenReset(); | 535 bool BookmarkHasBeenReset(); |
530 void DropBookmark(); | 536 void DropBookmark(); |
(...skipping 112 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
643 } | 649 } |
644 } | 650 } |
645 | 651 |
646 // Returns the literal string, if any, for the current token (the | 652 // Returns the literal string, if any, for the current token (the |
647 // token last returned by Next()). The string is 0-terminated. | 653 // token last returned by Next()). The string is 0-terminated. |
648 // Literal strings are collected for identifiers, strings, numbers as well | 654 // Literal strings are collected for identifiers, strings, numbers as well |
649 // as for template literals. For template literals we also collect the raw | 655 // as for template literals. For template literals we also collect the raw |
650 // form. | 656 // form. |
651 // These functions only give the correct result if the literal was scanned | 657 // These functions only give the correct result if the literal was scanned |
652 // when a LiteralScope object is alive. | 658 // when a LiteralScope object is alive. |
| 659 // |
| 660 // Current usage of these functions is unfortunately a little undisciplined, |
| 661 // and is_literal_one_byte() + is_literal_one_byte_string() is also |
| 662 // requested for tokens that do not have a literal. Hence, we treat any |
| 663 // token as a one-byte literal. E.g. Token::FUNCTION pretends to have a |
| 664 // literal "function". |
653 Vector<const uint8_t> literal_one_byte_string() { | 665 Vector<const uint8_t> literal_one_byte_string() { |
654 DCHECK_NOT_NULL(current_.literal_chars); | 666 if (current_.literal_chars) |
655 return current_.literal_chars->one_byte_literal(); | 667 return current_.literal_chars->one_byte_literal(); |
| 668 const char* str = Token::String(current_.token); |
| 669 const uint8_t* str_as_uint8 = reinterpret_cast<const uint8_t*>(str); |
| 670 return Vector<const uint8_t>(str_as_uint8, |
| 671 Token::StringLength(current_.token)); |
656 } | 672 } |
657 Vector<const uint16_t> literal_two_byte_string() { | 673 Vector<const uint16_t> literal_two_byte_string() { |
658 DCHECK_NOT_NULL(current_.literal_chars); | 674 DCHECK_NOT_NULL(current_.literal_chars); |
659 return current_.literal_chars->two_byte_literal(); | 675 return current_.literal_chars->two_byte_literal(); |
660 } | 676 } |
661 bool is_literal_one_byte() { | 677 bool is_literal_one_byte() { |
662 DCHECK_NOT_NULL(current_.literal_chars); | 678 return !current_.literal_chars || current_.literal_chars->is_one_byte(); |
663 return current_.literal_chars->is_one_byte(); | |
664 } | 679 } |
665 int literal_length() const { | 680 int literal_length() const { |
666 DCHECK_NOT_NULL(current_.literal_chars); | 681 if (current_.literal_chars) return current_.literal_chars->length(); |
667 return current_.literal_chars->length(); | 682 return Token::StringLength(current_.token); |
668 } | 683 } |
669 // Returns the literal string for the next token (the token that | 684 // Returns the literal string for the next token (the token that |
670 // would be returned if Next() were called). | 685 // would be returned if Next() were called). |
671 Vector<const uint8_t> next_literal_one_byte_string() { | 686 Vector<const uint8_t> next_literal_one_byte_string() { |
672 DCHECK_NOT_NULL(next_.literal_chars); | 687 DCHECK_NOT_NULL(next_.literal_chars); |
673 return next_.literal_chars->one_byte_literal(); | 688 return next_.literal_chars->one_byte_literal(); |
674 } | 689 } |
675 Vector<const uint16_t> next_literal_two_byte_string() { | 690 Vector<const uint16_t> next_literal_two_byte_string() { |
676 DCHECK_NOT_NULL(next_.literal_chars); | 691 DCHECK_NOT_NULL(next_.literal_chars); |
677 return next_.literal_chars->two_byte_literal(); | 692 return next_.literal_chars->two_byte_literal(); |
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
739 return static_cast<int>(source_->pos()) - kCharacterLookaheadBufferSize; | 754 return static_cast<int>(source_->pos()) - kCharacterLookaheadBufferSize; |
740 } | 755 } |
741 | 756 |
742 static bool LiteralContainsEscapes(const TokenDesc& token) { | 757 static bool LiteralContainsEscapes(const TokenDesc& token) { |
743 Location location = token.location; | 758 Location location = token.location; |
744 int source_length = (location.end_pos - location.beg_pos); | 759 int source_length = (location.end_pos - location.beg_pos); |
745 if (token.token == Token::STRING) { | 760 if (token.token == Token::STRING) { |
746 // Subtract delimiters. | 761 // Subtract delimiters. |
747 source_length -= 2; | 762 source_length -= 2; |
748 } | 763 } |
749 return token.literal_chars->length() != source_length; | 764 return token.literal_chars && |
| 765 (token.literal_chars->length() != source_length); |
750 } | 766 } |
751 | 767 |
| 768 #ifdef DEBUG |
| 769 void SanityCheckTokenDesc(const TokenDesc&) const; |
| 770 #endif |
| 771 |
752 UnicodeCache* unicode_cache_; | 772 UnicodeCache* unicode_cache_; |
753 | 773 |
754 // Buffers collecting literal strings, numbers, etc. | 774 // Buffers collecting literal strings, numbers, etc. |
755 LiteralBuffer literal_buffer0_; | 775 LiteralBuffer literal_buffer0_; |
756 LiteralBuffer literal_buffer1_; | 776 LiteralBuffer literal_buffer1_; |
757 LiteralBuffer literal_buffer2_; | 777 LiteralBuffer literal_buffer2_; |
758 | 778 |
759 // Values parsed from magic comments. | 779 // Values parsed from magic comments. |
760 LiteralBuffer source_url_; | 780 LiteralBuffer source_url_; |
761 LiteralBuffer source_mapping_url_; | 781 LiteralBuffer source_mapping_url_; |
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
823 bool found_html_comment_; | 843 bool found_html_comment_; |
824 | 844 |
825 MessageTemplate::Template scanner_error_; | 845 MessageTemplate::Template scanner_error_; |
826 Location scanner_error_location_; | 846 Location scanner_error_location_; |
827 }; | 847 }; |
828 | 848 |
829 } // namespace internal | 849 } // namespace internal |
830 } // namespace v8 | 850 } // namespace v8 |
831 | 851 |
832 #endif // V8_PARSING_SCANNER_H_ | 852 #endif // V8_PARSING_SCANNER_H_ |
OLD | NEW |