Scanner::LiteralBuffer usage cleanup.

src/parsing/scanner.h

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // Features shared by parsing and pre-parsing scanners.
 
 #ifndef V8_PARSING_SCANNER_H_
 #define V8_PARSING_SCANNER_H_
 
 #include "src/allocation.h"
@@ skipping 201 matching lines @@
   // Similar functions for the upcoming token.
 
   // One token look-ahead (past the token returned by Next()).
   Token::Value peek() const { return next_.token; }
 
   Location peek_location() const { return next_.location; }
 
   bool literal_contains_escapes() const {
     return LiteralContainsEscapes(current_);
   }
-  bool next_literal_contains_escapes() const {
-    return LiteralContainsEscapes(next_);
-  }
   bool is_literal_contextual_keyword(Vector<const char> keyword) {
+    DCHECK(current_.token == Token::IDENTIFIER ||
+           current_.token == Token::ESCAPED_STRICT_RESERVED_WORD);
     DCHECK_NOT_NULL(current_.literal_chars);
     return current_.literal_chars->is_contextual_keyword(keyword);
   }
   bool is_next_contextual_keyword(Vector<const char> keyword) {
     DCHECK_NOT_NULL(next_.literal_chars);
     return next_.literal_chars->is_contextual_keyword(keyword);
   }
 
   const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory);
   const AstRawString* NextSymbol(AstValueFactory* ast_value_factory);
   const AstRawString* CurrentRawSymbol(AstValueFactory* ast_value_factory);
 
   double DoubleValue();
   bool ContainsDot();
   bool LiteralMatches(const char* data, int length, bool allow_escapes = true) {
-    if (is_literal_one_byte() &&
-        literal_length() == length &&
-        (allow_escapes || !literal_contains_escapes())) {
+    if (!current_.literal_chars) {
+      return !strncmp(Token::Name(current_.token), data, length);
+    } else if (is_literal_one_byte() && literal_length() == length &&
+               (allow_escapes || !literal_contains_escapes())) {
       const char* token =
           reinterpret_cast<const char*>(literal_one_byte_string().start());
       return !strncmp(token, data, length);
     }
     return false;
   }
   inline bool UnescapedLiteralMatches(const char* data, int length) {
     return LiteralMatches(data, length, false);
   }
 
@@ skipping 38 matching lines @@
     return has_line_terminator_before_next_ ||
            has_multiline_comment_before_next_;
   }
 
   bool HasAnyLineTerminatorAfterNext() {
     Token::Value ensure_next_next = PeekAhead();
     USE(ensure_next_next);
     return has_line_terminator_after_next_;
   }
 
-  // Scans the input as a regular expression pattern, previous
-  // character(s) must be /(=). Returns true if a pattern is scanned.
-  bool ScanRegExpPattern(bool seen_equal);
+  // Scans the input as a regular expression pattern, next token must be /(=).
+  // Returns true if a pattern is scanned.
+  bool ScanRegExpPattern();
   // Scans the input as regular expression flags. Returns the flags on success.
   Maybe<RegExp::Flags> ScanRegExpFlags();
 
   // Scans the input as a template literal
   Token::Value ScanTemplateStart();
   Token::Value ScanTemplateContinuation();
 
   Handle<String> SourceUrl(Isolate* isolate) const {
     Handle<String> tmp;
     if (source_url_.length() > 0) tmp = source_url_.Internalize(isolate);
@@ skipping 193 matching lines @@
   // Scans octal escape sequence. Also accepts "\0" decimal escape sequence.
   template <bool capture_raw>
   uc32 ScanOctalEscape(uc32 c, int length);
 
   // Call this after setting source_ to the input.
   void Init() {
     // Set c0_ (one character ahead)
     STATIC_ASSERT(kCharacterLookaheadBufferSize == 1);
     Advance();
     // Initialize current_ to not refer to a literal.
+    current_.token = Token::UNINITIALIZED;
     current_.literal_chars = NULL;
     current_.raw_literal_chars = NULL;
+    next_.token = Token::UNINITIALIZED;
+    next_.literal_chars = NULL;
+    next_.raw_literal_chars = NULL;
     next_next_.token = Token::UNINITIALIZED;
+    next_next_.literal_chars = NULL;
+    next_next_.raw_literal_chars = NULL;
     found_html_comment_ = false;
     scanner_error_ = MessageTemplate::kNone;
   }
 
   // Support BookmarkScope functionality.
   bool SetBookmark();
   void ResetToBookmark();
   bool BookmarkHasBeenSet();
   bool BookmarkHasBeenReset();
   void DropBookmark();
@@ skipping 112 matching lines @@
     }
   }
 
   // Returns the literal string, if any, for the current token (the
   // token last returned by Next()). The string is 0-terminated.
   // Literal strings are collected for identifiers, strings, numbers as well
   // as for template literals. For template literals we also collect the raw
   // form.
   // These functions only give the correct result if the literal was scanned
   // when a LiteralScope object is alive.
+  //
+  // Current usage of these functions is unfortunately a little undisciplined,
+  // and is_literal_one_byte() + is_literal_one_byte_string() is also
+  // requested for tokens that do not have a literal. Hence, we treat any
+  // token as a one-byte literal. E.g. Token::FUNCTION pretends to have a
+  // literal "function".
   Vector<const uint8_t> literal_one_byte_string() {
-    DCHECK_NOT_NULL(current_.literal_chars);
-    return current_.literal_chars->one_byte_literal();
+    if (current_.literal_chars)
+      return current_.literal_chars->one_byte_literal();
+    const char* str = Token::String(current_.token);
+    const uint8_t* str_as_uint8 = reinterpret_cast<const uint8_t*>(str);
+    return Vector<const uint8_t>(str_as_uint8,
+                                 Token::StringLength(current_.token));
   }
   Vector<const uint16_t> literal_two_byte_string() {
     DCHECK_NOT_NULL(current_.literal_chars);
     return current_.literal_chars->two_byte_literal();
   }
   bool is_literal_one_byte() {
-    DCHECK_NOT_NULL(current_.literal_chars);
-    return current_.literal_chars->is_one_byte();
+    return !current_.literal_chars || current_.literal_chars->is_one_byte();
   }
   int literal_length() const {
-    DCHECK_NOT_NULL(current_.literal_chars);
-    return current_.literal_chars->length();
+    if (current_.literal_chars) return current_.literal_chars->length();
+    return Token::StringLength(current_.token);
   }
   // Returns the literal string for the next token (the token that
   // would be returned if Next() were called).
   Vector<const uint8_t> next_literal_one_byte_string() {
     DCHECK_NOT_NULL(next_.literal_chars);
     return next_.literal_chars->one_byte_literal();
   }
   Vector<const uint16_t> next_literal_two_byte_string() {
     DCHECK_NOT_NULL(next_.literal_chars);
     return next_.literal_chars->two_byte_literal();
@@ skipping 61 matching lines @@
     return static_cast<int>(source_->pos()) - kCharacterLookaheadBufferSize;
   }
 
   static bool LiteralContainsEscapes(const TokenDesc& token) {
     Location location = token.location;
     int source_length = (location.end_pos - location.beg_pos);
     if (token.token == Token::STRING) {
       // Subtract delimiters.
       source_length -= 2;
     }
-    return token.literal_chars->length() != source_length;
+    return token.literal_chars &&
+           (token.literal_chars->length() != source_length);
   }
 
+#ifdef DEBUG
+  void SanityCheckTokenDesc(const TokenDesc&) const;
+#endif
+
   UnicodeCache* unicode_cache_;
 
   // Buffers collecting literal strings, numbers, etc.
   LiteralBuffer literal_buffer0_;
   LiteralBuffer literal_buffer1_;
   LiteralBuffer literal_buffer2_;
 
   // Values parsed from magic comments.
   LiteralBuffer source_url_;
   LiteralBuffer source_mapping_url_;
@@ skipping 61 matching lines @@
   bool found_html_comment_;
 
   MessageTemplate::Template scanner_error_;
   Location scanner_error_location_;
 };
 
 }  // namespace internal
 }  // namespace v8
 
 #endif  // V8_PARSING_SCANNER_H_