Experimental lexer: fix internalization and allocation of literals.

src/lexer/experimental-scanner.h

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 139 matching lines @@
     harmony_numeric_literals_ = numeric_literals;
   }
 
   // Returns true if there was a line terminator before the peek'ed token,
   // possibly inside a multi-line comment.
   bool HasAnyLineTerminatorBeforeNext() const {
     return has_line_terminator_before_next_ ||
            has_multiline_comment_before_next_;
   }
 
+  Handle<String> GetLiteralSymbol() {
+    EnsureCurrentLiteralIsValid();
+    return InternalizeLiteral(current_literal_);
+  }
+
+  Handle<String> GetLiteralString(PretenureFlag tenured) {
+    EnsureCurrentLiteralIsValid();
+    return AllocateLiteral(current_literal_, tenured);
+  }
+
+  Handle<String> GetNextLiteralString(PretenureFlag tenured) {
+    EnsureNextLiteralIsValid();
+    return AllocateLiteral(next_literal_, tenured);
+  }
+
   Vector<const char> literal_ascii_string() {
-    if (!current_literal_->Valid(current_.beg_pos)) {
-      FillLiteral(current_, current_literal_);
-    }
+    EnsureCurrentLiteralIsValid();
     return current_literal_->ascii_string;
   }
 
   Vector<const uc16> literal_utf16_string() {
-    if (!current_literal_->Valid(current_.beg_pos)) {
-      FillLiteral(current_, current_literal_);
-    }
+    EnsureCurrentLiteralIsValid();
     return current_literal_->utf16_string;
   }
 
   int literal_length() {
-    if (!current_literal_->Valid(current_.beg_pos)) {
-      FillLiteral(current_, current_literal_);
-    }
+    EnsureCurrentLiteralIsValid();
     return current_literal_->length;
   }
 
   // This should be is_onebyte or is_latin1; it doesn't mean ASCII for real.
   bool is_literal_ascii() {
-    if (!current_literal_->Valid(current_.beg_pos)) {
-      FillLiteral(current_, current_literal_);
-    }
+    EnsureCurrentLiteralIsValid();
     return current_literal_->is_ascii;
   }
 
   bool is_literal_contextual_keyword(Vector<const char> keyword) {
     if (!is_literal_ascii()) return false;
     Vector<const char> literal = literal_ascii_string();
     return literal.length() == keyword.length() &&
         (memcmp(literal.start(), keyword.start(), literal.length()) == 0);
   }
 
   bool literal_contains_escapes() const {
     return current_.has_escapes;
   }
 
   Vector<const char> next_literal_ascii_string() {
-    if (!next_literal_->Valid(next_.beg_pos)) {
-      FillLiteral(next_, next_literal_);
-    }
+    EnsureNextLiteralIsValid();
     return next_literal_->ascii_string;
   }
 
   Vector<const uc16> next_literal_utf16_string() {
-    if (!next_literal_->Valid(next_.beg_pos)) {
-      FillLiteral(next_, next_literal_);
-    }
+    EnsureNextLiteralIsValid();
     return next_literal_->utf16_string;
   }
 
   int next_literal_length() {
-    if (!next_literal_->Valid(next_.beg_pos)) {
-      FillLiteral(next_, next_literal_);
-    }
+    EnsureNextLiteralIsValid();
     return next_literal_->length;
   }
 
   bool is_next_literal_ascii() {
-    if (!next_literal_->Valid(next_.beg_pos)) {
-      FillLiteral(next_, next_literal_);
-    }
+    EnsureNextLiteralIsValid();
     return next_literal_->is_ascii;
   }
 
   bool is_next_contextual_keyword(Vector<const char> keyword) {
     if (!is_next_literal_ascii()) return false;
     Vector<const char> literal = next_literal_ascii_string();
     return literal.length() == keyword.length() &&
         (memcmp(literal.start(), keyword.start(), literal.length()) == 0);
   }
 
  protected:
   struct TokenDesc {
     Token::Value token;
     int beg_pos;
     int end_pos;
     bool has_escapes;
   };
 
   struct LiteralDesc {
     int beg_pos;
     bool is_ascii;
+    bool is_in_buffer;
+    int offset;
     int length;
     Vector<const char> ascii_string;
     Vector<const uc16> utf16_string;
     LiteralBuffer buffer;
-    LiteralDesc() : beg_pos(-1), is_ascii(false), length(0) { }
+    LiteralDesc() : beg_pos(-1), is_ascii(false), is_in_buffer(false),
+                    offset(0), length(0) { }
     bool Valid(int pos) { return beg_pos == pos; }
   };
 
   virtual void Scan() = 0;
   virtual bool FillLiteral(const TokenDesc& token, LiteralDesc* literal) = 0;
 
   void ResetLiterals() {
-    current_literal_->beg_pos = -1;
-    next_literal_->beg_pos = -1;
+    if (!current_literal_->is_in_buffer) current_literal_->beg_pos = -1;
+    if (!next_literal_->is_in_buffer) next_literal_->beg_pos = -1;

[marja, 2014/01/20 07:54:33]

Shouldn't we reset is_in_buffer here too? Why not?

[ulan, 2014/01/20 09:15:00]

beg_pos == -1 indicates an invalid buffer, for invalid buffer we do not look at
is_in_buffer.

   }
 
+  void EnsureCurrentLiteralIsValid() {
+    if (!current_literal_->Valid(current_.beg_pos)) {
+      FillLiteral(current_, current_literal_);
+    }
+  }
+
+  void EnsureNextLiteralIsValid() {
+    if (!next_literal_->Valid(next_.beg_pos)) {
+      FillLiteral(next_, next_literal_);
+    }
+  }
+
+  virtual Handle<String> InternalizeLiteral(LiteralDesc* literal) = 0;
+  virtual Handle<String> AllocateLiteral(LiteralDesc* literal,
+                                         PretenureFlag tenured) = 0;
+
   Isolate* isolate_;
   UnicodeCache* unicode_cache_;
 
   bool has_line_terminator_before_next_;
   // Whether there is a multiline comment *with a line break* before the next
   // token.
   bool has_multiline_comment_before_next_;
 
   TokenDesc current_;  // desc for current token (as returned by Next())
   TokenDesc next_;     // desc for next token (one token look-ahead)
@@ skipping 60 matching lines @@
       ResetLiterals();
     }
   }
 
  protected:
   virtual void Scan();
 
   const Char* GetNewBufferBasedOnHandle() const;
 
   virtual bool FillLiteral(const TokenDesc& token, LiteralDesc* literal);
+  virtual Handle<String> InternalizeLiteral(LiteralDesc* literal);
+  virtual Handle<String> AllocateLiteral(LiteralDesc* literal,
+                                         PretenureFlag tenured);
+
 
  private:
   bool ValidIdentifierPart() {
       return unicode_cache_->IsIdentifierPart(ScanHexNumber(4));
   }
 
   bool ValidIdentifierStart() {
     return unicode_cache_->IsIdentifierStart(ScanHexNumber(4));
   }
 
   uc32 ScanHexNumber(int length);
   bool ScanLiteralUnicodeEscape();
 
   const Char* ScanHexNumber(const Char* start,
                             const Char* end,
                             uc32* result);
   const Char* ScanOctalEscape(const Char* start,
                               const Char* end,
                               uc32* result);
   const Char* ScanIdentifierUnicodeEscape(const Char* start,
                                           const Char* end,
                                           uc32* result);
   const Char* ScanEscape(const Char* start,
                          const Char* end,
                          LiteralBuffer* literal);
 
+  bool IsSubstringOfSource(const TokenDesc& token);
+
+  bool CopyToLiteralBuffer(const Char* start,
+                           const Char* end,
+                           const TokenDesc& token,
+                           LiteralDesc* literal);
+
   Handle<String> source_handle_;
   const Char* buffer_;
   const Char* buffer_end_;
   const Char* start_;
   const Char* cursor_;
 
   // Where we have seen the last octal number or an octal escape inside a
   // string. Used by octal_position().
   const Char* last_octal_end_;
 };
@@ skipping 228 matching lines @@
   // character.
   const Char* temp_cursor = last_octal_end_ - 1;
   while (temp_cursor >= buffer_ && *temp_cursor >= '0' && *temp_cursor <= '7')
     --temp_cursor;
   return Location(temp_cursor - buffer_ + 1, last_octal_end_ - buffer_);
 }
 
 } }
 
 #endif  // V8_LEXER_EXPERIMENTAL_SCANNER_H