Experimental lexer: fix internalization and allocation of literals.

src/lexer/experimental-scanner.h

Index: src/lexer/experimental-scanner.h
diff --git a/src/lexer/experimental-scanner.h b/src/lexer/experimental-scanner.h
index 8a9a96e1e4df2560ba0841ccd86d1201b60030d6..31c5e843808d29af097101b0c8721003b56176b8 100644
--- a/src/lexer/experimental-scanner.h
+++ b/src/lexer/experimental-scanner.h
@@ -157,32 +157,39 @@ class ScannerBase {
            has_multiline_comment_before_next_;
   }
 
+  Handle<String> GetLiteralSymbol() {
+    EnsureCurrentLiteralIsValid();
+    return InternalizeLiteral(current_literal_);
+  }
+
+  Handle<String> GetLiteralString(PretenureFlag tenured) {
+    EnsureCurrentLiteralIsValid();
+    return AllocateLiteral(current_literal_, tenured);
+  }
+
+  Handle<String> GetNextLiteralString(PretenureFlag tenured) {
+    EnsureNextLiteralIsValid();
+    return AllocateLiteral(next_literal_, tenured);
+  }
+
   Vector<const char> literal_ascii_string() {
-    if (!current_literal_->Valid(current_.beg_pos)) {
-      FillLiteral(current_, current_literal_);
-    }
+    EnsureCurrentLiteralIsValid();
     return current_literal_->ascii_string;
   }
 
   Vector<const uc16> literal_utf16_string() {
-    if (!current_literal_->Valid(current_.beg_pos)) {
-      FillLiteral(current_, current_literal_);
-    }
+    EnsureCurrentLiteralIsValid();
     return current_literal_->utf16_string;
   }
 
   int literal_length() {
-    if (!current_literal_->Valid(current_.beg_pos)) {
-      FillLiteral(current_, current_literal_);
-    }
+    EnsureCurrentLiteralIsValid();
     return current_literal_->length;
   }
 
   // This should be is_onebyte or is_latin1; it doesn't mean ASCII for real.
   bool is_literal_ascii() {
-    if (!current_literal_->Valid(current_.beg_pos)) {
-      FillLiteral(current_, current_literal_);
-    }
+    EnsureCurrentLiteralIsValid();
     return current_literal_->is_ascii;
   }
 
@@ -198,30 +205,22 @@ class ScannerBase {
   }
 
   Vector<const char> next_literal_ascii_string() {
-    if (!next_literal_->Valid(next_.beg_pos)) {
-      FillLiteral(next_, next_literal_);
-    }
+    EnsureNextLiteralIsValid();
     return next_literal_->ascii_string;
   }
 
   Vector<const uc16> next_literal_utf16_string() {
-    if (!next_literal_->Valid(next_.beg_pos)) {
-      FillLiteral(next_, next_literal_);
-    }
+    EnsureNextLiteralIsValid();
     return next_literal_->utf16_string;
   }
 
   int next_literal_length() {
-    if (!next_literal_->Valid(next_.beg_pos)) {
-      FillLiteral(next_, next_literal_);
-    }
+    EnsureNextLiteralIsValid();
     return next_literal_->length;
   }
 
   bool is_next_literal_ascii() {
-    if (!next_literal_->Valid(next_.beg_pos)) {
-      FillLiteral(next_, next_literal_);
-    }
+    EnsureNextLiteralIsValid();
     return next_literal_->is_ascii;
   }
 
@@ -243,11 +242,14 @@ class ScannerBase {
   struct LiteralDesc {
     int beg_pos;
     bool is_ascii;
+    bool is_in_buffer;
+    int offset;
     int length;
     Vector<const char> ascii_string;
     Vector<const uc16> utf16_string;
     LiteralBuffer buffer;
-    LiteralDesc() : beg_pos(-1), is_ascii(false), length(0) { }
+    LiteralDesc() : beg_pos(-1), is_ascii(false), is_in_buffer(false),
+                    offset(0), length(0) { }
     bool Valid(int pos) { return beg_pos == pos; }
   };
 
@@ -255,10 +257,26 @@ class ScannerBase {
   virtual bool FillLiteral(const TokenDesc& token, LiteralDesc* literal) = 0;
 
   void ResetLiterals() {
-    current_literal_->beg_pos = -1;
-    next_literal_->beg_pos = -1;
+    if (!current_literal_->is_in_buffer) current_literal_->beg_pos = -1;
+    if (!next_literal_->is_in_buffer) next_literal_->beg_pos = -1;

[marja, 2014/01/20 07:54:33]

Shouldn't we reset is_in_buffer here too? Why not?

[ulan, 2014/01/20 09:15:00]

beg_pos == -1 indicates an invalid buffer, for invalid buffer we do not look at
is_in_buffer.

+  }
+
+  void EnsureCurrentLiteralIsValid() {
+    if (!current_literal_->Valid(current_.beg_pos)) {
+      FillLiteral(current_, current_literal_);
+    }
   }
 
+  void EnsureNextLiteralIsValid() {
+    if (!next_literal_->Valid(next_.beg_pos)) {
+      FillLiteral(next_, next_literal_);
+    }
+  }
+
+  virtual Handle<String> InternalizeLiteral(LiteralDesc* literal) = 0;
+  virtual Handle<String> AllocateLiteral(LiteralDesc* literal,
+                                         PretenureFlag tenured) = 0;
+
   Isolate* isolate_;
   UnicodeCache* unicode_cache_;
 
@@ -339,6 +357,10 @@ class ExperimentalScanner : public ScannerBase {
   const Char* GetNewBufferBasedOnHandle() const;
 
   virtual bool FillLiteral(const TokenDesc& token, LiteralDesc* literal);
+  virtual Handle<String> InternalizeLiteral(LiteralDesc* literal);
+  virtual Handle<String> AllocateLiteral(LiteralDesc* literal,
+                                         PretenureFlag tenured);
+
 
  private:
   bool ValidIdentifierPart() {
@@ -365,6 +387,13 @@ class ExperimentalScanner : public ScannerBase {
                          const Char* end,
                          LiteralBuffer* literal);
 
+  bool IsSubstringOfSource(const TokenDesc& token);
+
+  bool CopyToLiteralBuffer(const Char* start,
+                           const Char* end,
+                           const TokenDesc& token,
+                           LiteralDesc* literal);
+
   Handle<String> source_handle_;
   const Char* buffer_;
   const Char* buffer_end_;