Experimental parser: allocate substrings

src/lexer/lexer.h

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 155 matching lines @@
     return next_literal_->is_one_byte();
   }
 
   bool is_next_contextual_keyword(Vector<const uint8_t> keyword) {
     if (!is_next_literal_one_byte()) return false;
     Vector<const uint8_t> literal = next_literal_one_byte_string();
     return literal.length() == keyword.length() &&
         (memcmp(literal.start(), keyword.start(), literal.length()) == 0);
   }
 
-  Handle<String> AllocateNextLiteralString(Isolate* isolate,
-                                           PretenureFlag tenured);
-  Handle<String> AllocateInternalizedString(Isolate* isolate);
+  virtual Handle<String> AllocateNextLiteralString(Isolate* isolate,
+                                                   PretenureFlag tenured) = 0;
+  virtual Handle<String> AllocateInternalizedString(Isolate* isolate) = 0;
 
   double DoubleValue();
   bool UnescapedLiteralMatches(const char* data, int length) {
     if (is_literal_one_byte() &&
         literal_length() == length &&
         !literal_contains_escapes()) {
       const char* token =
           reinterpret_cast<const char*>(literal_one_byte_string().start());
       return !strncmp(token, data, length);
     }
@@ skipping 51 matching lines @@
           is_in_buffer_(false),
           is_one_byte_string_owned_(false)  // TODO(dcarney): move to buffer
     { }
 
     ~LiteralDesc() {
       if (is_one_byte_string_owned_) {
         one_byte_string_.Dispose();
       }
     }
 
+    inline bool is_in_buffer() { return is_in_buffer_; }
     inline bool is_one_byte() { return is_one_byte_; }
     inline Vector<const uint8_t> one_byte_string() {
       ASSERT(is_one_byte_);
       return one_byte_string_;
     }
     inline  Vector<const uint16_t> two_byte_string() {
       ASSERT(!is_one_byte_);
       return two_byte_string_;
     }
 
     inline bool Valid(int pos) { return beg_pos == pos; }
-    inline void Invalidate() { if (is_in_buffer_) beg_pos = -1; }
+    inline void Invalidate() {
+      if (!is_in_buffer_ && !is_one_byte_string_owned_) beg_pos = -1;
+    }
 
     // TODO(dcarney): make private as well.
     int beg_pos;
     int offset;
     int length;
     LiteralBuffer buffer;
 
     void SetOneByteString(Vector<const uint8_t> string, bool owned);
     void SetTwoByteString(Vector<const uint16_t> string);
     void SetStringFromLiteralBuffer();
 
    private:
     bool is_one_byte_;
     bool is_in_buffer_;
     bool is_one_byte_string_owned_;
     Vector<const uint8_t> one_byte_string_;
     Vector<const uint16_t> two_byte_string_;
 
     DISALLOW_COPY_AND_ASSIGN(LiteralDesc);
   };
 
- protected:
   struct TokenDesc {
     int beg_pos;
     int end_pos;
     Token::Value token;
     bool has_escapes;
-    bool is_onebyte;
+    bool is_in_primary_range;
   };
 
+ protected:
   virtual void Scan() = 0;
   virtual void UpdateBufferBasedOnHandle() = 0;
   virtual bool FillLiteral(const TokenDesc& token, LiteralDesc* literal) = 0;
-  virtual Handle<String> InternalizeLiteral(LiteralDesc* literal) = 0;
-  virtual Handle<String> AllocateLiteral(LiteralDesc* literal,
-                                         PretenureFlag tenured) = 0;
+
+  void EnsureLiteralIsValid(const TokenDesc& token, LiteralDesc* literal) {
+    if (!literal->Valid(token.beg_pos)) FillLiteral(token, literal);
+  }
 
   void EnsureCurrentLiteralIsValid() {
-    if (!current_literal_->Valid(current_.beg_pos)) {
-      FillLiteral(current_, current_literal_);
-    }
+    EnsureLiteralIsValid(current_, current_literal_);
   }
 
   void EnsureNextLiteralIsValid() {
-    if (!next_literal_->Valid(next_.beg_pos)) {
-      FillLiteral(next_, next_literal_);
-    }
+    EnsureLiteralIsValid(next_, next_literal_);
   }
 
   UnicodeCache* unicode_cache_;
   LiteralDesc* current_literal_;
   LiteralDesc* next_literal_;
   LiteralDesc literals_[2];
 
   TokenDesc current_;  // desc for current token (as returned by Next())
   TokenDesc next_;     // desc for next token (one token look-ahead)
 
@@ skipping 20 matching lines @@
         int end_position);
   Lexer(UnicodeCache* unicode_cache, const Char* source_ptr, int length);
   virtual ~Lexer();
 
   virtual void SeekForward(int pos);
   virtual bool ScanRegExpPattern(bool seen_equal);
   virtual bool ScanRegExpFlags();
   virtual Location octal_position() const;
   virtual void clear_octal_position() { last_octal_end_ = NULL; }
 
+  static inline bool MustBeInBuffer(const TokenDesc& token) {
+    return token.has_escapes ||
+        (sizeof(Char) == 1 && !token.is_in_primary_range);
+  }
+
  protected:
   virtual void Scan();
 
  private:
   uc32 ScanHexNumber(int length);
 
   bool ScanLiteralUnicodeEscape();
 
   const Char* GetNewBufferBasedOnHandle() const;
   virtual void UpdateBufferBasedOnHandle();
 
   virtual bool FillLiteral(const TokenDesc& token, LiteralDesc* literal);
-  virtual Handle<String> InternalizeLiteral(LiteralDesc* literal);
-  virtual Handle<String> AllocateLiteral(LiteralDesc* literal,
-                                         PretenureFlag tenured);
+
+  virtual Handle<String> AllocateNextLiteralString(Isolate* isolate,
+                                                   PretenureFlag tenured);
+  virtual Handle<String> AllocateInternalizedString(Isolate* isolate);
 
   // Helper function for FillLiteral.
   template<bool is_one_byte>
-  static void SetLiteral(
-      const Char* start, const Char* end, LiteralDesc* literal);
+  static void SetLiteral(const Char* start, LiteralDesc* literal);
 
-  bool CopyToLiteralBuffer(const Char* start,
-                           const Char* end,
-                           const TokenDesc& token,
+  bool CopyToLiteralBuffer(const TokenDesc& token,
                            LiteralDesc* literal);
 
   // One of source_handle_ or source_ptr_ is set.
   // If source_ptr_ is set, isolate_ is 0 and no isolate accesses are allowed.
   Isolate* isolate_;
   const Handle<String> source_handle_;
   const Char* const source_ptr_;
   const int end_position_;
   // Stream variables.
   const Char* buffer_;
@@ skipping 128 matching lines @@
   bool harmony_scoping_;
 };
 
 
 #endif
 
 
 } }
 
 #endif  // V8_LEXER_EXPERIMENTAL_SCANNER_H