Experimental scanner += API which takes Handle<String>.

src/lexer/experimental-scanner.h

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 10 matching lines @@
 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #ifndef V8_LEXER_EXPERIMENTAL_SCANNER_H
 #define V8_LEXER_EXPERIMENTAL_SCANNER_H
 
+#include <set>
+
 #include "compiler.h"
 #include "isolate.h"
 #include "scanner.h"  // UnicodeCache.
 #include "token.h"
 #include "utils.h"
 #include "v8stdint.h"
 
 namespace v8 {
 namespace internal {
 
@@ skipping 11 matching lines @@
       return beg_pos >= 0 && end_pos >= beg_pos;
     }
 
     static Location invalid() { return Location(-1, -1); }
 
     int beg_pos;
     int end_pos;
   };
 
   explicit ScannerBase(Isolate* isolate)
-    : unicode_cache_(isolate->unicode_cache()),
+    : isolate_(isolate),
+      unicode_cache_(isolate->unicode_cache()),
       has_line_terminator_before_next_(true),
       harmony_numeric_literals_(false),
       harmony_modules_(false),
       harmony_scoping_(false) {
+    if (!scanners_) {
+      scanners_ = new std::set<ScannerBase*>();
+      isolate->heap()->AddGCEpilogueCallback(&ScannerBase::UpdateBuffersAfterGC,
+                                             kGCTypeAll, false);
+    }
+    scanners_->insert(this);
   }
 
-  virtual ~ScannerBase() { }
+  virtual ~ScannerBase() {
+    scanners_->erase(this);
+    if (scanners_->empty()) {
+      isolate_->heap()->RemoveGCEpilogueCallback(
+          &ScannerBase::UpdateBuffersAfterGC);
+      delete scanners_;
+      scanners_ = NULL;
+    }
+  }
 
   // Returns the next token and advances input.
   Token::Value Next() {
     has_line_terminator_before_next_ = false;
     current_ = next_;
     Scan();  // Virtual! Will fill in next_.
     return current_.token;
   }
 
   // Returns the current token again.
@@ skipping 95 matching lines @@
 
  protected:
   struct TokenDesc {
     Token::Value token;
     int beg_pos;
     int end_pos;
   };
 
   virtual void Scan() = 0;
   virtual uc32 ScanHexNumber(int length) = 0;
+  virtual void SetBufferBasedOnHandle() = 0;
+
+  static void UpdateBuffersAfterGC(v8::Isolate*, GCType, GCCallbackFlags);
+
 
   bool ValidIdentifierPart() {
       return unicode_cache_->IsIdentifierPart(ScanHexNumber(4));
   }
 
   bool ValidIdentifierStart() {
     return unicode_cache_->IsIdentifierStart(ScanHexNumber(4));
   }
 
+  Isolate* isolate_;
   UnicodeCache* unicode_cache_;
 
   bool has_line_terminator_before_next_;
 
   TokenDesc current_;  // desc for current token (as returned by Next())
   TokenDesc next_;     // desc for next token (one token look-ahead)
 
   bool harmony_numeric_literals_;
   bool harmony_modules_;
   bool harmony_scoping_;
+
+ private:
+  static std::set<ScannerBase*>* scanners_;
 };
 
 
 template<typename YYCTYPE>
 class ExperimentalScanner : public ScannerBase {
  public:
   explicit ExperimentalScanner(
-      YYCTYPE* source,
-      YYCTYPE* source_end,
-      Isolate* isolate);
+      Handle<String> source,
+      Isolate* isolate)
+      : ScannerBase(isolate),
+        source_handle_(source),
+        buffer_(NULL),
+        buffer_end_(NULL),
+        start_(NULL),
+        cursor_(NULL),
+        marker_(NULL) {
+    ASSERT(source->IsFlat());

[ulan, 2013/11/26 13:41:35]

You can use FlattenString() instead of assert.

+    SetBufferBasedOnHandle();
+    Scan();
+  }
 
-  virtual ~ExperimentalScanner();
+  virtual ~ExperimentalScanner() { }
 
   virtual void Scan();
   virtual uc32 ScanHexNumber(int length);
 
+  virtual void SetBufferBasedOnHandle() {
+    // We get a raw pointer from the Handle, but we also update it every time
+    // there is a GC, so it is safe.
+    DisallowHeapAllocation no_gc;
+    const YYCTYPE* new_buffer = GetNewBufferBasedOnHandle();
+    if (new_buffer != buffer_) {
+      int start_offset = start_ - buffer_;
+      int cursor_offset = cursor_ - buffer_;
+      int marker_offset = marker_ - buffer_;
+      buffer_ = new_buffer;
+      buffer_end_ = buffer_ + source_handle_->length();
+      start_ = buffer_ + start_offset;
+      cursor_ = buffer_ + cursor_offset;
+      marker_ = buffer_ + marker_offset;
+    }
+  }
+
+  const YYCTYPE* GetNewBufferBasedOnHandle() const;
+
  private:
+  Handle<String> source_handle_;
   YYCTYPE yych;

[dcarney, 2013/11/26 13:38:54]

yych should be unused now

-  YYCTYPE* buffer_;
-  YYCTYPE* buffer_end_;
-  YYCTYPE* start_;
-  YYCTYPE* cursor_;
-  YYCTYPE* marker_;
+  const YYCTYPE* buffer_;
+  const YYCTYPE* buffer_end_;
+  const YYCTYPE* start_;
+  const YYCTYPE* cursor_;
+  const YYCTYPE* marker_;
 };
 
 
-template<typename YYCTYPE>
-ExperimentalScanner<YYCTYPE>::ExperimentalScanner(
-    YYCTYPE* source,
-    YYCTYPE* source_end,
-    Isolate* isolate)
-    : ScannerBase(isolate),
-      buffer_(source),
-      buffer_end_(source_end),
-      start_(source),
-      cursor_(source),
-      marker_(source) {
-  Scan();
-}
+template<>

[dcarney, 2013/11/26 13:38:54]

do these need to be here?

+void ExperimentalScanner<uint8_t>::Scan();
+
+template<>
+void ExperimentalScanner<uint16_t>::Scan();
+
+template<>
+void ExperimentalScanner<int8_t>::Scan();
 
 
 template<typename YYCTYPE>
-ExperimentalScanner<YYCTYPE>::~ExperimentalScanner() {
-  delete[] buffer_;
-}
-
-
-template<typename YYCTYPE>
 uc32 ExperimentalScanner<YYCTYPE>::ScanHexNumber(int length) {
   // We have seen \uXXXX, let's see what it is.
   // FIXME: we never end up in here if only a subset of the 4 chars are valid
   // hex digits -> handle the case where they're not.
   uc32 x = 0;
-  for (YYCTYPE* s = cursor_ - length; s != cursor_; ++s) {
+  for (const YYCTYPE* s = cursor_ - length; s != cursor_; ++s) {
     int d = HexValue(*s);
     if (d < 0) {
       return -1;
     }
     x = x * 16 + d;
   }
   return x;
 }
 
 
 } }
 
 #endif  // V8_LEXER_EXPERIMENTAL_SCANNER_H