Optimized scanner to avoid virtual calls for every character read.

src/preparser-api.cc

Index: src/preparser-api.cc
diff --git a/src/preparser-api.cc b/src/preparser-api.cc
index f096e941b704ab29f3516176ea1aa000ed766bd7..b5a232b61129d04506bb82a5c77500ee465124db 100644
--- a/src/preparser-api.cc
+++ b/src/preparser-api.cc
@@ -39,39 +39,73 @@ namespace v8 {
 namespace internal {
 
 // UTF16Buffer based on a v8::UnicodeInputStream.
-class InputStreamUTF16Buffer : public UTF16Buffer {
+class InputStreamUTF16Buffer : public UC16CharacterStream {
  public:
   explicit InputStreamUTF16Buffer(UnicodeInputStream* stream)
-      : UTF16Buffer(),
-        stream_(stream) { }
+      : UC16CharacterStream(),
+        stream_(stream),
+        pushback_active_(false) {
+    buffer_cursor_ = buffer_end_ = buffer_;
+    ReadBlock();
+  }
 
   virtual ~InputStreamUTF16Buffer() { }
 
-  virtual void PushBack(uc32 ch) {
+  virtual void PushBack(uc16 ch) {
+    if (buffer_cursor_ > buffer_) {
+      // While we can stay within the buffer, just do so.
+      buffer_cursor_--;
+      ASSERT_EQ(*buffer_cursor_, ch);
+      return;
+    }

[Erik Corry, 2010/12/07 12:27:30]

Can the contents of this first if be put in a non-virtual method that calls the
virtual method for the last part?

[Lasse Reichstein, 2010/12/07 14:05:54]

Pushback is a rare operation. I doubt there is anything gained from splitting it
up.
It would also require changing the UC16CharacterStream base class, which doesn't
have a non-virtual PushBack method, and there isn't any generic way to handle it
with the current design.

I have redesigned the code to have a pushback scratch area before the buffer, so
you can push back characters before the currently read block. If that fails, it
falls back on pushing everything back and rereading a block from that position.

The whole interface should probably be reconsidered. Buffering doesn't play well
with having pushback int the stream.

+    if (!pushback_active_) {
+      // Push back the entire buffer to the stream and let the
+      // stream handle pushbacks from now.
+      // We leave buffer_cursor_ == buffer_end_, so the next read
+      // will fill the buffer from the current position.
+      while (buffer_end_ > buffer_) {
+        stream_->PushBack(*--buffer_end_);
+      }

[Erik Corry, 2010/12/07 12:27:30]

Do we need a PushBackBlock method?  This looks slow.  Alternatively the
ReadBlock method could read overlapping blocks into the buffer so that this
doesn't happen much?

[Lasse Reichstein, 2010/12/07 14:05:54]

The interface should be reconsidered. It might not be perfect for what we are
going to use it for. Perhaps it should drop the pushback, leaving it to the
reader to handle pushbacks. As it is now, a real streaming source would have to
implement its own pushback buffer too.

+      pushback_active_ = true;
+    }
     stream_->PushBack(ch);
     pos_--;
   }
 
-  virtual uc32 Advance() {
-    uc32 result = stream_->Next();
-    if (result >= 0) pos_++;
-    return result;
+ protected:
+  virtual bool ReadBlock() {
+    pushback_active_ = false;
+    int32_t value;
+    buffer_cursor_ = buffer_end_ = buffer_;
+    while ((value = stream_->Next()) >= 0) {
+      if (value > static_cast<int32_t>(unibrow::Utf8::kMaxThreeByteChar)) {
+        value = unibrow::Utf8::kBadChar;
+      }
+      // buffer_end_ is a const pointer, but buffer_ is writable.
+      buffer_[buffer_end_++ - buffer_] = static_cast<uc16>(value);
+      if (buffer_end_ == buffer_ + kBufferSize) break;
+    }
+    return buffer_end_ > buffer_;
   }
 
-  virtual void SeekForward(int pos) {
+  virtual unsigned SlowSeekForward(unsigned pos) {
     // Seeking in the input is not used by preparsing.
     // It's only used by the real parser based on preparser data.
     UNIMPLEMENTED();
+    return 0;
   }
 
  private:
+  static const unsigned kBufferSize = 512;
   v8::UnicodeInputStream* const stream_;
+  uc16 buffer_[kBufferSize];
+  bool pushback_active_;
 };
 
 
 class StandAloneJavaScriptScanner : public JavaScriptScanner {
  public:
-  void Initialize(UTF16Buffer* source) {
+  void Initialize(UC16CharacterStream* source) {
     source_ = source;
     literal_flags_ = kLiteralString | kLiteralIdentifier;
     Init();