Optimized scanner to avoid virtual calls for every character read.

src/preparser-api.cc

 // Copyright 2010 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 21 matching lines @@
 #include "utils.h"
 #include "list.h"
 #include "scanner-base.h"
 #include "preparse-data.h"
 #include "preparser.h"
 
 namespace v8 {
 namespace internal {
 
 // UTF16Buffer based on a v8::UnicodeInputStream.
-class InputStreamUTF16Buffer : public UTF16Buffer {
+class InputStreamUTF16Buffer : public UC16CharacterStream {
  public:
   explicit InputStreamUTF16Buffer(UnicodeInputStream* stream)
-      : UTF16Buffer(),
-        stream_(stream) { }
+      : UC16CharacterStream(),
+        stream_(stream),
+        pushback_active_(false) {
+    buffer_cursor_ = buffer_end_ = buffer_;
+    ReadBlock();
+  }
 
   virtual ~InputStreamUTF16Buffer() { }
 
-  virtual void PushBack(uc32 ch) {
+  virtual void PushBack(uc16 ch) {
+    if (buffer_cursor_ > buffer_) {
+      // While we can stay within the buffer, just do so.
+      buffer_cursor_--;
+      ASSERT_EQ(*buffer_cursor_, ch);
+      return;
+    }

[Erik Corry, 2010/12/07 12:27:30]

Can the contents of this first if be put in a non-virtual method that calls the
virtual method for the last part?

[Lasse Reichstein, 2010/12/07 14:05:54]

Pushback is a rare operation. I doubt there is anything gained from splitting it
up.
It would also require changing the UC16CharacterStream base class, which doesn't
have a non-virtual PushBack method, and there isn't any generic way to handle it
with the current design.

I have redesigned the code to have a pushback scratch area before the buffer, so
you can push back characters before the currently read block. If that fails, it
falls back on pushing everything back and rereading a block from that position.

The whole interface should probably be reconsidered. Buffering doesn't play well
with having pushback int the stream.

+    if (!pushback_active_) {
+      // Push back the entire buffer to the stream and let the
+      // stream handle pushbacks from now.
+      // We leave buffer_cursor_ == buffer_end_, so the next read
+      // will fill the buffer from the current position.
+      while (buffer_end_ > buffer_) {
+        stream_->PushBack(*--buffer_end_);
+      }

[Erik Corry, 2010/12/07 12:27:30]

Do we need a PushBackBlock method?  This looks slow.  Alternatively the
ReadBlock method could read overlapping blocks into the buffer so that this
doesn't happen much?

[Lasse Reichstein, 2010/12/07 14:05:54]

The interface should be reconsidered. It might not be perfect for what we are
going to use it for. Perhaps it should drop the pushback, leaving it to the
reader to handle pushbacks. As it is now, a real streaming source would have to
implement its own pushback buffer too.

+      pushback_active_ = true;
+    }
     stream_->PushBack(ch);
     pos_--;
   }
 
-  virtual uc32 Advance() {
-    uc32 result = stream_->Next();
-    if (result >= 0) pos_++;
-    return result;
+ protected:
+  virtual bool ReadBlock() {
+    pushback_active_ = false;
+    int32_t value;
+    buffer_cursor_ = buffer_end_ = buffer_;
+    while ((value = stream_->Next()) >= 0) {
+      if (value > static_cast<int32_t>(unibrow::Utf8::kMaxThreeByteChar)) {
+        value = unibrow::Utf8::kBadChar;
+      }
+      // buffer_end_ is a const pointer, but buffer_ is writable.
+      buffer_[buffer_end_++ - buffer_] = static_cast<uc16>(value);
+      if (buffer_end_ == buffer_ + kBufferSize) break;
+    }
+    return buffer_end_ > buffer_;
   }
 
-  virtual void SeekForward(int pos) {
+  virtual unsigned SlowSeekForward(unsigned pos) {
     // Seeking in the input is not used by preparsing.
     // It's only used by the real parser based on preparser data.
     UNIMPLEMENTED();
+    return 0;
   }
 
  private:
+  static const unsigned kBufferSize = 512;
   v8::UnicodeInputStream* const stream_;
+  uc16 buffer_[kBufferSize];
+  bool pushback_active_;
 };
 
 
 class StandAloneJavaScriptScanner : public JavaScriptScanner {
  public:
-  void Initialize(UTF16Buffer* source) {
+  void Initialize(UC16CharacterStream* source) {
     source_ = source;
     literal_flags_ = kLiteralString | kLiteralIdentifier;
     Init();
     // Skip initial whitespace allowing HTML comment ends just like
     // after a newline and scan first token.
     has_line_terminator_before_next_ = true;
     SkipWhiteSpace();
     Scan();
   }
 };
@@ skipping 34 matching lines @@
   return PreParserData(size, data);
 }
 
 }  // namespace v8.
 
 
 // Used by ASSERT macros and other immediate exits.
 extern "C" void V8_Fatal(const char* file, int line, const char* format, ...) {
   exit(EXIT_FAILURE);
 }