Optimized scanner to avoid virtual calls for every character read.

src/scanner-base.h

Index: src/scanner-base.h
diff --git a/src/scanner-base.h b/src/scanner-base.h
index 3d344f306ca0953e479ee90fe5446e0b999243f3..0f811fe319be0a4c40d0efecfde8c3b1c56db349 100644
--- a/src/scanner-base.h
+++ b/src/scanner-base.h
@@ -52,31 +52,73 @@ inline int HexValue(uc32 c) {
   return -1;
 }
 
-// ----------------------------------------------------------------------------
-// UTF16Buffer - scanner input source with pushback.
-
-class UTF16Buffer {
- public:
-  UTF16Buffer();
-  virtual ~UTF16Buffer() {}
 
-  virtual void PushBack(uc32 ch) = 0;
-  // Returns a value < 0 when the buffer end is reached.
-  virtual uc32 Advance() = 0;
-  virtual void SeekForward(int pos) = 0;
+// ---------------------------------------------------------------------
+// Buffered stream of characters, using an internal UC16 buffer.
 
-  int pos() const { return pos_; }
+class UC16CharacterStream {
+ public:
+  UC16CharacterStream() : pos_(0) { }
+  virtual ~UC16CharacterStream() { }
+
+  // Returns and advances past the next UC16 character in the input
+  // stream. If there are no more characters, it returns a negative
+  // value.
+  inline int32_t Advance() {
+    if (buffer_cursor_ < buffer_end_ || ReadBlock()) {
+      pos_++;
+      return *(buffer_cursor_++);
+    }
+    // Note: currently the following increment is necessary to avoid a
+    // parser problem! The scanner treats the final kEndOfInput as
+    // a character with a position, and does math relative to that
+    // position.
+    pos_++;
 
-  static const int kNoEndPosition = 1;
+    return kEndOfInput;
+  }
 
+  // Return the current position in the character stream.
+  // Starts at zero.
+  inline unsigned pos() const { return pos_; }
+
+  // Skips forward past the next character_count UC16 characters
+  // in the input, or until the end of input if that comes sooner.
+  // Returns the number of characters actually skipped. If less
+  // than character_count,
+  inline unsigned SeekForward(unsigned character_count) {
+    unsigned buffered_chars =
+        static_cast<unsigned>(buffer_end_ - buffer_cursor_);
+    if (character_count <= buffered_chars) {
+      buffer_cursor_ += character_count;
+      pos_ += character_count;
+      return character_count;
+    }
+    return SlowSeekForward(character_count);
+  }
+  // Pushes back the most recently read UC16 character, i.e.,
+  // the value returned by the most recent call to Advance.
+  // Must not be used right after calling SeekForward.
+  virtual void PushBack(uc16 character) = 0;

[Erik Corry, 2010/12/07 12:27:30]

Missing blank line

  protected:
-  // Initial value of end_ before the input stream is initialized.
-
-  int pos_;  // Current position in the buffer.
-  int end_;  // Position where scanning should stop (EOF).
+  static const int32_t kEndOfInput = -1;
+
+  // Ensures that the buffer_cursor_ points to the character at
+  // position pos_ of the input, if possible. If the position
+  // is at or after the end of the input, return false. If there
+  // are more characters available, return true.
+  virtual bool ReadBlock() = 0;
+  virtual unsigned SlowSeekForward(unsigned character_count) = 0;
+
+  const uc16* buffer_cursor_;
+  const uc16* buffer_end_;
+  unsigned pos_;
 };
 
 
+// ---------------------------------------------------------------------
+// Constants used by scanners.
+
 class ScannerConstants : AllStatic {
  public:
   typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder;
@@ -171,7 +213,7 @@ class Scanner {
     bool complete_;
   };
 
-  Scanner();
+  explicit Scanner();

[Erik Corry, 2010/12/07 12:27:30]

You only need this for single-argument constructors.

 
   // Returns the current token again.
   Token::Value current_token() { return current_.token; }
@@ -277,7 +319,7 @@ class Scanner {
   // Low-level scanning support.
   void Advance() { c0_ = source_->Advance(); }
   void PushBack(uc32 ch) {
-    source_->PushBack(ch);
+    source_->PushBack(c0_);
     c0_ = ch;
   }
 
@@ -307,8 +349,8 @@ class Scanner {
   TokenDesc current_;  // desc for current token (as returned by Next())
   TokenDesc next_;     // desc for next token (one token look-ahead)
 
-  // Input stream. Must be initialized to an UTF16Buffer.
-  UTF16Buffer* source_;
+  // Input stream. Must be initialized to an UC16CharacterStream.
+  UC16CharacterStream* source_;
 
   // Buffer to hold literal values (identifiers, strings, numbers)
   // using '\x00'-terminated UTF-8 encoding. Handles allocation internally.