Streamline the scanner for external two byte string input.

src/scanner.h

Index: src/scanner.h
===================================================================
--- src/scanner.h	(revision 2670)
+++ src/scanner.h	(working copy)
@@ -73,27 +73,56 @@
 class UTF16Buffer {
  public:
   UTF16Buffer();
+  virtual ~UTF16Buffer() {}
 
-  void Initialize(Handle<String> data, unibrow::CharacterStream* stream);
-  void PushBack(uc32 ch);
-  uc32 Advance();  // returns a value < 0 when the buffer end is reached
-  uint16_t CharAt(int index);
+  virtual void PushBack(uc32 ch) = 0;

[Kasper Lund, 2009/08/18 06:49:41]

Ideally, these functions could be made non-virtuals again if we change the
internal format of the CharacterStream to be two byte encodings.

[Feng Qian, 2009/08/18 07:14:10]

Let's do it later.

On 2009/08/18 06:49:41, Kasper Lund wrote:

+  // returns a value < 0 when the buffer end is reached
+  virtual uc32 Advance() = 0;
+  virtual void SeekForward(int pos) = 0;
+
   int pos() const { return pos_; }
   int size() const { return size_; }
   Handle<String> SubString(int start, int end);
-  List<uc32>* pushback_buffer() { return &pushback_buffer_; }
-  void SeekForward(int pos);
 
- private:
+ protected:
   Handle<String> data_;
   int pos_;
   int size_;
+};
+
+
+class CharacterStreamUTF16Buffer: public UTF16Buffer {
+ public:
+  CharacterStreamUTF16Buffer();
+  virtual ~CharacterStreamUTF16Buffer() {}
+  void Initialize(Handle<String> data, unibrow::CharacterStream* stream);
+  virtual void PushBack(uc32 ch);
+  virtual uc32 Advance();
+  virtual void SeekForward(int pos);
+
+ private:
   List<uc32> pushback_buffer_;
   uc32 last_;
   unibrow::CharacterStream* stream_;
+
+  List<uc32>* pushback_buffer() { return &pushback_buffer_; }
 };
 
 
+class TwoByteStringUTF16Buffer: public UTF16Buffer {
+ public:
+  TwoByteStringUTF16Buffer();
+  virtual ~TwoByteStringUTF16Buffer() {}
+  void Initialize(Handle<ExternalTwoByteString> data);
+  virtual void PushBack(uc32 ch);
+  virtual uc32 Advance();
+  virtual void SeekForward(int pos);
+
+ private:
+  const uint16_t* raw_data_;
+};
+
+
 class Scanner {
  public:
 
@@ -184,8 +213,11 @@
   static unibrow::Predicate<unibrow::WhiteSpace, 128> kIsWhiteSpace;
 
  private:
+  CharacterStreamUTF16Buffer char_stream_buffer_;
+  TwoByteStringUTF16Buffer two_byte_string_buffer_;
+
   // Source.
-  UTF16Buffer source_;
+  UTF16Buffer* source_;
   int position_;
 
   // Buffer to hold literal values (identifiers, strings, numbers)
@@ -219,8 +251,11 @@
   void TerminateLiteral();
 
   // Low-level scanning support.
-  void Advance();
-  void PushBack(uc32 ch);
+  void Advance() { c0_ = source_->Advance(); }
+  void PushBack(uc32 ch) {
+    source_->PushBack(ch);
+    c0_ = ch;
+  }
 
   bool SkipWhiteSpace();
   Token::Value SkipSingleLineComment();
@@ -243,7 +278,7 @@
 
   // Return the current source position.
   int source_pos() {
-    return source_.pos() - kCharacterLookaheadBufferSize + position_;
+    return source_->pos() - kCharacterLookaheadBufferSize + position_;
   }
 
   // Decodes a unicode escape-sequence which is part of an identifier.