Streamline the scanner for external two byte string input.

src/scanner.h

 // Copyright 2006-2008 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 55 matching lines @@
     return (data + capacity) - unibrow::Utf8::kMaxEncodedSize;
   }
 
   void AddCharSlow(uc32 c);
 };
 
 
 class UTF16Buffer {
  public:
   UTF16Buffer();
+  virtual ~UTF16Buffer() {}
 
-  void Initialize(Handle<String> data, unibrow::CharacterStream* stream);
-  void PushBack(uc32 ch);
-  uc32 Advance();  // returns a value < 0 when the buffer end is reached
-  uint16_t CharAt(int index);
+  virtual void PushBack(uc32 ch) = 0;

[Kasper Lund, 2009/08/18 06:49:41]

Ideally, these functions could be made non-virtuals again if we change the
internal format of the CharacterStream to be two byte encodings.

[Feng Qian, 2009/08/18 07:14:10]

Let's do it later.

On 2009/08/18 06:49:41, Kasper Lund wrote:

+  // returns a value < 0 when the buffer end is reached
+  virtual uc32 Advance() = 0;
+  virtual void SeekForward(int pos) = 0;
+
   int pos() const { return pos_; }
   int size() const { return size_; }
   Handle<String> SubString(int start, int end);
-  List<uc32>* pushback_buffer() { return &pushback_buffer_; }
-  void SeekForward(int pos);
 
- private:
+ protected:
   Handle<String> data_;
   int pos_;
   int size_;
+};
+
+
+class CharacterStreamUTF16Buffer: public UTF16Buffer {
+ public:
+  CharacterStreamUTF16Buffer();
+  virtual ~CharacterStreamUTF16Buffer() {}
+  void Initialize(Handle<String> data, unibrow::CharacterStream* stream);
+  virtual void PushBack(uc32 ch);
+  virtual uc32 Advance();
+  virtual void SeekForward(int pos);
+
+ private:
   List<uc32> pushback_buffer_;
   uc32 last_;
   unibrow::CharacterStream* stream_;
+
+  List<uc32>* pushback_buffer() { return &pushback_buffer_; }
+};
+
+
+class TwoByteStringUTF16Buffer: public UTF16Buffer {
+ public:
+  TwoByteStringUTF16Buffer();
+  virtual ~TwoByteStringUTF16Buffer() {}
+  void Initialize(Handle<ExternalTwoByteString> data);
+  virtual void PushBack(uc32 ch);
+  virtual uc32 Advance();
+  virtual void SeekForward(int pos);
+
+ private:
+  const uint16_t* raw_data_;
 };
 
 
 class Scanner {
  public:
 
   typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder;
 
   // Construction
   explicit Scanner(bool is_pre_parsing);
@@ skipping 73 matching lines @@
   // Tells whether the buffer contains an identifier (no escapes).
   // Used for checking if a property name is an identifier.
   static bool IsIdentifier(unibrow::CharacterStream* buffer);
 
   static unibrow::Predicate<IdentifierStart, 128> kIsIdentifierStart;
   static unibrow::Predicate<IdentifierPart, 128> kIsIdentifierPart;
   static unibrow::Predicate<unibrow::LineTerminator, 128> kIsLineTerminator;
   static unibrow::Predicate<unibrow::WhiteSpace, 128> kIsWhiteSpace;
 
  private:
+  CharacterStreamUTF16Buffer char_stream_buffer_;
+  TwoByteStringUTF16Buffer two_byte_string_buffer_;
+
   // Source.
-  UTF16Buffer source_;
+  UTF16Buffer* source_;
   int position_;
 
   // Buffer to hold literal values (identifiers, strings, numbers)
   // using 0-terminated UTF-8 encoding.
   UTF8Buffer literals_;
 
   bool stack_overflow_;
   static StaticResource<Utf8Decoder> utf8_decoder_;
 
   // One Unicode character look-ahead; c0_ < 0 at the end of the input.
@@ skipping 13 matching lines @@
 
   static const int kCharacterLookaheadBufferSize = 1;
 
   // Literal buffer support
   void StartLiteral();
   void AddChar(uc32 ch);
   void AddCharAdvance();
   void TerminateLiteral();
 
   // Low-level scanning support.
-  void Advance();
-  void PushBack(uc32 ch);
+  void Advance() { c0_ = source_->Advance(); }
+  void PushBack(uc32 ch) {
+    source_->PushBack(ch);
+    c0_ = ch;
+  }
 
   bool SkipWhiteSpace();
   Token::Value SkipSingleLineComment();
   Token::Value SkipMultiLineComment();
 
   inline Token::Value Select(Token::Value tok);
   inline Token::Value Select(uc32 next, Token::Value then, Token::Value else_);
 
   void Scan();
   void ScanDecimalDigits();
   Token::Value ScanNumber(bool seen_period);
   Token::Value ScanIdentifier();
   uc32 ScanHexEscape(uc32 c, int length);
   uc32 ScanOctalEscape(uc32 c, int length);
   void ScanEscape();
   Token::Value ScanString();
 
   // Scans a possible HTML comment -- begins with '<!'.
   Token::Value ScanHtmlComment();
 
   // Return the current source position.
   int source_pos() {
-    return source_.pos() - kCharacterLookaheadBufferSize + position_;
+    return source_->pos() - kCharacterLookaheadBufferSize + position_;
   }
 
   // Decodes a unicode escape-sequence which is part of an identifier.
   // If the escape sequence cannot be decoded the result is kBadRune.
   uc32 ScanIdentifierUnicodeEscape();
 };
 
 } }  // namespace v8::internal
 
 #endif  // V8_SCANNER_H_