Streamline the scanner for external two byte string input.

src/scanner.cc

Index: src/scanner.cc
===================================================================
--- src/scanner.cc	(revision 2670)
+++ src/scanner.cc	(working copy)
@@ -92,33 +92,35 @@
 
 
 UTF16Buffer::UTF16Buffer()
-  : pos_(0),
-    pushback_buffer_(0),
-    last_(0),
-    stream_(NULL) { }
+  : pos_(0), size_(0) { }

[Kasper Lund, 2009/08/18 06:49:41]

4 space indent.

[Feng Qian, 2009/08/18 07:14:10]

Done.

[Feng Qian, 2009/08/18 07:14:10]

Done.

 
 
-void UTF16Buffer::Initialize(Handle<String> data,
-                             unibrow::CharacterStream* input) {
+Handle<String> UTF16Buffer::SubString(int start, int end) {
+  return internal::SubString(data_, start, end);
+}
+
+
+// CharacterStreamUTF16Buffer
+CharacterStreamUTF16Buffer::CharacterStreamUTF16Buffer()
+  : pushback_buffer_(0), last_(0), stream_(NULL) { }

[Kasper Lund, 2009/08/18 06:49:41]

4 space indent.

[Feng Qian, 2009/08/18 07:14:10]

Done.

[Feng Qian, 2009/08/18 07:14:10]

Done.

+
+
+void CharacterStreamUTF16Buffer::Initialize(Handle<String> data,
+                                            unibrow::CharacterStream* input) {
   data_ = data;
   pos_ = 0;
   stream_ = input;
 }
 
 
-Handle<String> UTF16Buffer::SubString(int start, int end) {
-  return internal::SubString(data_, start, end);
-}
-
-
-void UTF16Buffer::PushBack(uc32 ch) {
+void CharacterStreamUTF16Buffer::PushBack(uc32 ch) {
   pushback_buffer()->Add(last_);
   last_ = ch;
   pos_--;
 }
 
 
-uc32 UTF16Buffer::Advance() {
+uc32 CharacterStreamUTF16Buffer::Advance() {
   // NOTE: It is of importance to Persian / Farsi resources that we do
   // *not* strip format control characters in the scanner; see
   //
@@ -143,13 +145,53 @@
 }
 
 
-void UTF16Buffer::SeekForward(int pos) {
+void CharacterStreamUTF16Buffer::SeekForward(int pos) {
   pos_ = pos;
   ASSERT(pushback_buffer()->is_empty());
   stream_->Seek(pos);
 }
 
 
+// TwoByteStringUTF16Buffer
+TwoByteStringUTF16Buffer::TwoByteStringUTF16Buffer()
+  : raw_data_(NULL) { }

[Kasper Lund, 2009/08/18 06:49:41]

4 space indent.

[Feng Qian, 2009/08/18 07:14:10]

Done.

+
+
+void TwoByteStringUTF16Buffer::Initialize(
+     Handle<ExternalTwoByteString> data) {
+  ASSERT(!data.is_null() && StringShape(*data).IsExternalTwoByte());

[Kasper Lund, 2009/08/18 06:49:41]

StringShape(*data).IsExternalTwoByte() -> data->IsExternalTwoByteString()

[Feng Qian, 2009/08/18 07:14:10]

IsExternalTwoByte check is unnecessary here, removed.

+
+  data_ = data;
+  pos_ = 0;
+
+  raw_data_ = data->resource()->data();
+  size_ = data->length();
+}
+
+
+uc32 TwoByteStringUTF16Buffer::Advance() {
+  if (pos_ < size_) {
+    return raw_data_[pos_++];
+  } else {
+    // note: currently the following increment is necessary to avoid a

[Kasper Lund, 2009/08/18 06:49:41]

note -> Note

[Feng Qian, 2009/08/18 07:14:10]

Done.

+    // test-parser problem!
+    pos_++;
+    return static_cast<uc32>(-1);
+  }
+}
+
+
+void TwoByteStringUTF16Buffer::PushBack(uc32 ch) {
+  pos_--;
+  ASSERT(pos_ >= 0 && raw_data_[pos_] == ch);
+}
+
+
+void TwoByteStringUTF16Buffer::SeekForward(int pos) {
+  pos_ = pos;
+}
+
+
 // ----------------------------------------------------------------------------
 // Scanner
 
@@ -161,7 +203,15 @@
 void Scanner::Init(Handle<String> source, unibrow::CharacterStream* stream,
     int position) {
   // Initialize the source buffer.
-  source_.Initialize(source, stream);
+  if (!source.is_null() && StringShape(*source).IsExternalTwoByte()) {

[Kasper Lund, 2009/08/18 06:49:41]

StringShape(*source).IsExternalTwoByte() -> source->IsExternalTwoByteString()?

[Feng Qian, 2009/08/18 07:14:10]

IsExternalTwoByteString is only implemented in #if DEBUG, and comments in
objects-inl.h states that one should consider use StringShape instead of moving
the implementation out of #if DEBUG

+    two_byte_string_buffer_.Initialize(
+        Handle<ExternalTwoByteString>::cast(source));
+    source_ = &two_byte_string_buffer_;
+  } else {
+    char_stream_buffer_.Initialize(source, stream);
+    source_ = &char_stream_buffer_;
+  }
+
   position_ = position;
 
   // Reset literals buffer
@@ -180,7 +230,7 @@
 
 
 Handle<String> Scanner::SubString(int start, int end) {
-  return source_.SubString(start - position_, end - position_);
+  return source_->SubString(start - position_, end - position_);
 }
 
 
@@ -223,17 +273,6 @@
 }
 
 
-void Scanner::Advance() {
-  c0_ = source_.Advance();
-}
-
-
-void Scanner::PushBack(uc32 ch) {
-  source_.PushBack(ch);
-  c0_ = ch;
-}
-
-
 static inline bool IsByteOrderMark(uc32 c) {
   // The Unicode value U+FFFE is guaranteed never to be assigned as a
   // Unicode character; this implies that in a Unicode context the
@@ -583,7 +622,7 @@
 
 
 void Scanner::SeekForward(int pos) {
-  source_.SeekForward(pos - 1);
+  source_->SeekForward(pos - 1);
   Advance();
   Scan();
 }