Streamline the scanner for external two byte string input.

src/scanner.cc

 // Copyright 2006-2008 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 74 matching lines @@
   }
   ASSERT(pos() <= Capacity());
 }
 
 
 // ----------------------------------------------------------------------------
 // UTF16Buffer
 
 
 UTF16Buffer::UTF16Buffer()
-  : pos_(0),
-    pushback_buffer_(0),
-    last_(0),
-    stream_(NULL) { }
-
-
-void UTF16Buffer::Initialize(Handle<String> data,
-                             unibrow::CharacterStream* input) {
-  data_ = data;
-  pos_ = 0;
-  stream_ = input;
-}
+  : pos_(0), size_(0) { }

[Kasper Lund, 2009/08/18 06:49:41]

4 space indent.

[Feng Qian, 2009/08/18 07:14:10]

Done.

[Feng Qian, 2009/08/18 07:14:10]

Done.

 
 
 Handle<String> UTF16Buffer::SubString(int start, int end) {
   return internal::SubString(data_, start, end);
 }
 
 
-void UTF16Buffer::PushBack(uc32 ch) {
+// CharacterStreamUTF16Buffer
+CharacterStreamUTF16Buffer::CharacterStreamUTF16Buffer()
+  : pushback_buffer_(0), last_(0), stream_(NULL) { }

[Kasper Lund, 2009/08/18 06:49:41]

4 space indent.

[Feng Qian, 2009/08/18 07:14:10]

Done.

[Feng Qian, 2009/08/18 07:14:10]

Done.

+
+
+void CharacterStreamUTF16Buffer::Initialize(Handle<String> data,
+                                            unibrow::CharacterStream* input) {
+  data_ = data;
+  pos_ = 0;
+  stream_ = input;
+}
+
+
+void CharacterStreamUTF16Buffer::PushBack(uc32 ch) {
   pushback_buffer()->Add(last_);
   last_ = ch;
   pos_--;
 }
 
 
-uc32 UTF16Buffer::Advance() {
+uc32 CharacterStreamUTF16Buffer::Advance() {
   // NOTE: It is of importance to Persian / Farsi resources that we do
   // *not* strip format control characters in the scanner; see
   //
   //    https://bugzilla.mozilla.org/show_bug.cgi?id=274152
   //
   // So, even though ECMA-262, section 7.1, page 11, dictates that we
   // must remove Unicode format-control characters, we do not. This is
   // in line with how IE and SpiderMonkey handles it.
   if (!pushback_buffer()->is_empty()) {
     pos_++;
     return last_ = pushback_buffer()->RemoveLast();
   } else if (stream_->has_more()) {
     pos_++;
     uc32 next = stream_->GetNext();
     return last_ = next;
   } else {
     // note: currently the following increment is necessary to avoid a
     // test-parser problem!
     pos_++;
     return last_ = static_cast<uc32>(-1);
   }
 }
 
 
-void UTF16Buffer::SeekForward(int pos) {
+void CharacterStreamUTF16Buffer::SeekForward(int pos) {
   pos_ = pos;
   ASSERT(pushback_buffer()->is_empty());
   stream_->Seek(pos);
 }
 
 
+// TwoByteStringUTF16Buffer
+TwoByteStringUTF16Buffer::TwoByteStringUTF16Buffer()
+  : raw_data_(NULL) { }

[Kasper Lund, 2009/08/18 06:49:41]

4 space indent.

[Feng Qian, 2009/08/18 07:14:10]

Done.

+
+
+void TwoByteStringUTF16Buffer::Initialize(
+     Handle<ExternalTwoByteString> data) {
+  ASSERT(!data.is_null() && StringShape(*data).IsExternalTwoByte());

[Kasper Lund, 2009/08/18 06:49:41]

StringShape(*data).IsExternalTwoByte() -> data->IsExternalTwoByteString()

[Feng Qian, 2009/08/18 07:14:10]

IsExternalTwoByte check is unnecessary here, removed.

+
+  data_ = data;
+  pos_ = 0;
+
+  raw_data_ = data->resource()->data();
+  size_ = data->length();
+}
+
+
+uc32 TwoByteStringUTF16Buffer::Advance() {
+  if (pos_ < size_) {
+    return raw_data_[pos_++];
+  } else {
+    // note: currently the following increment is necessary to avoid a

[Kasper Lund, 2009/08/18 06:49:41]

note -> Note

[Feng Qian, 2009/08/18 07:14:10]

Done.

+    // test-parser problem!
+    pos_++;
+    return static_cast<uc32>(-1);
+  }
+}
+
+
+void TwoByteStringUTF16Buffer::PushBack(uc32 ch) {
+  pos_--;
+  ASSERT(pos_ >= 0 && raw_data_[pos_] == ch);
+}
+
+
+void TwoByteStringUTF16Buffer::SeekForward(int pos) {
+  pos_ = pos;
+}
+
+
 // ----------------------------------------------------------------------------
 // Scanner
 
 Scanner::Scanner(bool pre) : stack_overflow_(false), is_pre_parsing_(pre) {
   Token::Initialize();
 }
 
 
 void Scanner::Init(Handle<String> source, unibrow::CharacterStream* stream,
     int position) {
   // Initialize the source buffer.
-  source_.Initialize(source, stream);
+  if (!source.is_null() && StringShape(*source).IsExternalTwoByte()) {

[Kasper Lund, 2009/08/18 06:49:41]

StringShape(*source).IsExternalTwoByte() -> source->IsExternalTwoByteString()?

[Feng Qian, 2009/08/18 07:14:10]

IsExternalTwoByteString is only implemented in #if DEBUG, and comments in
objects-inl.h states that one should consider use StringShape instead of moving
the implementation out of #if DEBUG

+    two_byte_string_buffer_.Initialize(
+        Handle<ExternalTwoByteString>::cast(source));
+    source_ = &two_byte_string_buffer_;
+  } else {
+    char_stream_buffer_.Initialize(source, stream);
+    source_ = &char_stream_buffer_;
+  }
+
   position_ = position;
 
   // Reset literals buffer
   literals_.Reset();
 
   // Set c0_ (one character ahead)
   ASSERT(kCharacterLookaheadBufferSize == 1);
   Advance();
 
   // Skip initial whitespace allowing HTML comment ends just like
   // after a newline and scan first token.
   has_line_terminator_before_next_ = true;
   SkipWhiteSpace();
   Scan();
 }
 
 
 Handle<String> Scanner::SubString(int start, int end) {
-  return source_.SubString(start - position_, end - position_);
+  return source_->SubString(start - position_, end - position_);
 }
 
 
 Token::Value Scanner::Next() {
   // BUG 1215673: Find a thread safe way to set a stack limit in
   // pre-parse mode. Otherwise, we cannot safely pre-parse from other
   // threads.
   current_ = next_;
   // Check for stack-overflow before returning any tokens.
   StackLimitCheck check;
@@ skipping 22 matching lines @@
   AddChar(0);
 }
 
 
 void Scanner::AddCharAdvance() {
   AddChar(c0_);
   Advance();
 }
 
 
-void Scanner::Advance() {
-  c0_ = source_.Advance();
-}
-
-
-void Scanner::PushBack(uc32 ch) {
-  source_.PushBack(ch);
-  c0_ = ch;
-}
-
-
 static inline bool IsByteOrderMark(uc32 c) {
   // The Unicode value U+FFFE is guaranteed never to be assigned as a
   // Unicode character; this implies that in a Unicode context the
   // 0xFF, 0xFE byte pattern can only be interpreted as the U+FEFF
   // character expressed in little-endian byte order (since it could
   // not be a U+FFFE character expressed in big-endian byte
   // order). Nevertheless, we check for it to be compatible with
   // Spidermonkey.
   return c == 0xFEFF || c == 0xFFFE;
 }
@@ skipping 329 matching lines @@
     // Continue scanning for tokens as long as we're just skipping
     // whitespace.
   } while (token == Token::WHITESPACE);
 
   next_.location.end_pos = source_pos();
   next_.token = token;
 }
 
 
 void Scanner::SeekForward(int pos) {
-  source_.SeekForward(pos - 1);
+  source_->SeekForward(pos - 1);
   Advance();
   Scan();
 }
 
 
 uc32 Scanner::ScanHexEscape(uc32 c, int length) {
   ASSERT(length <= 4);  // prevent overflow
 
   uc32 digits[4];
   uc32 x = 0;
@@ skipping 329 matching lines @@
     }
     AddCharAdvance();
   }
   TerminateLiteral();
 
   next_.location.end_pos = source_pos() - 1;
   return true;
 }
 
 } }  // namespace v8::internal