Refactor the scanner interface...

src/scanner.cc

 // Copyright 2006-2008 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 10 matching lines @@
 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "v8.h"
 
 #include "ast.h"
+#include "handles.h"
 #include "scanner.h"
 
 namespace v8 {
 namespace internal {
 
 // ----------------------------------------------------------------------------
 // Character predicates
 
 
 unibrow::Predicate<IdentifierStart, 128> Scanner::kIsIdentifierStart;
@@ skipping 38 matching lines @@
   }
   ASSERT(pos() <= Capacity());
 }
 
 
 // ----------------------------------------------------------------------------
 // UTF16Buffer
 
 
 UTF16Buffer::UTF16Buffer()
-    : pos_(0), size_(0) { }
-
-
-Handle<String> UTF16Buffer::SubString(int start, int end) {
-  return internal::SubString(data_, start, end);
-}
+    : pos_(0), end_(Scanner::kNoEndPosition) { }
 
 
 // CharacterStreamUTF16Buffer
 CharacterStreamUTF16Buffer::CharacterStreamUTF16Buffer()
     : pushback_buffer_(0), last_(0), stream_(NULL) { }
 
 
 void CharacterStreamUTF16Buffer::Initialize(Handle<String> data,
-                                            unibrow::CharacterStream* input) {
-  data_ = data;
-  pos_ = 0;
+                                            unibrow::CharacterStream* input,
+                                            int start_position,
+                                            int end_position) {
   stream_ = input;
+  if (start_position > 0) {
+    SeekForward(start_position);
+  }
+  end_ = end_position != Scanner::kNoEndPosition ? end_position : kMaxInt;
 }
 
 
 void CharacterStreamUTF16Buffer::PushBack(uc32 ch) {
   pushback_buffer()->Add(last_);
   last_ = ch;
   pos_--;
 }
 
 
 uc32 CharacterStreamUTF16Buffer::Advance() {
+  ASSERT(end_ != Scanner::kNoEndPosition);
+  ASSERT(end_ >= 0);
   // NOTE: It is of importance to Persian / Farsi resources that we do
   // *not* strip format control characters in the scanner; see
   //
   //    https://bugzilla.mozilla.org/show_bug.cgi?id=274152
   //
   // So, even though ECMA-262, section 7.1, page 11, dictates that we
   // must remove Unicode format-control characters, we do not. This is
   // in line with how IE and SpiderMonkey handles it.
   if (!pushback_buffer()->is_empty()) {
     pos_++;
     return last_ = pushback_buffer()->RemoveLast();
-  } else if (stream_->has_more()) {
+  } else if (stream_->has_more() && pos_ < end_) {
     pos_++;
     uc32 next = stream_->GetNext();
     return last_ = next;
   } else {
     // Note: currently the following increment is necessary to avoid a
     // test-parser problem!
     pos_++;
     return last_ = static_cast<uc32>(-1);
   }
 }
 
 
 void CharacterStreamUTF16Buffer::SeekForward(int pos) {
   pos_ = pos;
   ASSERT(pushback_buffer()->is_empty());
   stream_->Seek(pos);
 }
 
 
-// TwoByteStringUTF16Buffer
-TwoByteStringUTF16Buffer::TwoByteStringUTF16Buffer()
+// ExternalStringUTF16Buffer
+template <typename StringType, typename CharType>
+ExternalStringUTF16Buffer<StringType, CharType>::ExternalStringUTF16Buffer()
     : raw_data_(NULL) { }
 
 
-void TwoByteStringUTF16Buffer::Initialize(
-     Handle<ExternalTwoByteString> data) {
+template <typename StringType, typename CharType>
+void ExternalStringUTF16Buffer<StringType, CharType>::Initialize(
+     Handle<StringType> data,
+     int start_position,
+     int end_position) {
   ASSERT(!data.is_null());
+  raw_data_ = data->resource()->data();
 
-  data_ = data;
-  pos_ = 0;
-
-  raw_data_ = data->resource()->data();
-  size_ = data->length();
+  ASSERT(end_position <= data->length());
+  if (start_position > 0) {
+    SeekForward(start_position);
+  }
+  end_ =
+      end_position != Scanner::kNoEndPosition ? end_position : data->length();
 }
 
 
-uc32 TwoByteStringUTF16Buffer::Advance() {
-  if (pos_ < size_) {
+template <typename StringType, typename CharType>
+uc32 ExternalStringUTF16Buffer<StringType, CharType>::Advance() {
+  if (pos_ < end_) {
     return raw_data_[pos_++];
   } else {
     // note: currently the following increment is necessary to avoid a
     // test-parser problem!
     pos_++;
     return static_cast<uc32>(-1);
   }
 }
 
 
-void TwoByteStringUTF16Buffer::PushBack(uc32 ch) {
+template <typename StringType, typename CharType>
+void ExternalStringUTF16Buffer<StringType, CharType>::PushBack(uc32 ch) {
   pos_--;
   ASSERT(pos_ >= Scanner::kCharacterLookaheadBufferSize);
   ASSERT(raw_data_[pos_ - Scanner::kCharacterLookaheadBufferSize] == ch);
 }
 
 
-void TwoByteStringUTF16Buffer::SeekForward(int pos) {
+template <typename StringType, typename CharType>
+void ExternalStringUTF16Buffer<StringType, CharType>::SeekForward(int pos) {
   pos_ = pos;
 }
 
 
 // ----------------------------------------------------------------------------
 // Keyword Matcher
 KeywordMatcher::FirstState KeywordMatcher::first_states_[] = {
   { "break",  KEYWORD_PREFIX, Token::BREAK },
   { NULL,     C,              Token::ILLEGAL },
   { NULL,     D,              Token::ILLEGAL },
@@ skipping 124 matching lines @@
 }
 
 
 // ----------------------------------------------------------------------------
 // Scanner
 
 Scanner::Scanner(ParserMode pre)
     : stack_overflow_(false), is_pre_parsing_(pre == PREPARSE) { }
 
 
+void Scanner::Initialize(Handle<String> source,
+                         ParserLanguage language) {
+  safe_string_input_buffer_.Reset(source.location());
+  Init(source, &safe_string_input_buffer_, 0, source->length(), language);
+}
+
+
+void Scanner::Initialize(Handle<String> source,
+                         unibrow::CharacterStream* stream,
+                         ParserLanguage language) {
+  Init(source, stream, 0, kNoEndPosition, language);
+}
+
+
+void Scanner::Initialize(Handle<String> source,
+                         int start_position,
+                         int end_position,
+                         ParserLanguage language) {
+  safe_string_input_buffer_.Reset(source.location());
+  Init(source, &safe_string_input_buffer_,
+       start_position, end_position, language);
+}
+
+
 void Scanner::Init(Handle<String> source,
                    unibrow::CharacterStream* stream,
-                   int position,
+                   int start_position,
+                   int end_position,
                    ParserLanguage language) {
   // Initialize the source buffer.
   if (!source.is_null() && StringShape(*source).IsExternalTwoByte()) {
     two_byte_string_buffer_.Initialize(
-        Handle<ExternalTwoByteString>::cast(source));
+        Handle<ExternalTwoByteString>::cast(source),
+        start_position,
+        end_position);
     source_ = &two_byte_string_buffer_;
+  } else if (!source.is_null() && StringShape(*source).IsExternalAscii()) {
+    ascii_string_buffer_.Initialize(
+        Handle<ExternalAsciiString>::cast(source),
+        start_position,
+        end_position);
+    source_ = &ascii_string_buffer_;
   } else {
-    char_stream_buffer_.Initialize(source, stream);
+    char_stream_buffer_.Initialize(source,
+                                   stream,
+                                   start_position,
+                                   end_position);
     source_ = &char_stream_buffer_;
   }
 
-  position_ = position;
   is_parsing_json_ = (language == JSON);
 
   // Set c0_ (one character ahead)
   ASSERT(kCharacterLookaheadBufferSize == 1);
   Advance();
   // Initializer current_ to not refer to a literal buffer.
   current_.literal_buffer = NULL;
 
   // Skip initial whitespace allowing HTML comment ends just like
   // after a newline and scan first token.
   has_line_terminator_before_next_ = true;
   SkipWhiteSpace();
   Scan();
 }
 
 
-Handle<String> Scanner::SubString(int start, int end) {
-  return source_->SubString(start - position_, end - position_);
-}
-
-
 Token::Value Scanner::Next() {
   // BUG 1215673: Find a thread safe way to set a stack limit in
   // pre-parse mode. Otherwise, we cannot safely pre-parse from other
   // threads.
   current_ = next_;
   // Check for stack-overflow before returning any tokens.
   StackLimitCheck check;
   if (check.HasOverflowed()) {
     stack_overflow_ = true;
     next_.token = Token::ILLEGAL;
@@ skipping 914 matching lines @@
     }
     AddCharAdvance();
   }
   TerminateLiteral();
 
   next_.location.end_pos = source_pos() - 1;
   return true;
 }
 
 } }  // namespace v8::internal