Version 3.0.2.

src/scanner.h

Index: src/scanner.h
diff --git a/src/scanner.h b/src/scanner.h
index 572778f8ac4688cdc43832a7b2423d378e97cbe0..adeea9b23a00d50d7f07a9203dcc311fd50cebe4 100644
--- a/src/scanner.h
+++ b/src/scanner.h
@@ -35,97 +35,67 @@
 namespace v8 {
 namespace internal {
 
-// A buffered character stream based on a random access character
-// source (ReadBlock can be called with pos_ pointing to any position,
-// even positions before the current).
-class BufferedUC16CharacterStream: public UC16CharacterStream {
+// UTF16 buffer to read characters from a character stream.
+class CharacterStreamUTF16Buffer: public UTF16Buffer {
  public:
-  BufferedUC16CharacterStream();
-  virtual ~BufferedUC16CharacterStream();
-
-  virtual void PushBack(uc16 character);
-
- protected:
-  static const unsigned kBufferSize = 512;
-  static const unsigned kPushBackStepSize = 16;
-
-  virtual unsigned SlowSeekForward(unsigned delta);
-  virtual bool ReadBlock();
-  virtual void SlowPushBack(uc16 character);
-
-  virtual unsigned BufferSeekForward(unsigned delta) = 0;
-  virtual unsigned FillBuffer(unsigned position, unsigned length) = 0;
-
-  const uc16* pushback_limit_;
-  uc16 buffer_[kBufferSize];
+  CharacterStreamUTF16Buffer();
+  virtual ~CharacterStreamUTF16Buffer() {}
+  void Initialize(Handle<String> data,
+                  unibrow::CharacterStream* stream,
+                  int start_position,
+                  int end_position);
+  virtual void PushBack(uc32 ch);
+  virtual uc32 Advance();
+  virtual void SeekForward(int pos);
+
+ private:
+  List<uc32> pushback_buffer_;
+  uc32 last_;
+  unibrow::CharacterStream* stream_;
+
+  List<uc32>* pushback_buffer() { return &pushback_buffer_; }
 };
 
 
-// Generic string stream.
-class GenericStringUC16CharacterStream: public BufferedUC16CharacterStream {
- public:
-  GenericStringUC16CharacterStream(Handle<String> data,
-                                   unsigned start_position,
-                                   unsigned end_position);
-  virtual ~GenericStringUC16CharacterStream();
-
- protected:
-  virtual unsigned BufferSeekForward(unsigned delta);
-  virtual unsigned FillBuffer(unsigned position, unsigned length);
-
-  Handle<String> string_;
-  unsigned start_position_;
-  unsigned length_;
-};
-
-
-// UC16 stream based on a literal UTF-8 string.
-class Utf8ToUC16CharacterStream: public BufferedUC16CharacterStream {
+// UTF16 buffer to read characters from an external string.
+template <typename StringType, typename CharType>
+class ExternalStringUTF16Buffer: public UTF16Buffer {
  public:
-  Utf8ToUC16CharacterStream(const byte* data, unsigned length);
-  virtual ~Utf8ToUC16CharacterStream();
-
- protected:
-  virtual unsigned BufferSeekForward(unsigned delta);
-  virtual unsigned FillBuffer(unsigned char_position, unsigned length);
-  void SetRawPosition(unsigned char_position);
-
-  const byte* raw_data_;
-  unsigned raw_data_length_;  // Measured in bytes, not characters.
-  unsigned raw_data_pos_;
-  // The character position of the character at raw_data[raw_data_pos_].
-  // Not necessarily the same as pos_.
-  unsigned raw_character_position_;
+  ExternalStringUTF16Buffer();
+  virtual ~ExternalStringUTF16Buffer() {}
+  void Initialize(Handle<StringType> data,
+                  int start_position,
+                  int end_position);
+  virtual void PushBack(uc32 ch);
+  virtual uc32 Advance();
+  virtual void SeekForward(int pos);
+
+ private:
+  const CharType* raw_data_;  // Pointer to the actual array of characters.
 };
 
 
-// UTF16 buffer to read characters from an external string.
-class ExternalTwoByteStringUC16CharacterStream: public UC16CharacterStream {
+// Initializes a UTF16Buffer as input stream, using one of a number
+// of strategies depending on the available character sources.
+class StreamInitializer {
  public:
-  ExternalTwoByteStringUC16CharacterStream(Handle<ExternalTwoByteString> data,
-                                           int start_position,
-                                           int end_position);
-  virtual ~ExternalTwoByteStringUC16CharacterStream();
-
-  virtual void PushBack(uc16 character) {
-    ASSERT(buffer_cursor_ > raw_data_);
-    buffer_cursor_--;
-    pos_--;
-  }
- protected:
-  virtual unsigned SlowSeekForward(unsigned delta) {
-    // Fast case always handles seeking.
-    return 0;
-  }
-  virtual bool ReadBlock() {
-    // Entire string is read at start.
-    return false;
-  }
-  Handle<ExternalTwoByteString> source_;
-  const uc16* raw_data_;  // Pointer to the actual array of characters.
+  UTF16Buffer* Init(Handle<String> source,
+                    unibrow::CharacterStream* stream,
+                    int start_position,
+                    int end_position);
+ private:
+  // Different UTF16 buffers used to pull characters from. Based on input one of
+  // these will be initialized as the actual data source.
+  CharacterStreamUTF16Buffer char_stream_buffer_;
+  ExternalStringUTF16Buffer<ExternalTwoByteString, uint16_t>
+      two_byte_string_buffer_;
+  ExternalStringUTF16Buffer<ExternalAsciiString, char> ascii_string_buffer_;
+
+  // Used to convert the source string into a character stream when a stream
+  // is not passed to the scanner.
+  SafeStringInputBuffer safe_string_input_buffer_;
 };
 
-
 // ----------------------------------------------------------------------------
 // V8JavaScriptScanner
 // JavaScript scanner getting its input from either a V8 String or a unicode
@@ -133,9 +103,19 @@ class ExternalTwoByteStringUC16CharacterStream: public UC16CharacterStream {
 
 class V8JavaScriptScanner : public JavaScriptScanner {
  public:
-  V8JavaScriptScanner();
-  void Initialize(UC16CharacterStream* source,
+  V8JavaScriptScanner() {}
+
+  // Initialize the Scanner to scan source.
+  void Initialize(Handle<String> source, int literal_flags = kAllLiterals);
+  void Initialize(Handle<String> source,
+                  unibrow::CharacterStream* stream,
+                  int literal_flags = kAllLiterals);
+  void Initialize(Handle<String> source,
+                  int start_position, int end_position,
                   int literal_flags = kAllLiterals);
+
+ protected:
+  StreamInitializer stream_initializer_;
 };
 
 
@@ -143,7 +123,8 @@ class JsonScanner : public Scanner {
  public:
   JsonScanner();
 
-  void Initialize(UC16CharacterStream* source);
+  // Initialize the Scanner to scan source.
+  void Initialize(Handle<String> source);
 
   // Returns the next token.
   Token::Value Next();
@@ -157,7 +138,7 @@ class JsonScanner : public Scanner {
   // Recognizes all of the single-character tokens directly, or calls a function
   // to scan a number, string or identifier literal.
   // The only allowed whitespace characters between tokens are tab,
-  // carriage-return, newline and space.
+  // carrige-return, newline and space.
   void ScanJson();
 
   // A JSON number (production JSONNumber) is a subset of the valid JavaScript
@@ -178,8 +159,60 @@ class JsonScanner : public Scanner {
   // are the only valid JSON identifiers (productions JSONBooleanLiteral,
   // JSONNullLiteral).
   Token::Value ScanJsonIdentifier(const char* text, Token::Value token);
+
+  StreamInitializer stream_initializer_;
 };
 
+
+// ExternalStringUTF16Buffer
+template <typename StringType, typename CharType>
+ExternalStringUTF16Buffer<StringType, CharType>::ExternalStringUTF16Buffer()
+    : raw_data_(NULL) { }
+
+
+template <typename StringType, typename CharType>
+void ExternalStringUTF16Buffer<StringType, CharType>::Initialize(
+     Handle<StringType> data,
+     int start_position,
+     int end_position) {
+  ASSERT(!data.is_null());
+  raw_data_ = data->resource()->data();
+
+  ASSERT(end_position <= data->length());
+  if (start_position > 0) {
+    SeekForward(start_position);
+  }
+  end_ =
+      end_position != kNoEndPosition ? end_position : data->length();
+}
+
+
+template <typename StringType, typename CharType>
+uc32 ExternalStringUTF16Buffer<StringType, CharType>::Advance() {
+  if (pos_ < end_) {
+    return raw_data_[pos_++];
+  } else {
+    // note: currently the following increment is necessary to avoid a
+    // test-parser problem!
+    pos_++;
+    return static_cast<uc32>(-1);
+  }
+}
+
+
+template <typename StringType, typename CharType>
+void ExternalStringUTF16Buffer<StringType, CharType>::PushBack(uc32 ch) {
+  pos_--;
+  ASSERT(pos_ >= Scanner::kCharacterLookaheadBufferSize);
+  ASSERT(raw_data_[pos_ - Scanner::kCharacterLookaheadBufferSize] == ch);
+}
+
+
+template <typename StringType, typename CharType>
+void ExternalStringUTF16Buffer<StringType, CharType>::SeekForward(int pos) {
+  pos_ = pos;
+}
+
 } }  // namespace v8::internal
 
 #endif  // V8_SCANNER_H_