Refactor the scanner interface...

src/scanner.h

 // Copyright 2006-2008 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 66 matching lines @@
   }
 
   static char* ComputeLimit(char* data, int capacity) {
     return (data + capacity) - unibrow::Utf8::kMaxEncodedSize;
   }
 
   void AddCharSlow(uc32 c);
 };
 
 
+// Interface through which the scanner reads characters from the input source.
 class UTF16Buffer {
  public:
   UTF16Buffer();
   virtual ~UTF16Buffer() {}
 
   virtual void PushBack(uc32 ch) = 0;
-  // returns a value < 0 when the buffer end is reached
+  // Returns a value < 0 when the buffer end is reached.
   virtual uc32 Advance() = 0;
   virtual void SeekForward(int pos) = 0;
 
   int pos() const { return pos_; }
-  int size() const { return size_; }
-  Handle<String> SubString(int start, int end);
 
  protected:
-  Handle<String> data_;
-  int pos_;
-  int size_;
+  int pos_;  // Current position in the buffer.
+  int end_;  // Position where scanning should stop (EOF).
 };
 
 
+// UTF16 buffer to read characters from a character stream.
 class CharacterStreamUTF16Buffer: public UTF16Buffer {
  public:
   CharacterStreamUTF16Buffer();
   virtual ~CharacterStreamUTF16Buffer() {}
-  void Initialize(Handle<String> data, unibrow::CharacterStream* stream);
+  void Initialize(Handle<String> data,
+                  unibrow::CharacterStream* stream,
+                  int start_position,
+                  int end_position);
   virtual void PushBack(uc32 ch);
   virtual uc32 Advance();
   virtual void SeekForward(int pos);
 
  private:
   List<uc32> pushback_buffer_;
   uc32 last_;
   unibrow::CharacterStream* stream_;
 
   List<uc32>* pushback_buffer() { return &pushback_buffer_; }
 };
 
 
-class TwoByteStringUTF16Buffer: public UTF16Buffer {
+// UTF16 buffer to read characters from an external string.
+template <typename StringType, typename CharType>
+class ExternalStringUTF16Buffer: public UTF16Buffer {
  public:
-  TwoByteStringUTF16Buffer();
-  virtual ~TwoByteStringUTF16Buffer() {}
-  void Initialize(Handle<ExternalTwoByteString> data);
+  ExternalStringUTF16Buffer();
+  virtual ~ExternalStringUTF16Buffer() {}
+  void Initialize(Handle<StringType> data,
+                  int start_position,
+                  int end_position);
   virtual void PushBack(uc32 ch);
   virtual uc32 Advance();
   virtual void SeekForward(int pos);
 
  private:
-  const uint16_t* raw_data_;
+  const CharType* raw_data_;  // Pointer to the actual array of characters.
 };
 
 
 class KeywordMatcher {
 //  Incrementally recognize keywords.
 //
 //  Recognized keywords:
 //      break case catch const* continue debugger* default delete do else
 //      finally false for function if in instanceof native* new null
 //      return switch this throw true try typeof var void while with
@@ skipping 109 matching lines @@
 enum ParserLanguage { JAVASCRIPT, JSON };
 
 
 class Scanner {
  public:
   typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder;
 
   // Construction
   explicit Scanner(ParserMode parse_mode);
 
-  // Initialize the Scanner to scan source:
-  void Init(Handle<String> source,
-            unibrow::CharacterStream* stream,
-            int position,
-            ParserLanguage language);
+  // Initialize the Scanner to scan source.
+  void Initialize(Handle<String> source,
+                  ParserLanguage language);
+  void Initialize(Handle<String> source,
+                  unibrow::CharacterStream* stream,
+                  ParserLanguage language);
+  void Initialize(Handle<String> source,
+                  int start_position, int end_position,
+                  ParserLanguage language);
 
   // Returns the next token.
   Token::Value Next();
 
   // One token look-ahead (past the token returned by Next()).
   Token::Value peek() const  { return next_.token; }
 
   // Returns true if there was a line terminator before the peek'ed token.
   bool has_line_terminator_before_next() const {
     return has_line_terminator_before_next_;
@@ skipping 47 matching lines @@
   // Returns true if regexp flags are scanned (always since flags can
   // be empty).
   bool ScanRegExpFlags();
 
   // Seek forward to the given position.  This operation does not
   // work in general, for instance when there are pushed back
   // characters, but works for seeking forward until simple delimiter
   // tokens, which is what it is used for.
   void SeekForward(int pos);
 
-  Handle<String> SubString(int start_pos, int end_pos);
   bool stack_overflow() { return stack_overflow_; }
 
   static StaticResource<Utf8Decoder>* utf8_decoder() { return &utf8_decoder_; }
 
   // Tells whether the buffer contains an identifier (no escapes).
   // Used for checking if a property name is an identifier.
   static bool IsIdentifier(unibrow::CharacterStream* buffer);
 
   static unibrow::Predicate<IdentifierStart, 128> kIsIdentifierStart;
   static unibrow::Predicate<IdentifierPart, 128> kIsIdentifierPart;
   static unibrow::Predicate<unibrow::LineTerminator, 128> kIsLineTerminator;
   static unibrow::Predicate<unibrow::WhiteSpace, 128> kIsWhiteSpace;
 
   static const int kCharacterLookaheadBufferSize = 1;
+  static const int kNoEndPosition = 1;
 
  private:
+  void Init(Handle<String> source,
+            unibrow::CharacterStream* stream,
+            int start_position, int end_position,
+            ParserLanguage language);
+
+
+  // Different UTF16 buffers used to pull characters from. Based on input one of
+  // these will be initialized as the actual data source.
   CharacterStreamUTF16Buffer char_stream_buffer_;
-  TwoByteStringUTF16Buffer two_byte_string_buffer_;
+  ExternalStringUTF16Buffer<ExternalTwoByteString, uint16_t>
+      two_byte_string_buffer_;
+  ExternalStringUTF16Buffer<ExternalAsciiString, char> ascii_string_buffer_;
 
-  // Source.
+  // Source. Will point to one of the buffers declared above.
   UTF16Buffer* source_;
-  int position_;
+
+  // Used to convert the source string into a character stream when a stream
+  // is not passed to the scanner.
+  SafeStringInputBuffer safe_string_input_buffer_;
 
   // Buffer to hold literal values (identifiers, strings, numbers)
   // using 0-terminated UTF-8 encoding.
   UTF8Buffer literal_buffer_1_;
   UTF8Buffer literal_buffer_2_;
 
   bool stack_overflow_;
   static StaticResource<Utf8Decoder> utf8_decoder_;
 
   // One Unicode character look-ahead; c0_ < 0 at the end of the input.
@@ skipping 82 matching lines @@
   uc32 ScanHexEscape(uc32 c, int length);
   uc32 ScanOctalEscape(uc32 c, int length);
   void ScanEscape();
   Token::Value ScanString();
 
   // Scans a possible HTML comment -- begins with '<!'.
   Token::Value ScanHtmlComment();
 
   // Return the current source position.
   int source_pos() {
-    return source_->pos() - kCharacterLookaheadBufferSize + position_;
+    return source_->pos() - kCharacterLookaheadBufferSize;
   }
 
   // Decodes a unicode escape-sequence which is part of an identifier.
   // If the escape sequence cannot be decoded the result is kBadRune.
   uc32 ScanIdentifierUnicodeEscape();
 };
 
 } }  // namespace v8::internal
 
 #endif  // V8_SCANNER_H_