Version 3.0.2.

src/scanner-base.h

 // Copyright 2010 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 34 matching lines @@
 // Returns the value (0 .. 15) of a hexadecimal character c.
 // If c is not a legal hexadecimal character, returns a value < 0.
 inline int HexValue(uc32 c) {
   c -= '0';
   if (static_cast<unsigned>(c) <= 9) return c;
   c = (c | 0x20) - ('a' - '0');  // detect 0x11..0x16 and 0x31..0x36.
   if (static_cast<unsigned>(c) <= 5) return c + 10;
   return -1;
 }
 
+// ----------------------------------------------------------------------------
+// UTF16Buffer - scanner input source with pushback.
 
-// ---------------------------------------------------------------------
-// Buffered stream of characters, using an internal UC16 buffer.
+class UTF16Buffer {
+ public:
+  UTF16Buffer();
+  virtual ~UTF16Buffer() {}
 
-class UC16CharacterStream {
- public:
-  UC16CharacterStream() : pos_(0) { }
-  virtual ~UC16CharacterStream() { }
+  virtual void PushBack(uc32 ch) = 0;
+  // Returns a value < 0 when the buffer end is reached.
+  virtual uc32 Advance() = 0;
+  virtual void SeekForward(int pos) = 0;
 
-  // Returns and advances past the next UC16 character in the input
-  // stream. If there are no more characters, it returns a negative
-  // value.
-  inline int32_t Advance() {
-    if (buffer_cursor_ < buffer_end_ || ReadBlock()) {
-      pos_++;
-      return *(buffer_cursor_++);
-    }
-    // Note: currently the following increment is necessary to avoid a
-    // parser problem! The scanner treats the final kEndOfInput as
-    // a character with a position, and does math relative to that
-    // position.
-    pos_++;
+  int pos() const { return pos_; }
 
-    return kEndOfInput;
-  }
-
-  // Return the current position in the character stream.
-  // Starts at zero.
-  inline unsigned pos() const { return pos_; }
-
-  // Skips forward past the next character_count UC16 characters
-  // in the input, or until the end of input if that comes sooner.
-  // Returns the number of characters actually skipped. If less
-  // than character_count,
-  inline unsigned SeekForward(unsigned character_count) {
-    unsigned buffered_chars =
-        static_cast<unsigned>(buffer_end_ - buffer_cursor_);
-    if (character_count <= buffered_chars) {
-      buffer_cursor_ += character_count;
-      pos_ += character_count;
-      return character_count;
-    }
-    return SlowSeekForward(character_count);
-  }
-
-  // Pushes back the most recently read UC16 character, i.e.,
-  // the value returned by the most recent call to Advance.
-  // Must not be used right after calling SeekForward.
-  virtual void PushBack(uc16 character) = 0;
+  static const int kNoEndPosition = 1;
 
  protected:
-  static const int32_t kEndOfInput = -1;
+  // Initial value of end_ before the input stream is initialized.
 
-  // Ensures that the buffer_cursor_ points to the character at
-  // position pos_ of the input, if possible. If the position
-  // is at or after the end of the input, return false. If there
-  // are more characters available, return true.
-  virtual bool ReadBlock() = 0;
-  virtual unsigned SlowSeekForward(unsigned character_count) = 0;
-
-  const uc16* buffer_cursor_;
-  const uc16* buffer_end_;
-  unsigned pos_;
+  int pos_;  // Current position in the buffer.
+  int end_;  // Position where scanning should stop (EOF).
 };
 
 
-// ---------------------------------------------------------------------
-// Constants used by scanners.
-
 class ScannerConstants : AllStatic {
  public:
   typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder;
 
   static StaticResource<Utf8Decoder>* utf8_decoder() {
     return &utf8_decoder_;
   }
 
   static unibrow::Predicate<IdentifierStart, 128> kIsIdentifierStart;
   static unibrow::Predicate<IdentifierPart, 128> kIsIdentifierPart;
@@ skipping 180 matching lines @@
   }
 
   inline void AddLiteralCharAdvance() {
     AddLiteralChar(c0_);
     Advance();
   }
 
   // Low-level scanning support.
   void Advance() { c0_ = source_->Advance(); }
   void PushBack(uc32 ch) {
-    source_->PushBack(c0_);
+    source_->PushBack(ch);
     c0_ = ch;
   }
 
   inline Token::Value Select(Token::Value tok) {
     Advance();
     return tok;
   }
 
   inline Token::Value Select(uc32 next, Token::Value then, Token::Value else_) {
     Advance();
     if (c0_ == next) {
       Advance();
       return then;
     } else {
       return else_;
     }
   }
 
   uc32 ScanHexEscape(uc32 c, int length);
   uc32 ScanOctalEscape(uc32 c, int length);
 
   // Return the current source position.
   int source_pos() {
     return source_->pos() - kCharacterLookaheadBufferSize;
   }
 
   TokenDesc current_;  // desc for current token (as returned by Next())
   TokenDesc next_;     // desc for next token (one token look-ahead)
 
-  // Input stream. Must be initialized to an UC16CharacterStream.
-  UC16CharacterStream* source_;
+  // Input stream. Must be initialized to an UTF16Buffer.
+  UTF16Buffer* source_;
 
   // Buffer to hold literal values (identifiers, strings, numbers)
   // using '\x00'-terminated UTF-8 encoding. Handles allocation internally.
   LiteralCollector literal_buffer_;
 
   // One Unicode character look-ahead; c0_ < 0 at the end of the input.
   uc32 c0_;
 };
 
 // ----------------------------------------------------------------------------
@@ skipping 218 matching lines @@
   // keyword with the current prefix).
   const char* keyword_;
   int counter_;
   Token::Value keyword_token_;
 };
 
 
 } }  // namespace v8::internal
 
 #endif  // V8_SCANNER_BASE_H_