Index: src/scanner-base.h |
diff --git a/src/scanner-base.h b/src/scanner-base.h |
index c50b8f3ef62dee77f0daba35aa61225de62a9d04..3d344f306ca0953e479ee90fe5446e0b999243f3 100644 |
--- a/src/scanner-base.h |
+++ b/src/scanner-base.h |
@@ -52,74 +52,30 @@ inline int HexValue(uc32 c) { |
return -1; |
} |
+// ---------------------------------------------------------------------------- |
+// UTF16Buffer - scanner input source with pushback. |
-// --------------------------------------------------------------------- |
-// Buffered stream of characters, using an internal UC16 buffer. |
- |
-class UC16CharacterStream { |
+class UTF16Buffer { |
public: |
- UC16CharacterStream() : pos_(0) { } |
- virtual ~UC16CharacterStream() { } |
- |
- // Returns and advances past the next UC16 character in the input |
- // stream. If there are no more characters, it returns a negative |
- // value. |
- inline int32_t Advance() { |
- if (buffer_cursor_ < buffer_end_ || ReadBlock()) { |
- pos_++; |
- return *(buffer_cursor_++); |
- } |
- // Note: currently the following increment is necessary to avoid a |
- // parser problem! The scanner treats the final kEndOfInput as |
- // a character with a position, and does math relative to that |
- // position. |
- pos_++; |
+ UTF16Buffer(); |
+ virtual ~UTF16Buffer() {} |
- return kEndOfInput; |
- } |
+ virtual void PushBack(uc32 ch) = 0; |
+ // Returns a value < 0 when the buffer end is reached. |
+ virtual uc32 Advance() = 0; |
+ virtual void SeekForward(int pos) = 0; |
- // Return the current position in the character stream. |
- // Starts at zero. |
- inline unsigned pos() const { return pos_; } |
- |
- // Skips forward past the next character_count UC16 characters |
- // in the input, or until the end of input if that comes sooner. |
- // Returns the number of characters actually skipped. If less |
- // than character_count, |
- inline unsigned SeekForward(unsigned character_count) { |
- unsigned buffered_chars = |
- static_cast<unsigned>(buffer_end_ - buffer_cursor_); |
- if (character_count <= buffered_chars) { |
- buffer_cursor_ += character_count; |
- pos_ += character_count; |
- return character_count; |
- } |
- return SlowSeekForward(character_count); |
- } |
+ int pos() const { return pos_; } |
- // Pushes back the most recently read UC16 character, i.e., |
- // the value returned by the most recent call to Advance. |
- // Must not be used right after calling SeekForward. |
- virtual void PushBack(uc16 character) = 0; |
+ static const int kNoEndPosition = 1; |
protected: |
- static const int32_t kEndOfInput = -1; |
- |
- // Ensures that the buffer_cursor_ points to the character at |
- // position pos_ of the input, if possible. If the position |
- // is at or after the end of the input, return false. If there |
- // are more characters available, return true. |
- virtual bool ReadBlock() = 0; |
- virtual unsigned SlowSeekForward(unsigned character_count) = 0; |
- |
- const uc16* buffer_cursor_; |
- const uc16* buffer_end_; |
- unsigned pos_; |
-}; |
+ // Initial value of end_ before the input stream is initialized. |
+ int pos_; // Current position in the buffer. |
+ int end_; // Position where scanning should stop (EOF). |
+}; |
-// --------------------------------------------------------------------- |
-// Constants used by scanners. |
class ScannerConstants : AllStatic { |
public: |
@@ -321,7 +277,7 @@ class Scanner { |
// Low-level scanning support. |
void Advance() { c0_ = source_->Advance(); } |
void PushBack(uc32 ch) { |
- source_->PushBack(c0_); |
+ source_->PushBack(ch); |
c0_ = ch; |
} |
@@ -351,8 +307,8 @@ class Scanner { |
TokenDesc current_; // desc for current token (as returned by Next()) |
TokenDesc next_; // desc for next token (one token look-ahead) |
- // Input stream. Must be initialized to an UC16CharacterStream. |
- UC16CharacterStream* source_; |
+ // Input stream. Must be initialized to an UTF16Buffer. |
+ UTF16Buffer* source_; |
// Buffer to hold literal values (identifiers, strings, numbers) |
// using '\x00'-terminated UTF-8 encoding. Handles allocation internally. |