Fix bug when the scanner does a pushback at the end of input.

src/scanner-base.h

 // Copyright 2010 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 46 matching lines @@
 // Buffered stream of characters, using an internal UC16 buffer.
 
 class UC16CharacterStream {
  public:
   UC16CharacterStream() : pos_(0) { }
   virtual ~UC16CharacterStream() { }
 
   // Returns and advances past the next UC16 character in the input
   // stream. If there are no more characters, it returns a negative
   // value.
-  inline int32_t Advance() {
+  inline uc32 Advance() {
     if (buffer_cursor_ < buffer_end_ || ReadBlock()) {
       pos_++;
-      return *(buffer_cursor_++);
+      return static_cast<uc32>(*(buffer_cursor_++));
     }
     // Note: currently the following increment is necessary to avoid a
     // parser problem! The scanner treats the final kEndOfInput as
     // a character with a position, and does math relative to that
     // position.
     pos_++;
 
     return kEndOfInput;
   }
 
   // Return the current position in the character stream.
   // Starts at zero.
   inline unsigned pos() const { return pos_; }
 
   // Skips forward past the next character_count UC16 characters
   // in the input, or until the end of input if that comes sooner.
   // Returns the number of characters actually skipped. If less
   // than character_count,
   inline unsigned SeekForward(unsigned character_count) {
     unsigned buffered_chars =
         static_cast<unsigned>(buffer_end_ - buffer_cursor_);
     if (character_count <= buffered_chars) {
       buffer_cursor_ += character_count;
       pos_ += character_count;
       return character_count;
     }
     return SlowSeekForward(character_count);
   }
 
-  // Pushes back the most recently read UC16 character, i.e.,
-  // the value returned by the most recent call to Advance.
+  // Pushes back the most recently read UC16 character (or negative

[Søren Thygesen Gjesse, 2011/01/14 10:37:16]

negative value -> kEndOfInput?

[Lasse Reichstein, 2011/01/14 10:49:04]

The kEndOfInput name isn't publicly visible. All we guarantee is that we return
negative for end-of-input.
The constant is used internally only to give meaningful name to the value
returned in those case.

+  // value if at end of input), i.e., the value returned by the most recent
+  // call to Advance.
   // Must not be used right after calling SeekForward.
-  virtual void PushBack(uc16 character) = 0;
+  virtual void PushBack(int32_t character) = 0;
 
  protected:
-  static const int32_t kEndOfInput = -1;
+  static const uc32 kEndOfInput = -1;
 
   // Ensures that the buffer_cursor_ points to the character at
   // position pos_ of the input, if possible. If the position
   // is at or after the end of the input, return false. If there
   // are more characters available, return true.
   virtual bool ReadBlock() = 0;
   virtual unsigned SlowSeekForward(unsigned character_count) = 0;
 
   const uc16* buffer_cursor_;
   const uc16* buffer_end_;
@@ skipping 499 matching lines @@
   // keyword with the current prefix).
   const char* keyword_;
   int counter_;
   Token::Value keyword_token_;
 };
 
 
 } }  // namespace v8::internal
 
 #endif  // V8_SCANNER_BASE_H_