Merge 6168:6800 from bleeding_edge to experimental/gc branch.

src/scanner-base.h

 // Copyright 2010 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 46 matching lines @@
 // Buffered stream of characters, using an internal UC16 buffer.
 
 class UC16CharacterStream {
  public:
   UC16CharacterStream() : pos_(0) { }
   virtual ~UC16CharacterStream() { }
 
   // Returns and advances past the next UC16 character in the input
   // stream. If there are no more characters, it returns a negative
   // value.
-  inline int32_t Advance() {
+  inline uc32 Advance() {
     if (buffer_cursor_ < buffer_end_ || ReadBlock()) {
       pos_++;
-      return *(buffer_cursor_++);
+      return static_cast<uc32>(*(buffer_cursor_++));
     }
     // Note: currently the following increment is necessary to avoid a
     // parser problem! The scanner treats the final kEndOfInput as
     // a character with a position, and does math relative to that
     // position.
     pos_++;
 
     return kEndOfInput;
   }
 
   // Return the current position in the character stream.
   // Starts at zero.
   inline unsigned pos() const { return pos_; }
 
   // Skips forward past the next character_count UC16 characters
   // in the input, or until the end of input if that comes sooner.
   // Returns the number of characters actually skipped. If less
   // than character_count,
   inline unsigned SeekForward(unsigned character_count) {
     unsigned buffered_chars =
         static_cast<unsigned>(buffer_end_ - buffer_cursor_);
     if (character_count <= buffered_chars) {
       buffer_cursor_ += character_count;
       pos_ += character_count;
       return character_count;
     }
     return SlowSeekForward(character_count);
   }
 
-  // Pushes back the most recently read UC16 character, i.e.,
-  // the value returned by the most recent call to Advance.
+  // Pushes back the most recently read UC16 character (or negative
+  // value if at end of input), i.e., the value returned by the most recent
+  // call to Advance.
   // Must not be used right after calling SeekForward.
-  virtual void PushBack(uc16 character) = 0;
+  virtual void PushBack(int32_t character) = 0;
 
  protected:
-  static const int32_t kEndOfInput = -1;
+  static const uc32 kEndOfInput = -1;
 
   // Ensures that the buffer_cursor_ points to the character at
   // position pos_ of the input, if possible. If the position
   // is at or after the end of the input, return false. If there
   // are more characters available, return true.
   virtual bool ReadBlock() = 0;
   virtual unsigned SlowSeekForward(unsigned character_count) = 0;
 
   const uc16* buffer_cursor_;
   const uc16* buffer_end_;
@@ skipping 122 matching lines @@
   Vector<byte> backing_store_;
 };
 
 
 // ----------------------------------------------------------------------------
 // Scanner base-class.
 
 // Generic functionality used by both JSON and JavaScript scanners.
 class Scanner {
  public:
+  // -1 is outside of the range of any real source code.
+  static const int kNoOctalLocation = -1;
+
   typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder;
 
   class LiteralScope {
    public:
     explicit LiteralScope(Scanner* self);
     ~LiteralScope();
     void Complete();
 
    private:
     Scanner* scanner_;
     bool complete_;
   };
 
   Scanner();
 
   // Returns the current token again.
   Token::Value current_token() { return current_.token; }
 
   // One token look-ahead (past the token returned by Next()).
   Token::Value peek() const { return next_.token; }
 
   struct Location {
     Location(int b, int e) : beg_pos(b), end_pos(e) { }
     Location() : beg_pos(0), end_pos(0) { }
+
+    bool IsValid() const {
+      return beg_pos >= 0 && end_pos >= beg_pos;
+    }
+
     int beg_pos;
     int end_pos;
   };
 
+  static Location NoLocation() {
+    return Location(-1, -1);
+  }
+
   // Returns the location information for the current token
   // (the token returned by Next()).
   Location location() const { return current_.location; }
   Location peek_location() const { return next_.location; }
 
+  // Returns the location of the last seen octal literal
+  int octal_position() const { return octal_pos_; }
+  void clear_octal_position() { octal_pos_ = -1; }
+
   // Returns the literal string, if any, for the current token (the
   // token returned by Next()). The string is 0-terminated and in
   // UTF-8 format; they may contain 0-characters. Literal strings are
   // collected for identifiers, strings, and numbers.
   // These functions only give the correct result if the literal
   // was scanned between calls to StartLiteral() and TerminateLiteral().
   bool is_literal_ascii() {
     ASSERT_NOT_NULL(current_.literal_chars);
     return current_.literal_chars->is_ascii();
   }
@@ skipping 93 matching lines @@
     Advance();
     if (c0_ == next) {
       Advance();
       return then;
     } else {
       return else_;
     }
   }
 
   uc32 ScanHexEscape(uc32 c, int length);
+
+  // Scans octal escape sequence. Also accepts "\0" decimal escape sequence.
   uc32 ScanOctalEscape(uc32 c, int length);
 
   // Return the current source position.
   int source_pos() {
     return source_->pos() - kCharacterLookaheadBufferSize;
   }
 
   // Buffers collecting literal strings, numbers, etc.
   LiteralBuffer literal_buffer1_;
   LiteralBuffer literal_buffer2_;
 
   TokenDesc current_;  // desc for current token (as returned by Next())
   TokenDesc next_;     // desc for next token (one token look-ahead)
 
   // Input stream. Must be initialized to an UC16CharacterStream.
   UC16CharacterStream* source_;
 
+  // Start position of the octal literal last scanned.
+  int octal_pos_;
 
   // One Unicode character look-ahead; c0_ < 0 at the end of the input.
   uc32 c0_;
 };
 
 // ----------------------------------------------------------------------------
 // JavaScriptScanner - base logic for JavaScript scanning.
 
 class JavaScriptScanner : public Scanner {
  public:
@@ skipping 114 matching lines @@
     UNMATCHABLE,
     INITIAL,
     KEYWORD_PREFIX,
     KEYWORD_MATCHED,
     C,
     CA,
     CO,
     CON,
     D,
     DE,
+    E,
+    EX,
     F,
     I,
+    IM,
+    IMP,
     IN,
     N,
+    P,
+    PR,
+    S,
     T,
     TH,
     TR,
     V,
     W
   };
 
   struct FirstState {
     const char* keyword;
     State state;
     Token::Value token;
   };
 
   // Range of possible first characters of a keyword.
   static const unsigned int kFirstCharRangeMin = 'b';
-  static const unsigned int kFirstCharRangeMax = 'w';
+  static const unsigned int kFirstCharRangeMax = 'y';
   static const unsigned int kFirstCharRangeLength =
       kFirstCharRangeMax - kFirstCharRangeMin + 1;
   // State map for first keyword character range.
   static FirstState first_states_[kFirstCharRangeLength];
 
   // If input equals keyword's character at position, continue matching keyword
   // from that position.
   inline bool MatchKeywordStart(unibrow::uchar input,
                                 const char* keyword,
                                 int position,
@@ skipping 40 matching lines @@
   // keyword with the current prefix).
   const char* keyword_;
   int counter_;
   Token::Value keyword_token_;
 };
 
 
 } }  // namespace v8::internal
 
 #endif  // V8_SCANNER_BASE_H_