Experimental parser: merge r19949

src/scanner.h

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 26 matching lines @@
 #include "hashmap.h"
 #include "list.h"
 #include "token.h"
 #include "unicode-inl.h"
 #include "utils.h"
 
 namespace v8 {
 namespace internal {
 
 
+class ParserRecorder;
+
+
 // Returns the value (0 .. 15) of a hexadecimal character c.
 // If c is not a legal hexadecimal character, returns a value < 0.
 inline int HexValue(uc32 c) {
   c -= '0';
   if (static_cast<unsigned>(c) <= 9) return c;
   c = (c | 0x20) - ('a' - '0');  // detect 0x11..0x16 and 0x31..0x36.
   if (static_cast<unsigned>(c) <= 5) return c + 10;
   return -1;
 }
 
@@ skipping 56 matching lines @@
  protected:
   static const uc32 kEndOfInput = -1;
 
   // Ensures that the buffer_cursor_ points to the code_unit at
   // position pos_ of the input, if possible. If the position
   // is at or after the end of the input, return false. If there
   // are more code_units available, return true.
   virtual bool ReadBlock() = 0;
   virtual unsigned SlowSeekForward(unsigned code_unit_count) = 0;
 
-  const uc16* buffer_cursor_;
-  const uc16* buffer_end_;
+  const uint16_t* buffer_cursor_;
+  const uint16_t* buffer_end_;
   unsigned pos_;
 };
 
 
 #else
 
 class Utf16CharacterStream {
  public:
   enum StreamType {
     kUtf8ToUtf16, kGenericStringUtf16, kExternalTwoByteStringUtf16
@@ skipping 55 matching lines @@
 // ---------------------------------------------------------------------
 // DuplicateFinder discovers duplicate symbols.
 
 class DuplicateFinder {
  public:
   explicit DuplicateFinder(UnicodeCache* constants)
       : unicode_constants_(constants),
         backing_store_(16),
         map_(&Match) { }
 
-  int AddAsciiSymbol(Vector<const char> key, int value);
-  int AddUtf16Symbol(Vector<const uint16_t> key, int value);
+  int AddOneByteSymbol(Vector<const uint8_t> key, int value);
+  int AddTwoByteSymbol(Vector<const uint16_t> key, int value);
   // Add a a number literal by converting it (if necessary)
   // to the string that ToString(ToNumber(literal)) would generate.
   // and then adding that string with AddAsciiSymbol.
   // This string is the actual value used as key in an object literal,
   // and the one that must be different from the other keys.
-  int AddNumber(Vector<const char> key, int value);
+  int AddNumber(Vector<const uint8_t> key, int value);
 
  private:
-  int AddSymbol(Vector<const byte> key, bool is_ascii, int value);
+  int AddSymbol(Vector<const uint8_t> key, bool is_one_byte, int value);
   // Backs up the key and its length in the backing store.
   // The backup is stored with a base 127 encoding of the
-  // length (plus a bit saying whether the string is ASCII),
+  // length (plus a bit saying whether the string is one byte),
   // followed by the bytes of the key.
-  byte* BackupKey(Vector<const byte> key, bool is_ascii);
+  uint8_t* BackupKey(Vector<const uint8_t> key, bool is_one_byte);
 
   // Compare two encoded keys (both pointing into the backing store)
   // for having the same base-127 encoded lengths and ASCII-ness,
   // and then having the same 'length' bytes following.
   static bool Match(void* first, void* second);
   // Creates a hash from a sequence of bytes.
-  static uint32_t Hash(Vector<const byte> key, bool is_ascii);
+  static uint32_t Hash(Vector<const uint8_t> key, bool is_one_byte);
   // Checks whether a string containing a JS number is its canonical
   // form.
-  static bool IsNumberCanonical(Vector<const char> key);
+  static bool IsNumberCanonical(Vector<const uint8_t> key);
 
   // Size of buffer. Sufficient for using it to call DoubleToCString in
   // from conversions.h.
   static const int kBufferSize = 100;
 
   UnicodeCache* unicode_constants_;
   // Backing store used to store strings used as hashmap keys.
   SequenceCollector<unsigned char> backing_store_;
   HashMap map_;
   // Buffer used for string->number->canonical string conversions.
   char number_buffer_[kBufferSize];
 };
 
 
 // ----------------------------------------------------------------------------
 // LiteralBuffer -  Collector of chars of literals.
 
 class LiteralBuffer {
  public:
-  LiteralBuffer() : is_ascii_(true), position_(0), backing_store_() { }
+  LiteralBuffer() : is_one_byte_(true), position_(0), backing_store_() { }
 
   ~LiteralBuffer() {
     if (backing_store_.length() > 0) {
       backing_store_.Dispose();
     }
   }
 
   INLINE(void AddChar(uint32_t code_unit)) {
     if (position_ >= backing_store_.length()) ExpandBuffer();
-    if (is_ascii_) {
+    if (is_one_byte_) {
       if (code_unit <= unibrow::Latin1::kMaxChar) {
         backing_store_[position_] = static_cast<byte>(code_unit);
         position_ += kOneByteSize;
         return;
       }
-      ConvertToUtf16();
+      ConvertToTwoByte();
     }
     ASSERT(code_unit < 0x10000u);
-    *reinterpret_cast<uc16*>(&backing_store_[position_]) = code_unit;
+    *reinterpret_cast<uint16_t*>(&backing_store_[position_]) = code_unit;
     position_ += kUC16Size;
   }
 
-  bool is_ascii() { return is_ascii_; }
+  bool is_one_byte() { return is_one_byte_; }
 
   bool is_contextual_keyword(Vector<const char> keyword) {
-    return is_ascii() && keyword.length() == position_ &&
+    return is_one_byte() && keyword.length() == position_ &&
         (memcmp(keyword.start(), backing_store_.start(), position_) == 0);
   }
 
-  Vector<const uc16> utf16_literal() {
-    ASSERT(!is_ascii_);
+  Vector<const uint16_t> two_byte_literal() {
+    ASSERT(!is_one_byte_);
     ASSERT((position_ & 0x1) == 0);
-    return Vector<const uc16>(
-        reinterpret_cast<const uc16*>(backing_store_.start()),
+    return Vector<const uint16_t>(
+        reinterpret_cast<const uint16_t*>(backing_store_.start()),
         position_ >> 1);
   }
 
-  Vector<const char> ascii_literal() {
-    ASSERT(is_ascii_);
-    return Vector<const char>(
-        reinterpret_cast<const char*>(backing_store_.start()),
+  Vector<const uint8_t> one_byte_literal() {
+    ASSERT(is_one_byte_);
+    return Vector<const uint8_t>(
+        reinterpret_cast<const uint8_t*>(backing_store_.start()),
         position_);
   }
 
   int length() {
-    return is_ascii_ ? position_ : (position_ >> 1);
+    return is_one_byte_ ? position_ : (position_ >> 1);
   }
 
   void Reset() {
     position_ = 0;
-    is_ascii_ = true;
+    is_one_byte_ = true;
   }
 
  private:
   static const int kInitialCapacity = 16;
   static const int kGrowthFactory = 4;
   static const int kMinConversionSlack = 256;
   static const int kMaxGrowth = 1 * MB;
   inline int NewCapacity(int min_capacity) {
     int capacity = Max(min_capacity, backing_store_.length());
     int new_capacity = Min(capacity * kGrowthFactory, capacity + kMaxGrowth);
     return new_capacity;
   }
 
   void ExpandBuffer() {
     Vector<byte> new_store = Vector<byte>::New(NewCapacity(kInitialCapacity));
     OS::MemCopy(new_store.start(), backing_store_.start(), position_);
     backing_store_.Dispose();
     backing_store_ = new_store;
   }
 
-  void ConvertToUtf16() {
-    ASSERT(is_ascii_);
+  void ConvertToTwoByte() {
+    ASSERT(is_one_byte_);
     Vector<byte> new_store;
     int new_content_size = position_ * kUC16Size;
     if (new_content_size >= backing_store_.length()) {
       // Ensure room for all currently read code units as UC16 as well
       // as the code unit about to be stored.
       new_store = Vector<byte>::New(NewCapacity(new_content_size));
     } else {
       new_store = backing_store_;
     }
     uint8_t* src = backing_store_.start();
-    uc16* dst = reinterpret_cast<uc16*>(new_store.start());
+    uint16_t* dst = reinterpret_cast<uint16_t*>(new_store.start());
     for (int i = position_ - 1; i >= 0; i--) {
       dst[i] = src[i];
     }
     if (new_store.start() != backing_store_.start()) {
       backing_store_.Dispose();
       backing_store_ = new_store;
     }
     position_ = new_content_size;
-    is_ascii_ = false;
+    is_one_byte_ = false;
   }
 
-  bool is_ascii_;
+  bool is_one_byte_;
   int position_;
   Vector<byte> backing_store_;
 
   DISALLOW_COPY_AND_ASSIGN(LiteralBuffer);
 };
 
 
 #ifndef V8_USE_GENERATED_LEXER
 
 
@@ skipping 45 matching lines @@
 
   void Initialize(Utf16CharacterStream* source);
 
   // Returns the next token and advances input.
   Token::Value Next();
   // Returns the current token again.
   Token::Value current_token() { return current_.token; }
   // Returns the location information for the current token
   // (the token last returned by Next()).
   Location location() const { return current_.location; }
-  // Returns the literal string, if any, for the current token (the
-  // token last returned by Next()). The string is 0-terminated.
-  // Literal strings are collected for identifiers, strings, and
-  // numbers.
-  // These functions only give the correct result if the literal
-  // was scanned between calls to StartLiteral() and TerminateLiteral().
-  Vector<const char> literal_ascii_string() {
-    ASSERT_NOT_NULL(current_.literal_chars);
-    return current_.literal_chars->ascii_literal();
-  }
-  Vector<const uc16> literal_utf16_string() {
-    ASSERT_NOT_NULL(current_.literal_chars);
-    return current_.literal_chars->utf16_literal();
-  }
-  bool is_literal_ascii() {
-    ASSERT_NOT_NULL(current_.literal_chars);
-    return current_.literal_chars->is_ascii();
-  }
-  bool is_literal_contextual_keyword(Vector<const char> keyword) {
-    ASSERT_NOT_NULL(current_.literal_chars);
-    return current_.literal_chars->is_contextual_keyword(keyword);
-  }
-  int literal_length() const {
-    ASSERT_NOT_NULL(current_.literal_chars);
-    return current_.literal_chars->length();
-  }
+
+  // Similar functions for the upcoming token.
+
+  // One token look-ahead (past the token returned by Next()).
+  Token::Value peek() const { return next_.token; }
+
+  Location peek_location() const { return next_.location; }
 
   bool literal_contains_escapes() const {
     Location location = current_.location;
     int source_length = (location.end_pos - location.beg_pos);
     if (current_.token == Token::STRING) {
       // Subtract delimiters.
       source_length -= 2;
     }
     return current_.literal_chars->length() != source_length;
   }
-
-  // Similar functions for the upcoming token.
-
-  // One token look-ahead (past the token returned by Next()).
-  Token::Value peek() const { return next_.token; }
-
-  Location peek_location() const { return next_.location; }
-
-  // Returns the literal string for the next token (the token that
-  // would be returned if Next() were called).
-  Vector<const char> next_literal_ascii_string() {
-    ASSERT_NOT_NULL(next_.literal_chars);
-    return next_.literal_chars->ascii_literal();
-  }
-  Vector<const uc16> next_literal_utf16_string() {
-    ASSERT_NOT_NULL(next_.literal_chars);
-    return next_.literal_chars->utf16_literal();
-  }
-  bool is_next_literal_ascii() {
-    ASSERT_NOT_NULL(next_.literal_chars);
-    return next_.literal_chars->is_ascii();
+  bool is_literal_contextual_keyword(Vector<const char> keyword) {
+    ASSERT_NOT_NULL(current_.literal_chars);
+    return current_.literal_chars->is_contextual_keyword(keyword);
   }
   bool is_next_contextual_keyword(Vector<const char> keyword) {
     ASSERT_NOT_NULL(next_.literal_chars);
     return next_.literal_chars->is_contextual_keyword(keyword);
   }
-  int next_literal_length() const {
-    ASSERT_NOT_NULL(next_.literal_chars);
-    return next_.literal_chars->length();
+
+  Handle<String> AllocateNextLiteralString(Isolate* isolate,
+                                           PretenureFlag tenured);
+  Handle<String> AllocateInternalizedString(Isolate* isolate);
+
+  double DoubleValue();
+  bool UnescapedLiteralMatches(const char* data, int length) {
+    if (is_literal_one_byte() &&
+        literal_length() == length &&
+        !literal_contains_escapes()) {
+      const char* token =
+          reinterpret_cast<const char*>(literal_one_byte_string().start());
+      return !strncmp(token, data, length);
+    }
+    return false;
   }
+  void IsGetOrSet(bool* is_get, bool* is_set) {
+    if (is_literal_one_byte() &&
+        literal_length() == 3 &&
+        !literal_contains_escapes()) {
+      const char* token =
+          reinterpret_cast<const char*>(literal_one_byte_string().start());
+      *is_get = strncmp(token, "get", 3) == 0;
+      *is_set = !*is_get && strncmp(token, "set", 3) == 0;
+    }
+  }
+
+  int FindNumber(DuplicateFinder* finder, int value);
+  int FindSymbol(DuplicateFinder* finder, int value);
+
+  void LogSymbol(ParserRecorder* log, int position);
 
   UnicodeCache* unicode_cache() { return unicode_cache_; }
 
-  static const int kCharacterLookaheadBufferSize = 1;
-
-  // Scans octal escape sequence. Also accepts "\0" decimal escape sequence.
-  uc32 ScanOctalEscape(uc32 c, int length);
-
   // Returns the location of the last seen octal literal.
   Location octal_position() const { return octal_pos_; }
   void clear_octal_position() { octal_pos_ = Location::invalid(); }
 
   // Seek forward to the given position.  This operation does not
   // work in general, for instance when there are pushed back
   // characters, but works for seeking forward until simple delimiter
   // tokens, which is what it is used for.
   void SeekForward(int pos);
 
@@ skipping 31 matching lines @@
   bool ScanRegExpFlags();
 
  private:
   // The current and look-ahead token.
   struct TokenDesc {
     Token::Value token;
     Location location;
     LiteralBuffer* literal_chars;
   };
 
+  static const int kCharacterLookaheadBufferSize = 1;
+
+  // Scans octal escape sequence. Also accepts "\0" decimal escape sequence.
+  uc32 ScanOctalEscape(uc32 c, int length);
+
   // Call this after setting source_ to the input.
   void Init() {
     // Set c0_ (one character ahead)
     STATIC_ASSERT(kCharacterLookaheadBufferSize == 1);
     Advance();
     // Initialize current_ to not refer to a literal.
     current_.literal_chars = NULL;
   }
 
   // Literal buffer support
@@ skipping 40 matching lines @@
   inline Token::Value Select(uc32 next, Token::Value then, Token::Value else_) {
     Advance();
     if (c0_ == next) {
       Advance();
       return then;
     } else {
       return else_;
     }
   }
 
+  // Returns the literal string, if any, for the current token (the
+  // token last returned by Next()). The string is 0-terminated.
+  // Literal strings are collected for identifiers, strings, and
+  // numbers.
+  // These functions only give the correct result if the literal
+  // was scanned between calls to StartLiteral() and TerminateLiteral().
+  Vector<const uint8_t> literal_one_byte_string() {
+    ASSERT_NOT_NULL(current_.literal_chars);
+    return current_.literal_chars->one_byte_literal();
+  }
+  Vector<const uint16_t> literal_two_byte_string() {
+    ASSERT_NOT_NULL(current_.literal_chars);
+    return current_.literal_chars->two_byte_literal();
+  }
+  bool is_literal_one_byte() {
+    ASSERT_NOT_NULL(current_.literal_chars);
+    return current_.literal_chars->is_one_byte();
+  }
+  int literal_length() const {
+    ASSERT_NOT_NULL(current_.literal_chars);
+    return current_.literal_chars->length();
+  }
+  // Returns the literal string for the next token (the token that
+  // would be returned if Next() were called).
+  Vector<const uint8_t> next_literal_one_byte_string() {
+    ASSERT_NOT_NULL(next_.literal_chars);
+    return next_.literal_chars->one_byte_literal();
+  }
+  Vector<const uint16_t> next_literal_two_byte_string() {
+    ASSERT_NOT_NULL(next_.literal_chars);
+    return next_.literal_chars->two_byte_literal();
+  }
+  bool is_next_literal_one_byte() {
+    ASSERT_NOT_NULL(next_.literal_chars);
+    return next_.literal_chars->is_one_byte();
+  }
+  int next_literal_length() const {
+    ASSERT_NOT_NULL(next_.literal_chars);
+    return next_.literal_chars->length();
+  }
+
   uc32 ScanHexNumber(int expected_length);
 
   // Scans a single JavaScript token.
   void Scan();
 
   bool SkipWhiteSpace();
   Token::Value SkipSingleLineComment();
   Token::Value SkipMultiLineComment();
   // Scans a possible HTML comment -- begins with '<!'.
   Token::Value ScanHtmlComment();
@@ skipping 56 matching lines @@
   bool harmony_numeric_literals_;
 };
 
 
 #endif
 
 
 } }  // namespace v8::internal
 
 #endif  // V8_SCANNER_H_