Experimental parser: merge to r19637

src/lexer/lexer.h

Index: src/lexer/lexer.h
diff --git a/src/lexer/experimental-scanner.h b/src/lexer/lexer.h
similarity index 50%
rename from src/lexer/experimental-scanner.h
rename to src/lexer/lexer.h
index 8bcf61f207c113f201db092c100959e508282396..85c65289a6679ee48acb3d4ab8996ba6a0c05beb 100644
--- a/src/lexer/experimental-scanner.h
+++ b/src/lexer/lexer.h
@@ -28,22 +28,29 @@
 #ifndef V8_LEXER_EXPERIMENTAL_SCANNER_H
 #define V8_LEXER_EXPERIMENTAL_SCANNER_H
 
-#include "compiler.h"
-#include "isolate.h"
-#include "scanner.h"  // UnicodeCache.
-#include "token.h"
-#include "utils.h"
-#include "v8stdint.h"
-#include "char-predicates-inl.h"
+#include <set>
+#include "handles.h"
+#include "scanner.h"
 
 namespace v8 {
 namespace internal {
 
-class UnicodeCache;
+class LexerBase;
 
-// Base class for scanners for different encodings. The meat is the pure virtual
-// Scan() which each of them specializes.
-class ScannerBase {
+class LexerGCHandler {
+ public:
+  explicit LexerGCHandler(Isolate* isolate) : isolate_(isolate) {}
+  void AddLexer(LexerBase* lexer);
+  void RemoveLexer(LexerBase* lexer);
+  void UpdateLexersAfterGC();
+
+ private:
+  Isolate* isolate_;
+  std::set<LexerBase*> lexers_;
+};
+
+
+class LexerBase {
  public:
   struct Location {
     Location(int b, int e) : beg_pos(b), end_pos(e) { }
@@ -59,25 +66,37 @@ class ScannerBase {
     int end_pos;
   };
 
-  explicit ScannerBase(Isolate* isolate)
-    : isolate_(isolate),
-      unicode_cache_(isolate->unicode_cache()),
-      has_line_terminator_before_next_(true),
-      has_multiline_comment_before_next_(false),
-      current_literal_(&literals_[0]),
-      next_literal_(&literals_[1]),
-      harmony_numeric_literals_(false),
-      harmony_modules_(false),
-      harmony_scoping_(false) {
-    isolate->AddScanner(this);
+  explicit LexerBase(UnicodeCache* unicode_cache)
+      : unicode_cache_(unicode_cache),
+        has_line_terminator_before_next_(true),
+        has_multiline_comment_before_next_(false),
+        current_literal_(&literals_[0]),
+        next_literal_(&literals_[1]),
+        harmony_numeric_literals_(false),
+        harmony_modules_(false),
+        harmony_scoping_(false) {
   }
 
-  virtual ~ScannerBase() {
-    isolate_->RemoveScanner(this);
+  virtual ~LexerBase();
+
+  // Returns the next token and advances input.
+  Token::Value Next();
+
+  // Returns the current token again.
+  Token::Value current_token() const { return current_.token; }
+
+  // Returns the location information for the current token
+  // (the token last returned by Next()).
+  Location location() const {
+    return Location(current_.beg_pos, current_.end_pos);
   }
 
-  // Has to be called after creating the scanner and setting the flags.
-  virtual void Init() = 0;
+  // One token look-ahead (past the token returned by Next()).
+  Token::Value peek() const { return next_.token; }
+
+  Location peek_location() const {
+    return Location(next_.beg_pos, next_.end_pos);
+  }
 
   // Seek forward to the given position. This operation works for simple cases
   // such as seeking forward until simple delimiter tokens, which is what it is
@@ -85,70 +104,22 @@ class ScannerBase {
   // the "next" token. The "current" token will be invalid. FIXME: for utf-8,
   // we need to decide if pos is counted in characters or in bytes.
   virtual void SeekForward(int pos) = 0;
+
   virtual void SetEnd(int pos) = 0;
 
   // Scans the input as a regular expression pattern, previous character(s) must
   // be /(=). Returns true if a pattern is scanned. FIXME: this won't work for
   // utf-8 newlines.
   virtual bool ScanRegExpPattern(bool seen_equal) = 0;
+
   // Returns true if regexp flags are scanned (always since flags can
   // be empty).
   virtual bool ScanRegExpFlags() = 0;
 
-  // Returns the location of the last seen octal literal.
+  // // Returns the location of the last seen octal literal.
   virtual Location octal_position() const = 0;
-  virtual void clear_octal_position() = 0;
 
-  // Sets the raw string pointer based on the string handle. Needs to be called
-  // right after GC.
-  virtual void UpdateBufferBasedOnHandle() = 0;
-
-  // Returns the next token and advances input.
-  Token::Value Next() {
-    has_line_terminator_before_next_ = false;
-    has_multiline_comment_before_next_ = false;
-    current_ = next_;
-    std::swap(current_literal_, next_literal_);
-    Scan();  // Virtual! Will fill in next_.
-    return current_.token;
-  }
-
-  // Returns the current token again.
-  Token::Value current_token() { return current_.token; }
-
-  // Returns the location information for the current token
-  // (the token last returned by Next()).
-  Location location() {
-    return Location(current_.beg_pos, current_.end_pos);
-  }
-
-  // One token look-ahead (past the token returned by Next()).
-  Token::Value peek() const { return next_.token; }
-
-  Location peek_location() const {
-    return Location(next_.beg_pos, next_.end_pos);
-  }
-
-  UnicodeCache* unicode_cache() { return unicode_cache_; }
-
-  bool HarmonyScoping() const {
-    return harmony_scoping_;
-  }
-  void SetHarmonyScoping(bool scoping) {
-    harmony_scoping_ = scoping;
-  }
-  bool HarmonyModules() const {
-    return harmony_modules_;
-  }
-  void SetHarmonyModules(bool modules) {
-    harmony_modules_ = modules;
-  }
-  bool HarmonyNumericLiterals() const {
-    return harmony_numeric_literals_;
-  }
-  void SetHarmonyNumericLiterals(bool numeric_literals) {
-    harmony_numeric_literals_ = numeric_literals;
-  }
+  virtual void clear_octal_position() = 0;
 
   // Returns true if there was a line terminator before the peek'ed token,
   // possibly inside a multi-line comment.
@@ -187,7 +158,6 @@ class ScannerBase {
     return current_literal_->length;
   }
 
-  // This should be is_onebyte or is_latin1; it doesn't mean ASCII for real.
   bool is_literal_ascii() {
     EnsureCurrentLiteralIsValid();
     return current_literal_->is_ascii;
@@ -231,6 +201,32 @@ class ScannerBase {
         (memcmp(literal.start(), keyword.start(), literal.length()) == 0);
   }
 
+  bool HarmonyScoping() const {
+    return harmony_scoping_;
+  }
+
+  void SetHarmonyScoping(bool scoping) {
+    harmony_scoping_ = scoping;
+  }
+
+  bool HarmonyModules() const {
+    return harmony_modules_;
+  }
+
+  void SetHarmonyModules(bool modules) {
+    harmony_modules_ = modules;
+  }
+
+  bool HarmonyNumericLiterals() const {
+    return harmony_numeric_literals_;
+  }
+
+  void SetHarmonyNumericLiterals(bool numeric_literals) {
+    harmony_numeric_literals_ = numeric_literals;
+  }
+
+  UnicodeCache* unicode_cache() { return unicode_cache_; }
+
  protected:
   struct TokenDesc {
     Token::Value token;
@@ -255,7 +251,12 @@ class ScannerBase {
   };
 
   virtual void Scan() = 0;
+
+  virtual void UpdateBufferBasedOnHandle() = 0;
   virtual bool FillLiteral(const TokenDesc& token, LiteralDesc* literal) = 0;
+  virtual Handle<String> InternalizeLiteral(LiteralDesc* literal) = 0;
+  virtual Handle<String> AllocateLiteral(LiteralDesc* literal,
+                                         PretenureFlag tenured) = 0;
 
   void ResetLiterals() {
     if (!current_literal_->is_in_buffer) current_literal_->beg_pos = -1;
@@ -274,11 +275,6 @@ class ScannerBase {
     }
   }
 
-  virtual Handle<String> InternalizeLiteral(LiteralDesc* literal) = 0;
-  virtual Handle<String> AllocateLiteral(LiteralDesc* literal,
-                                         PretenureFlag tenured) = 0;
-
-  Isolate* isolate_;
   UnicodeCache* unicode_cache_;
 
   bool has_line_terminator_before_next_;
@@ -296,83 +292,43 @@ class ScannerBase {
   bool harmony_numeric_literals_;
   bool harmony_modules_;
   bool harmony_scoping_;
+
+  friend class Scanner;
+  friend class LexerGCHandler;
 };
 
 
 template<typename Char>
-class ExperimentalScanner : public ScannerBase {
+class Lexer : public LexerBase {
  public:
-  explicit ExperimentalScanner(
-      Handle<String> source,
-      Isolate* isolate)
-      : ScannerBase(isolate),
-        source_handle_(source),
-        buffer_(NULL),
-        buffer_end_(NULL),
-        start_(NULL),
-        cursor_(NULL),
-        last_octal_end_(NULL) {
-    ASSERT(source->IsFlat());
-    UpdateBufferBasedOnHandle();
-    current_.beg_pos = current_.end_pos = next_.beg_pos = next_.end_pos = 0;
-  }
-
-  virtual void Init() {
-    Scan();
-  }
-
-  virtual ~ExperimentalScanner() { }
+  Lexer(UnicodeCache* unicode_cache,
+        Handle<String> source,
+        int start_position_,
+        int end_position_);
+  Lexer(UnicodeCache* unicode_cache, const Char* source_ptr, int length);
+  virtual ~Lexer();
 
   virtual void SeekForward(int pos);
   virtual void SetEnd(int pos);
   virtual bool ScanRegExpPattern(bool seen_equal);
   virtual bool ScanRegExpFlags();
   virtual Location octal_position() const;
-  virtual void clear_octal_position() {
-    last_octal_end_ = NULL;
-  }
-
-  virtual void UpdateBufferBasedOnHandle() {
-    // We get a raw pointer from the Handle, but we also update it every time
-    // there is a GC, so it is safe.
-    DisallowHeapAllocation no_gc;
-    const Char* new_buffer = GetNewBufferBasedOnHandle();
-    if (new_buffer != buffer_) {
-      int start_offset = start_ - buffer_;
-      int cursor_offset = cursor_ - buffer_;
-      int last_octal_end_offset = last_octal_end_ - buffer_;
-      buffer_ = new_buffer;
-      buffer_end_ = buffer_ + source_handle_->length();
-      start_ = buffer_ + start_offset;
-      cursor_ = buffer_ + cursor_offset;
-      if (last_octal_end_ != NULL) {
-        last_octal_end_ = buffer_ + last_octal_end_offset;
-      }
-      ResetLiterals();
-    }
-  }
+  virtual void clear_octal_position() { last_octal_end_ = NULL; }
 
  protected:
   virtual void Scan();
 
   const Char* GetNewBufferBasedOnHandle() const;
+  virtual void UpdateBufferBasedOnHandle();
 
   virtual bool FillLiteral(const TokenDesc& token, LiteralDesc* literal);
   virtual Handle<String> InternalizeLiteral(LiteralDesc* literal);
   virtual Handle<String> AllocateLiteral(LiteralDesc* literal,
                                          PretenureFlag tenured);
 
-
  private:
-  bool ValidIdentifierPart() {
-      return unicode_cache_->IsIdentifierPart(ScanHexNumber(4));
-  }
-
-  bool ValidIdentifierStart() {
-    return unicode_cache_->IsIdentifierStart(ScanHexNumber(4));
-  }
-
   uc32 ScanHexNumber(int length);
+
   bool ScanLiteralUnicodeEscape();
 
   const Char* ScanHexNumber(const Char* start,
@@ -397,7 +353,11 @@ class ExperimentalScanner : public ScannerBase {
                            const TokenDesc& token,
                            LiteralDesc* literal);
 
-  Handle<String> source_handle_;
+  Isolate* isolate_;
+  const Handle<String> source_handle_;
+  const Char* const source_ptr_;
+  const int start_position_;
+  const int end_position_;
   const Char* buffer_;
   const Char* buffer_end_;
   const Char* start_;
@@ -409,239 +369,146 @@ class ExperimentalScanner : public ScannerBase {
 };
 
 
-template<typename Char>
-void ExperimentalScanner<Char>::SeekForward(int pos) {
-  cursor_ = buffer_ + pos;
-  start_ = cursor_;
-  has_line_terminator_before_next_ = false;
-  has_multiline_comment_before_next_ = false;
-  Scan();  // Fills in next_.
-}
+#ifdef V8_USE_GENERATED_LEXER
 
 
-template<typename Char>
-void ExperimentalScanner<Char>::SetEnd(int pos) {
-  buffer_end_ = buffer_ + pos;
-}
+// Match old scanner interface.
+class Scanner {
+ public:
+  typedef LexerBase::Location Location;
 
+  explicit Scanner(UnicodeCache* unicode_cache);
 
-template<typename Char>
-bool ExperimentalScanner<Char>::ScanRegExpPattern(bool seen_equal) {
-  // Scan: ('/' | '/=') RegularExpressionBody '/' RegularExpressionFlags
-  bool in_character_class = false;
-
-  // Previous token is either '/' or '/=', in the second case, the
-  // pattern starts at =.
-  next_.beg_pos = next_.end_pos = (cursor_ - buffer_) - (seen_equal ? 1 : 0);
-
-  // Scan regular expression body: According to ECMA-262, 3rd, 7.8.5,
-  // the scanner should pass uninterpreted bodies to the RegExp
-  // constructor.
-  if (cursor_ >= buffer_end_) return false;
-
-  while (*cursor_ != '/' || in_character_class) {
-    if (unicode_cache_->IsLineTerminator(*cursor_)) return false;
-    if (*cursor_ == '\\') {  // Escape sequence.
-      ++cursor_;
-      if (cursor_ >= buffer_end_ || unicode_cache_->IsLineTerminator(*cursor_))
-        return false;
-      ++cursor_;
-      if (cursor_ >= buffer_end_) return false;
-      // If the escape allows more characters, i.e., \x??, \u????, or \c?,
-      // only "safe" characters are allowed (letters, digits, underscore),
-      // otherwise the escape isn't valid and the invalid character has
-      // its normal meaning. I.e., we can just continue scanning without
-      // worrying whether the following characters are part of the escape
-      // or not, since any '/', '\\' or '[' is guaranteed to not be part
-      // of the escape sequence.
-
-      // TODO(896): At some point, parse RegExps more throughly to capture
-      // octal esacpes in strict mode.
-    } else {  // Unescaped character.
-      if (*cursor_ == '[') in_character_class = true;
-      if (*cursor_ == ']') in_character_class = false;
-      if (++cursor_ >= buffer_end_) return false;
-    }
+  ~Scanner() { delete lexer_; }
+
+  void Initialize(Utf16CharacterStream* source);
+
+  inline void SeekForward(int pos) { lexer_->SeekForward(pos); }
+
+  inline void SetEnd(int pos) { lexer_->SetEnd(pos); }
+
+  inline bool ScanRegExpPattern(bool seen_equal) {
+    return lexer_->ScanRegExpPattern(seen_equal);
   }
-  next_.end_pos = (cursor_ - buffer_);
-  ++cursor_;  // consume '/'
-  return true;
-}
 
+  inline bool ScanRegExpFlags() { return lexer_->ScanRegExpFlags(); }
 
-template<typename Char>
-bool ExperimentalScanner<Char>::ScanRegExpFlags() {
-  next_.beg_pos = cursor_ - buffer_;
-  // Scan regular expression flags.
-  while (cursor_ < buffer_end_ && unicode_cache_->IsIdentifierPart(*cursor_)) {
-    if (*cursor_ != '\\') {
-      if (++cursor_ >= buffer_end_) break;
-    } else {
-      if (!ScanLiteralUnicodeEscape()) break;
-      if (++cursor_ >= buffer_end_) break;
-    }
+  inline Location octal_position() const { return lexer_->octal_position(); }
+
+  inline void clear_octal_position() { lexer_->clear_octal_position(); }
+
+  inline Token::Value Next() { return lexer_->Next(); }
+
+  inline Token::Value current_token() { return lexer_->current_token(); }
+
+  inline Location location() { return lexer_->location(); }
+
+  inline Token::Value peek() const { return lexer_->peek(); }
+
+  inline Location peek_location() const { return lexer_->peek_location(); }
+
+  inline UnicodeCache* unicode_cache() { return lexer_->unicode_cache(); }
+
+  inline bool HarmonyScoping() const {
+    return harmony_scoping_;
   }
-  next_.end_pos = cursor_ - buffer_;
-  return true;
-}
 
+  inline void SetHarmonyScoping(bool scoping) {
+    harmony_scoping_ = scoping;
+    SyncSettings();
+  }
 
-template<typename Char>
-uc32 ExperimentalScanner<Char>::ScanHexNumber(int length) {
-  // We have seen \uXXXX, let's see what it is.
-  uc32 x = 0;
-  for (const Char* s = cursor_ - length; s != cursor_; ++s) {
-    int d = HexValue(*s);
-    if (d < 0) {
-      return -1;
-    }
-    x = x * 16 + d;
+  inline bool HarmonyModules() const {
+    return harmony_modules_;
   }
-  return x;
-}
 
+  inline void SetHarmonyModules(bool modules) {
+    harmony_modules_ = modules;
+    SyncSettings();
+  }
 
-template<typename Char>
-const Char* ExperimentalScanner<Char>::ScanHexNumber(
-    const Char* cursor, const Char* end, uc32* result) {
-  uc32 x = 0;
-  for ( ; cursor < end; ++cursor) {
-    int d = HexValue(*cursor);
-    if (d < 0) {
-      *result = -1;
-      return NULL;
-    }
-    x = x * 16 + d;
+  inline bool HarmonyNumericLiterals() const {
+    return harmony_numeric_literals_;
   }
-  *result = x;
-  return cursor;
-}
 
+  inline void SetHarmonyNumericLiterals(bool numeric_literals) {
+    harmony_numeric_literals_ = numeric_literals;
+    SyncSettings();
+  }
 
-// Octal escapes of the forms '\0xx' and '\xxx' are not a part of
-// ECMA-262. Other JS VMs support them.
-template<typename Char>
-const Char* ExperimentalScanner<Char>::ScanOctalEscape(
-    const Char* start, const Char* end, uc32* result) {
-  uc32 x = *result - '0';
-  const Char* cursor;
-  for (cursor = start; cursor < end; cursor++) {
-    int d = *cursor - '0';
-    if (d < 0 || d > 7) break;
-    int nx = x * 8 + d;
-    if (nx >= 256) break;
-    x = nx;
-  }
-  *result = x;
-  return cursor;
-}
+  inline bool HasAnyLineTerminatorBeforeNext() const {
+    return lexer_->HasAnyLineTerminatorBeforeNext();
+  }
 
+  inline Handle<String> GetLiteralSymbol() {
+    return lexer_->GetLiteralSymbol();
+  }
 
-template<typename Char>
-bool ExperimentalScanner<Char>::ScanLiteralUnicodeEscape() {
-  ASSERT(cursor_ < buffer_end_);
-  Char primary_char = *(cursor_);
-  ASSERT(primary_char == '\\');
-  if (++cursor_ >= buffer_end_) return false;
-  primary_char = *(cursor_);
-  int i = 1;
-  if (primary_char == 'u') {
-    i++;
-    while (i < 6) {
-      if (++cursor_ >= buffer_end_) return false;
-      primary_char = *(cursor_);
-      if (!IsHexDigit(primary_char)) break;
-      i++;
-    }
+  inline Handle<String> GetLiteralString(PretenureFlag tenured) {
+    return lexer_->GetLiteralString(tenured);
   }
-  return i == 6;
-}
 
+  inline Handle<String> GetNextLiteralString(PretenureFlag tenured) {
+    return lexer_->GetNextLiteralString(tenured);
+  }
 
-template<typename Char>
-const Char* ExperimentalScanner<Char>::ScanIdentifierUnicodeEscape(
-    const Char* cursor, const Char* end, uc32* result) {
-  ASSERT(*cursor == '\\');
-  if (++cursor >= end) return NULL;
-  if (*cursor != 'u') return NULL;
-  ++cursor;
-  if (cursor + 4 > end) return NULL;
-  cursor = ScanHexNumber(cursor, cursor + 4, result);
-  return cursor;
-}
+  inline Vector<const char> literal_ascii_string() {
+    return lexer_->literal_ascii_string();
+  }
 
+  inline Vector<const uc16> literal_utf16_string() {
+    return lexer_->literal_utf16_string();
+  }
 
-template<typename Char>
-const Char* ExperimentalScanner<Char>::ScanEscape(
-    const Char* cursor, const Char* end, LiteralBuffer* literal) {
-  ASSERT(*cursor == '\\');
-  if (++cursor >= end) return NULL;
-  uc32 c = *cursor;
-  if (++cursor > end) return NULL;
-  // Skip escaped newlines.
-  if (unicode_cache_->IsLineTerminator(c)) {
-    uc32 peek = *cursor;
-    // Allow CR+LF newlines in multiline string literals.
-    if (IsCarriageReturn(c) && IsLineFeed(peek)) cursor++;
-    // Allow LF+CR newlines in multiline string literals.
-    if (IsLineFeed(c) && IsCarriageReturn(peek)) cursor++;
-    return cursor;
-  }
-
-  switch (c) {
-    case '\'':  // fall through
-    case '"' :  // fall through
-    case '\\': break;
-    case 'b' : c = '\b'; break;
-    case 'f' : c = '\f'; break;
-    case 'n' : c = '\n'; break;
-    case 'r' : c = '\r'; break;
-    case 't' : c = '\t'; break;
-    case 'u' : {
-      ASSERT(cursor + 4 <= end);
-      cursor = ScanHexNumber(cursor, cursor + 4, &c);
-      if (cursor == NULL) return NULL;
-      break;
-    }
-    case 'v' : c = '\v'; break;
-    case 'x' : {
-      ASSERT(cursor + 2 <= end);
-      cursor = ScanHexNumber(cursor, cursor + 2, &c);
-      if (cursor == NULL) return NULL;
-      break;
-    }
-    case '0' :  // fall through
-    case '1' :  // fall through
-    case '2' :  // fall through
-    case '3' :  // fall through
-    case '4' :  // fall through
-    case '5' :  // fall through
-    case '6' :  // fall through
-    case '7' :
-      if (end > cursor + 2) end = cursor + 2;
-      cursor = ScanOctalEscape(cursor, end, &c); break;
-  }
-
-  // According to ECMA-262, section 7.8.4, characters not covered by the
-  // above cases should be illegal, but they are commonly handled as
-  // non-escaped characters by JS VMs.
-  literal->AddChar(c);
-  return cursor;
-}
+  inline int literal_length() {
+    return lexer_->literal_length();
+  }
 
+  inline bool is_literal_ascii() {
+    return lexer_->is_literal_ascii();
+  }
+
+  inline bool is_literal_contextual_keyword(Vector<const char> keyword) {
+    return lexer_->is_literal_contextual_keyword(keyword);
+  }
+
+  inline bool literal_contains_escapes() const {
+    return lexer_->literal_contains_escapes();
+  }
+
+  inline Vector<const char> next_literal_ascii_string() {
+    return lexer_->next_literal_ascii_string();
+  }
+
+  inline Vector<const uc16> next_literal_utf16_string() {
+    return lexer_->next_literal_utf16_string();
+  }
+
+  inline int next_literal_length() {
+    return lexer_->next_literal_length();
+  }
+
+  inline bool is_next_literal_ascii() {
+    return lexer_->is_next_literal_ascii();
+  }
+
+  inline bool is_next_contextual_keyword(Vector<const char> keyword) {
+    return lexer_->is_next_contextual_keyword(keyword);
+  }
+
+ private:
+  void SyncSettings();
+
+  UnicodeCache* unicode_cache_;
+  LexerBase* lexer_;
+  bool harmony_numeric_literals_;
+  bool harmony_modules_;
+  bool harmony_scoping_;
+};
+
+
+#endif
 
-template<typename Char>
-ScannerBase::Location ExperimentalScanner<Char>::octal_position() const {
-  if (!last_octal_end_)
-    return Location::invalid();
-  // The last octal might be an octal escape or an octal number. Whichever it
-  // is, we'll find the start by just scanning back until we hit a non-octal
-  // character.
-  const Char* temp_cursor = last_octal_end_ - 1;
-  while (temp_cursor >= buffer_ && *temp_cursor >= '0' && *temp_cursor <= '7')
-    --temp_cursor;
-  return Location(temp_cursor - buffer_ + 1, last_octal_end_ - buffer_);
-}
 
 } }