Experimental parser: merge to r19637

src/parser.h

Index: src/parser.h
diff --git a/src/parser.h b/src/parser.h
index c0b280c4ba70db7027994606cac6417fe71e593e..4a2cb0290b4b20d97d12ea6204028784a6afe137 100644
--- a/src/parser.h
+++ b/src/parser.h
@@ -30,7 +30,6 @@
 
 #include "allocation.h"
 #include "ast.h"
-#include "lexer/experimental-scanner.h"
 #include "preparse-data-format.h"
 #include "preparse-data.h"
 #include "scopes.h"
@@ -104,7 +103,7 @@ class ScriptDataImpl : public ScriptData {
   int GetSymbolIdentifier();
   bool SanityCheck();
 
-  ScannerBase::Location MessageLocation();
+  Scanner::Location MessageLocation();
   const char* BuildMessage();
   Vector<const char*> BuildArgs();
 
@@ -154,7 +153,7 @@ class PreParserApi {
   // preparser recorder object that is suited to the parser's purposes.  Also,
   // the preparser doesn't know about ScriptDataImpl.
   static ScriptDataImpl* PreParse(Isolate* isolate,
-                                  Handle<String> source);
+                                  Utf16CharacterStream* source);
 };
 
 
@@ -405,17 +404,104 @@ class RegExpParser BASE_EMBEDDED {
 // ----------------------------------------------------------------------------
 // JAVASCRIPT PARSING
 
-// Forward declaration.
+class Parser;
 class SingletonLogger;
 
-class Parser : public ParserBase {
+class ParserTraits {
+ public:
+  struct Type {
+    typedef v8::internal::Parser* Parser;
+
+    // Types used by FunctionState and BlockState.
+    typedef v8::internal::Scope Scope;
+    typedef AstNodeFactory<AstConstructionVisitor> Factory;
+    typedef Variable GeneratorVariable;
+    typedef v8::internal::Zone Zone;
+
+    // Return types for traversing functions.
+    typedef Handle<String> Identifier;
+    typedef v8::internal::Expression* Expression;
+    typedef ZoneList<v8::internal::Expression*>* ExpressionList;
+  };
+
+  explicit ParserTraits(Parser* parser) : parser_(parser) {}
+
+  // Custom operations executed when FunctionStates are created and destructed.
+  template<typename FunctionState>
+  static void SetUpFunctionState(FunctionState* function_state, Zone* zone) {
+    Isolate* isolate = zone->isolate();
+    function_state->isolate_ = isolate;
+    function_state->saved_ast_node_id_ = isolate->ast_node_id();
+    isolate->set_ast_node_id(BailoutId::FirstUsable().ToInt());
+  }
+
+  template<typename FunctionState>
+  static void TearDownFunctionState(FunctionState* function_state) {
+    if (function_state->outer_function_state_ != NULL) {
+      function_state->isolate_->set_ast_node_id(
+          function_state->saved_ast_node_id_);
+    }
+  }
+
+  // Helper functions for recursive descent.
+  bool IsEvalOrArguments(Handle<String> identifier) const;
+
+  // Reporting errors.
+  void ReportMessageAt(Scanner::Location source_location,
+                       const char* message,
+                       Vector<const char*> args);
+  void ReportMessage(const char* message, Vector<Handle<String> > args);
+  void ReportMessageAt(Scanner::Location source_location,
+                       const char* message,
+                       Vector<Handle<String> > args);
+
+  // "null" return type creators.
+  static Handle<String> EmptyIdentifier() {
+    return Handle<String>();
+  }
+  static Expression* EmptyExpression() {
+    return NULL;
+  }
+
+  // Odd-ball literal creators.
+  Literal* GetLiteralTheHole(int position,
+                             AstNodeFactory<AstConstructionVisitor>* factory);
+
+  // Producing data during the recursive descent.
+  Handle<String> GetSymbol(Scanner* scanner = NULL);
+  Handle<String> NextLiteralString(Scanner* scanner,
+                                   PretenureFlag tenured);
+  Expression* ThisExpression(Scope* scope,
+                             AstNodeFactory<AstConstructionVisitor>* factory);
+  Expression* ExpressionFromLiteral(
+      Token::Value token, int pos, Scanner* scanner,
+      AstNodeFactory<AstConstructionVisitor>* factory);
+  Expression* ExpressionFromIdentifier(
+      Handle<String> name, int pos, Scope* scope,
+      AstNodeFactory<AstConstructionVisitor>* factory);
+  Expression* ExpressionFromString(
+      int pos, Scanner* scanner,
+      AstNodeFactory<AstConstructionVisitor>* factory);
+  ZoneList<v8::internal::Expression*>* NewExpressionList(int size, Zone* zone) {
+    return new(zone) ZoneList<v8::internal::Expression*>(size, zone);
+  }
+
+  // Temporary glue; these functions will move to ParserBase.
+  Expression* ParseObjectLiteral(bool* ok);
+  Expression* ParseAssignmentExpression(bool accept_IN, bool* ok);
+  Expression* ParseV8Intrinsic(bool* ok);
+
+ private:
+  Parser* parser_;
+};
+
+
+class Parser : public ParserBase<ParserTraits> {
  public:
   explicit Parser(CompilationInfo* info);
   ~Parser() {
     delete reusable_preparser_;
     reusable_preparser_ = NULL;
-    delete scanner_;
-    scanner_ = NULL;
   }
 
   // Parses the source code represented by the compilation info and sets its
@@ -430,6 +516,8 @@ class Parser : public ParserBase {
   bool Parse();
 
  private:
+  friend class ParserTraits;
+
   static const int kMaxNumFunctionLocals = 131071;  // 2^17-1
 
   enum Mode {
@@ -450,66 +538,6 @@ class Parser : public ParserBase {
     kHasNoInitializers
   };
 
-  class BlockState;
-
-  class FunctionState BASE_EMBEDDED {
-   public:
-    FunctionState(Parser* parser,
-                  Scope* scope,
-                  Isolate* isolate);
-    ~FunctionState();
-
-    int NextMaterializedLiteralIndex() {
-      return next_materialized_literal_index_++;
-    }
-    int materialized_literal_count() {
-      return next_materialized_literal_index_ - JSFunction::kLiteralsPrefixSize;
-    }
-
-    int NextHandlerIndex() { return next_handler_index_++; }
-    int handler_count() { return next_handler_index_; }
-
-    void AddProperty() { expected_property_count_++; }
-    int expected_property_count() { return expected_property_count_; }
-
-    void set_generator_object_variable(Variable *variable) {
-      ASSERT(variable != NULL);
-      ASSERT(!is_generator());
-      generator_object_variable_ = variable;
-    }
-    Variable* generator_object_variable() const {
-      return generator_object_variable_;
-    }
-    bool is_generator() const {
-      return generator_object_variable_ != NULL;
-    }
-
-    AstNodeFactory<AstConstructionVisitor>* factory() { return &factory_; }
-
-   private:
-    // Used to assign an index to each literal that needs materialization in
-    // the function.  Includes regexp literals, and boilerplate for object and
-    // array literals.
-    int next_materialized_literal_index_;
-
-    // Used to assign a per-function index to try and catch handlers.
-    int next_handler_index_;
-
-    // Properties count estimation.
-    int expected_property_count_;
-
-    // For generators, the variable that holds the generator object.  This
-    // variable is used by yield expressions and return statements.  NULL
-    // indicates that this function is not a generator.
-    Variable* generator_object_variable_;
-
-    Parser* parser_;
-    FunctionState* outer_function_state_;
-    Scope* outer_scope_;
-    int saved_ast_node_id_;
-    AstNodeFactory<AstConstructionVisitor> factory_;
-  };
-
   class ParsingModeScope BASE_EMBEDDED {
    public:
     ParsingModeScope(Parser* parser, Mode mode)
@@ -530,10 +558,9 @@ class Parser : public ParserBase {
   FunctionLiteral* ParseProgram();
 
   FunctionLiteral* ParseLazy();
-  FunctionLiteral* ParseLazy(Handle<String> source, int start, int end);
+  FunctionLiteral* ParseLazy(Utf16CharacterStream* source);
 
   Isolate* isolate() { return isolate_; }
-  Zone* zone() const { return zone_; }
   CompilationInfo* info() const { return info_; }
 
   // Called by ParseProgram after setting up the scanner.
@@ -541,41 +568,25 @@ class Parser : public ParserBase {
                                   Handle<String> source);
 
   // Report syntax error
-  void ReportUnexpectedToken(Token::Value token);
   void ReportInvalidPreparseData(Handle<String> name, bool* ok);
-  void ReportMessage(const char* message, Vector<const char*> args);
-  void ReportMessage(const char* message, Vector<Handle<String> > args);
-  void ReportMessageAt(ScannerBase::Location location, const char* type) {
-    ReportMessageAt(location, type, Vector<const char*>::empty());
-  }
-  void ReportMessageAt(ScannerBase::Location loc,
-                       const char* message,
-                       Vector<const char*> args);
-  void ReportMessageAt(ScannerBase::Location loc,
-                       const char* message,
-                       Vector<Handle<String> > args);
 
   void set_pre_parse_data(ScriptDataImpl *data) {
     pre_parse_data_ = data;
     symbol_cache_.Initialize(data ? data->symbol_count() : 0, zone());
   }
 
-  bool inside_with() const { return top_scope_->inside_with(); }
-  ScannerBase& scanner()  { return *scanner_; }
+  bool inside_with() const { return scope_->inside_with(); }
   Mode mode() const { return mode_; }
   ScriptDataImpl* pre_parse_data() const { return pre_parse_data_; }
   bool is_extended_mode() {
-    ASSERT(top_scope_ != NULL);
-    return top_scope_->is_extended_mode();
+    ASSERT(scope_ != NULL);
+    return scope_->is_extended_mode();
   }
   Scope* DeclarationScope(VariableMode mode) {
     return IsLexicalVariableMode(mode)
-        ? top_scope_ : top_scope_->DeclarationScope();
+        ? scope_ : scope_->DeclarationScope();
   }
 
-  // Check if the given string is 'eval' or 'arguments'.
-  bool IsEvalOrArguments(Handle<String> string);
-
   // All ParseXXX functions take as the last argument an *ok parameter
   // which is set to false if parsing failed; it is unchanged otherwise.
   // By making the 'exception handling' explicit, we are forced to check
@@ -625,7 +636,6 @@ class Parser : public ParserBase {
   // Support for hamony block scoped bindings.
   Block* ParseScopedBlock(ZoneStringList* labels, bool* ok);
 
-  Expression* ParseExpression(bool accept_IN, bool* ok);
   Expression* ParseAssignmentExpression(bool accept_IN, bool* ok);
   Expression* ParseYieldExpression(bool* ok);
   Expression* ParseConditionalExpression(bool accept_IN, bool* ok);
@@ -633,15 +643,11 @@ class Parser : public ParserBase {
   Expression* ParseUnaryExpression(bool* ok);
   Expression* ParsePostfixExpression(bool* ok);
   Expression* ParseLeftHandSideExpression(bool* ok);
-  Expression* ParseNewExpression(bool* ok);
+  Expression* ParseMemberWithNewPrefixesExpression(bool* ok);
   Expression* ParseMemberExpression(bool* ok);
-  Expression* ParseNewPrefix(PositionStack* stack, bool* ok);
-  Expression* ParseMemberWithNewPrefixesExpression(PositionStack* stack,
-                                                   bool* ok);
-  Expression* ParsePrimaryExpression(bool* ok);
-  Expression* ParseArrayLiteral(bool* ok);
+  Expression* ParseMemberExpressionContinuation(Expression* expression,
+                                                bool* ok);
   Expression* ParseObjectLiteral(bool* ok);
-  Expression* ParseRegExpLiteral(bool seen_equal, bool* ok);
 
   // Initialize the components of a for-in / for-of statement.
   void InitializeForEachStatement(ForEachStatement* stmt,
@@ -650,42 +656,32 @@ class Parser : public ParserBase {
                                   Statement* body);
 
   ZoneList<Expression*>* ParseArguments(bool* ok);
-  FunctionLiteral* ParseFunctionLiteral(Handle<String> var_name,
-                                        bool name_is_reserved,
-                                        bool is_generator,
-                                        int function_token_position,
-                                        FunctionLiteral::FunctionType type,
-                                        bool* ok);
-
+  FunctionLiteral* ParseFunctionLiteral(
+      Handle<String> name,
+      Scanner::Location function_name_location,
+      bool name_is_strict_reserved,
+      bool is_generator,
+      int function_token_position,
+      FunctionLiteral::FunctionType type,
+      bool* ok);
 
   // Magical syntax support.
   Expression* ParseV8Intrinsic(bool* ok);
 
-  bool is_generator() const { return current_function_state_->is_generator(); }
-
   bool CheckInOrOf(bool accept_OF, ForEachStatement::VisitMode* visit_mode);
 
   Handle<String> LiteralString(PretenureFlag tenured) {
-    return scanner().GetLiteralString(tenured);
-  }
-
-  Handle<String> NextLiteralString(PretenureFlag tenured) {
-    return scanner().GetNextLiteralString(tenured);
+    if (scanner()->is_literal_ascii()) {
+      return isolate_->factory()->NewStringFromAscii(
+          scanner()->literal_ascii_string(), tenured);
+    } else {
+      return isolate_->factory()->NewStringFromTwoByte(
+            scanner()->literal_utf16_string(), tenured);
+    }
   }
 
-  Handle<String> GetSymbol();
-
   // Get odd-ball literals.
   Literal* GetLiteralUndefined(int position);
-  Literal* GetLiteralTheHole(int position);
-
-  Handle<String> ParseIdentifier(bool* ok);
-  Handle<String> ParseIdentifierOrStrictReservedWord(
-      bool* is_strict_reserved, bool* ok);
-  Handle<String> ParseIdentifierName(bool* ok);
-  Handle<String> ParseIdentifierNameOrGetOrSet(bool* is_get,
-                                               bool* is_set,
-                                               bool* ok);
 
   // Determine if the expression is a variable proxy and mark it as being used
   // in an assignment or with a increment/decrement operator. This is currently
@@ -694,7 +690,6 @@ class Parser : public ParserBase {
 
   // Strict mode validation of LValue expressions
   void CheckStrictModeLValue(Expression* expression,
-                             const char* error,
                              bool* ok);
 
   // For harmony block scoping mode: Check if the scope has conflicting var/let
@@ -750,36 +745,20 @@ class Parser : public ParserBase {
   PreParser::PreParseResult LazyParseFunctionLiteral(
        SingletonLogger* logger);
 
-  AstNodeFactory<AstConstructionVisitor>* factory() {
-    return current_function_state_->factory();
-  }
-
-  void SetScannerFlags();
-
   Isolate* isolate_;
   ZoneList<Handle<String> > symbol_cache_;
 
   Handle<Script> script_;
+  Scanner scanner_;
   PreParser* reusable_preparser_;
-  Scope* top_scope_;
   Scope* original_scope_;  // for ES5 function declarations in sloppy eval
-  FunctionState* current_function_state_;
   Target* target_stack_;  // for break, continue statements
-  v8::Extension* extension_;
   ScriptDataImpl* pre_parse_data_;
   FuncNameInferrer* fni_;
 
   Mode mode_;
-  // If true, the next (and immediately following) function literal is
-  // preceded by a parenthesis.
-  // Heuristically that means that the function will be called immediately,
-  // so never lazily compile it.
-  bool parenthesized_function_;
 
-  Zone* zone_;
   CompilationInfo* info_;
-  friend class BlockState;
-  friend class FunctionState;
 };