Add parser support for generators.

src/preparser.h

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 99 matching lines @@
     kPreParseStackOverflow,
     kPreParseSuccess
   };
 
 
   PreParser(i::Scanner* scanner,
             i::ParserRecorder* log,
             uintptr_t stack_limit,
             bool allow_lazy,
             bool allow_natives_syntax,
-            bool allow_modules)
+            bool allow_modules,
+            bool allow_generators)
       : scanner_(scanner),
         log_(log),
         scope_(NULL),
         stack_limit_(stack_limit),
         strict_mode_violation_location_(i::Scanner::Location::invalid()),
         strict_mode_violation_type_(NULL),
         stack_overflow_(false),
         allow_lazy_(allow_lazy),
         allow_modules_(allow_modules),
         allow_natives_syntax_(allow_natives_syntax),
+        allow_generators_(allow_generators),
         parenthesized_function_(false),
         harmony_scoping_(scanner->HarmonyScoping()) { }
 
   ~PreParser() {}
 
   // Pre-parse the program from the character stream; returns true on
   // success (even if parsing failed, the pre-parse data successfully
   // captured the syntax error), and false if a stack-overflow happened
   // during parsing.
   static PreParseResult PreParseProgram(i::Scanner* scanner,
                                         i::ParserRecorder* log,
                                         int flags,
                                         uintptr_t stack_limit) {
     bool allow_lazy = (flags & i::kAllowLazy) != 0;
     bool allow_natives_syntax = (flags & i::kAllowNativesSyntax) != 0;
     bool allow_modules = (flags & i::kAllowModules) != 0;
+    bool allow_generators = (flags & i::kAllowGenerators) != 0;
     return PreParser(scanner, log, stack_limit, allow_lazy,
-                     allow_natives_syntax, allow_modules).PreParse();
+                     allow_natives_syntax, allow_modules,
+                     allow_generators).PreParse();
   }
 
   // Parses a single function literal, from the opening parentheses before
   // parameters to the closing brace after the body.
   // Returns a FunctionEntry describing the body of the function in enough
   // detail that it can be lazily compiled.
-  // The scanner is expected to have matched the "function" keyword and
-  // parameters, and have consumed the initial '{'.
+  // The scanner is expected to have matched the "function" or "function*"
+  // keyword and parameters, and have consumed the initial '{'.
   // At return, unless an error occurred, the scanner is positioned before the
   // the final '}'.
   PreParseResult PreParseLazyFunction(i::LanguageMode mode,
+                                      bool is_generator,
                                       i::ParserRecorder* log);
 
  private:
   // Used to detect duplicates in object literals. Each of the values
   // kGetterProperty, kSetterProperty and kValueProperty represents
   // a type of object literal property. When parsing a property, its
   // type value is stored in the DuplicateFinder for the property name.
   // Values are chosen so that having intersection bits means the there is
   // an incompatibility.
   // I.e., you can add a getter to a property that already has a setter, since
@@ skipping 63 matching lines @@
     }
     static Identifier Arguments()  {
       return Identifier(kArgumentsIdentifier);
     }
     static Identifier FutureReserved()  {
       return Identifier(kFutureReservedIdentifier);
     }
     static Identifier FutureStrictReserved()  {
       return Identifier(kFutureStrictReservedIdentifier);
     }
+    static Identifier Yield()  {
+      return Identifier(kYieldIdentifier);
+    }
     bool IsEval() { return type_ == kEvalIdentifier; }
     bool IsArguments() { return type_ == kArgumentsIdentifier; }
     bool IsEvalOrArguments() { return type_ >= kEvalIdentifier; }
+    bool IsYield() { return type_ == kYieldIdentifier; }
     bool IsFutureReserved() { return type_ == kFutureReservedIdentifier; }
     bool IsFutureStrictReserved() {
       return type_ == kFutureStrictReservedIdentifier;
     }
     bool IsValidStrictVariable() { return type_ == kUnknownIdentifier; }
 
    private:
     enum Type {
       kUnknownIdentifier,
       kFutureReservedIdentifier,
       kFutureStrictReservedIdentifier,
+      kYieldIdentifier,
       kEvalIdentifier,
       kArgumentsIdentifier
     };
     explicit Identifier(Type type) : type_(type) { }
     Type type_;
 
     friend class Expression;
   };
 
   // Bits 0 and 1 are used to identify the type of expression:
@@ skipping 73 matching lines @@
     bool IsStrictFunction() {
       return code_ == kStrictFunctionExpression;
     }
 
     Expression Parenthesize() {
       int type = code_ & 3;
       if (type != 0) {
         // Identifiers and string literals can be parenthesized.
         // They no longer work as labels or directive prologues,
         // but are still recognized in other contexts.
-        return Expression(code_ | kParentesizedExpressionFlag);
+        return Expression(code_ | kParenthesizedExpressionFlag);
       }
       // For other types of expressions, it's not important to remember
       // the parentheses.
       return *this;
     }
 
    private:
     // First two/three bits are used as flags.
     // Bit 0 and 1 represent identifiers or strings literals, and are
     // mutually exclusive, but can both be absent.
     // If bit 0 or 1 are set, bit 2 marks that the expression has
     // been wrapped in parentheses (a string literal can no longer
     // be a directive prologue, and an identifier can no longer be
     // a label.
     enum  {
       kUnknownExpression = 0,
       // Identifiers
       kIdentifierFlag = 1,  // Used to detect labels.
       kIdentifierShift = 3,
 
       kStringLiteralFlag = 2,  // Used to detect directive prologue.
       kUnknownStringLiteral = kStringLiteralFlag,
       kUseStrictString = kStringLiteralFlag | 8,
       kStringLiteralMask = kUseStrictString,
 
-      kParentesizedExpressionFlag = 4,  // Only if identifier or string literal.
+      // Only if identifier or string literal.
+      kParenthesizedExpressionFlag = 4,
 
       // Below here applies if neither identifier nor string literal.
       kThisExpression = 4,
       kThisPropertyExpression = 8,
       kStrictFunctionExpression = 12
     };
 
     explicit Expression(int expression_code) : code_(expression_code) { }
 
     int code_;
@@ skipping 57 matching lines @@
   class Scope {
    public:
     Scope(Scope** variable, ScopeType type)
         : variable_(variable),
           prev_(*variable),
           type_(type),
           materialized_literal_count_(0),
           expected_properties_(0),
           with_nesting_count_(0),
           language_mode_(
-              (prev_ != NULL) ? prev_->language_mode() : i::CLASSIC_MODE) {
+              (prev_ != NULL) ? prev_->language_mode() : i::CLASSIC_MODE),
+          is_generator_(false) {
       *variable = this;
     }
     ~Scope() { *variable_ = prev_; }
     void NextMaterializedLiteralIndex() { materialized_literal_count_++; }
     void AddProperty() { expected_properties_++; }
     ScopeType type() { return type_; }
     int expected_properties() { return expected_properties_; }
     int materialized_literal_count() { return materialized_literal_count_; }
     bool IsInsideWith() { return with_nesting_count_ != 0; }
+    bool is_generator() { return is_generator_; }
+    void set_is_generator(bool is_generator) { is_generator_ = is_generator; }
     bool is_classic_mode() {
       return language_mode_ == i::CLASSIC_MODE;
     }
     i::LanguageMode language_mode() {
       return language_mode_;
     }
     void set_language_mode(i::LanguageMode language_mode) {
       language_mode_ = language_mode;
     }
 
@@ skipping 11 matching lines @@
     };
 
    private:
     Scope** const variable_;
     Scope* const prev_;
     const ScopeType type_;
     int materialized_literal_count_;
     int expected_properties_;
     int with_nesting_count_;
     i::LanguageMode language_mode_;
+    bool is_generator_;
   };
 
   // Preparse the program. Only called in PreParseProgram after creating
   // the instance.
   PreParseResult PreParse() {
     Scope top_scope(&scope_, kTopLevelScope);
     bool ok = true;
     int start_position = scanner_->peek_location().beg_pos;
     ParseSourceElements(i::Token::EOS, &ok);
     if (stack_overflow_) return kPreParseStackOverflow;
@@ skipping 45 matching lines @@
   Statement ParseSwitchStatement(bool* ok);
   Statement ParseDoWhileStatement(bool* ok);
   Statement ParseWhileStatement(bool* ok);
   Statement ParseForStatement(bool* ok);
   Statement ParseThrowStatement(bool* ok);
   Statement ParseTryStatement(bool* ok);
   Statement ParseDebuggerStatement(bool* ok);
 
   Expression ParseExpression(bool accept_IN, bool* ok);
   Expression ParseAssignmentExpression(bool accept_IN, bool* ok);
+  Expression ParseYieldExpression(bool* ok);
   Expression ParseConditionalExpression(bool accept_IN, bool* ok);
   Expression ParseBinaryExpression(int prec, bool accept_IN, bool* ok);
   Expression ParseUnaryExpression(bool* ok);
   Expression ParsePostfixExpression(bool* ok);
   Expression ParseLeftHandSideExpression(bool* ok);
   Expression ParseNewExpression(bool* ok);
   Expression ParseMemberExpression(bool* ok);
   Expression ParseMemberWithNewPrefixesExpression(unsigned new_count, bool* ok);
   Expression ParsePrimaryExpression(bool* ok);
   Expression ParseArrayLiteral(bool* ok);
   Expression ParseObjectLiteral(bool* ok);
   Expression ParseRegExpLiteral(bool seen_equal, bool* ok);
   Expression ParseV8Intrinsic(bool* ok);
 
   Arguments ParseArguments(bool* ok);
-  Expression ParseFunctionLiteral(bool* ok);
+  Expression ParseFunctionLiteral(bool is_generator, bool* ok);
   void ParseLazyFunctionLiteralBody(bool* ok);
 
   Identifier ParseIdentifier(bool* ok);
   Identifier ParseIdentifierName(bool* ok);
   Identifier ParseIdentifierNameOrGetOrSet(bool* is_get,
                                            bool* is_set,
                                            bool* ok);
 
   // Logs the currently parsed literal as a symbol in the preparser data.
   void LogSymbol();
@@ skipping 71 matching lines @@
   i::Scanner* scanner_;
   i::ParserRecorder* log_;
   Scope* scope_;
   uintptr_t stack_limit_;
   i::Scanner::Location strict_mode_violation_location_;
   const char* strict_mode_violation_type_;
   bool stack_overflow_;
   bool allow_lazy_;
   bool allow_modules_;
   bool allow_natives_syntax_;
+  bool allow_generators_;
   bool parenthesized_function_;
   bool harmony_scoping_;
 };
 } }  // v8::preparser
 
 #endif  // V8_PREPARSER_H