Add parser support for generators.

src/parser.cc

Index: src/parser.cc
diff --git a/src/parser.cc b/src/parser.cc
index 17d5a461de2a4ee3f6c19d34342e84494cb44fce..a4d55490b68b4b9063335a806685f54b7f4b2675 100644
--- a/src/parser.cc
+++ b/src/parser.cc
@@ -680,7 +680,8 @@ FunctionLiteral* Parser::DoParseProgram(CompilationInfo* info,
           FunctionLiteral::kNoDuplicateParameters,
           FunctionLiteral::ANONYMOUS_EXPRESSION,
           FunctionLiteral::kGlobalOrEval,
-          FunctionLiteral::kNotParenthesized);
+          FunctionLiteral::kNotParenthesized,
+          FunctionLiteral::kNotGenerator);
       result->set_ast_properties(factory()->visitor()->ast_properties());
     } else if (stack_overflow_) {
       isolate()->StackOverflow();
@@ -768,6 +769,7 @@ FunctionLiteral* Parser::ParseLazy(Utf16CharacterStream* source,
     bool ok = true;
     result = ParseFunctionLiteral(name,
                                   false,  // Strict mode name already checked.
+                                  shared_info->is_generator(),
                                   RelocInfo::kNoPosition,
                                   type,
                                   &ok);
@@ -1132,6 +1134,7 @@ Statement* Parser::ParseModuleElement(ZoneStringList* labels,
   //    ModuleDeclaration
   //    ImportDeclaration
   //    ExportDeclaration
+  //    GeneratorDeclaration
 
   switch (peek()) {
     case Token::FUNCTION:
@@ -1430,6 +1433,7 @@ Statement* Parser::ParseExportDeclaration(bool* ok) {
   //    'export' Identifier (',' Identifier)* ';'
   //    'export' VariableDeclaration
   //    'export' FunctionDeclaration
+  //    'export' GeneratorDeclaration
   //    'export' ModuleDeclaration
   //
   // TODO(ES6): implement structuring ExportSpecifiers
@@ -1509,6 +1513,7 @@ Statement* Parser::ParseBlockElement(ZoneStringList* labels,
   // BlockElement (aka SourceElement):
   //    LetDeclaration
   //    ConstDeclaration
+  //    GeneratorDeclaration
 
   switch (peek()) {
     case Token::FUNCTION:
@@ -1628,6 +1633,10 @@ Statement* Parser::ParseStatement(ZoneStringList* labels, bool* ok) {
       //    FunctionDeclaration
       // Common language extension is to allow function declaration in place
       // of any statement. This language extension is disabled in strict mode.
+      //
+      // In Harmony mode, this case also handles the extension:
+      // Statement:
+      //    GeneratorDeclaration
       if (!top_scope_->is_classic_mode()) {
         ReportMessageAt(scanner().peek_location(), "strict_function",
                         Vector<const char*>::empty());
@@ -1892,11 +1901,13 @@ Statement* Parser::ParseFunctionDeclaration(ZoneStringList* names, bool* ok) {
   //   'function' Identifier '(' FormalParameterListopt ')' '{' FunctionBody '}'

[Michael Starzinger, 2013/03/14 22:29:24]

Could we add a comment like the following here ...

// GeneratorDeclaration ::
//   'function' '*' Identifier '(' FormalParameter ')' '{' FunctionBody '}'

   Expect(Token::FUNCTION, CHECK_OK);
   int function_token_position = scanner().location().beg_pos;
+  bool is_generator = FLAG_harmony_generators && Check(Token::MUL);
   bool is_strict_reserved = false;
   Handle<String> name = ParseIdentifierOrStrictReservedWord(
       &is_strict_reserved, CHECK_OK);
   FunctionLiteral* fun = ParseFunctionLiteral(name,
                                               is_strict_reserved,
+                                              is_generator,
                                               function_token_position,
                                               FunctionLiteral::DECLARATION,
                                               CHECK_OK);
@@ -3004,8 +3015,13 @@ Expression* Parser::ParseExpression(bool accept_IN, bool* ok) {
 Expression* Parser::ParseAssignmentExpression(bool accept_IN, bool* ok) {
   // AssignmentExpression ::
   //   ConditionalExpression
+  //   YieldExpression
   //   LeftHandSideExpression AssignmentOperator AssignmentExpression
 
+  if (peek() == Token::YIELD && inside_generator()) {
+    return ParseYieldExpression(ok);
+  }
+
   if (fni_ != NULL) fni_->Enter();
   Expression* expression = ParseConditionalExpression(accept_IN, CHECK_OK);
 
@@ -3074,6 +3090,17 @@ Expression* Parser::ParseAssignmentExpression(bool accept_IN, bool* ok) {
 }
 
 
+Expression* Parser::ParseYieldExpression(bool* ok) {
+  // YieldExpression ::
+  //   'yield' '*'? AssignmentExpression
+  int position = scanner().peek_location().beg_pos;
+  Expect(Token::YIELD, CHECK_OK);
+  bool is_yield_star = Check(Token::MUL);
+  Expression* expression = ParseAssignmentExpression(false, CHECK_OK);
+  return factory()->NewYield(expression, is_yield_star, position);
+}
+
+
 // Precedence = 3
 Expression* Parser::ParseConditionalExpression(bool accept_IN, bool* ok) {
   // ConditionalExpression ::
@@ -3450,6 +3477,7 @@ Expression* Parser::ParseMemberWithNewPrefixesExpression(PositionStack* stack,
   if (peek() == Token::FUNCTION) {
     Expect(Token::FUNCTION, CHECK_OK);
     int function_token_position = scanner().location().beg_pos;
+    bool is_generator = FLAG_harmony_generators && Check(Token::MUL);
     Handle<String> name;
     bool is_strict_reserved_name = false;
     if (peek_any_identifier()) {
@@ -3461,6 +3489,7 @@ Expression* Parser::ParseMemberWithNewPrefixesExpression(PositionStack* stack,
         : FunctionLiteral::NAMED_EXPRESSION;
     result = ParseFunctionLiteral(name,
                                   is_strict_reserved_name,
+                                  is_generator,
                                   function_token_position,
                                   type,
                                   CHECK_OK);
@@ -3604,6 +3633,7 @@ Expression* Parser::ParsePrimaryExpression(bool* ok) {
       break;
 
     case Token::IDENTIFIER:
+    case Token::YIELD:
     case Token::FUTURE_STRICT_RESERVED_WORD: {
       Handle<String> name = ParseIdentifier(CHECK_OK);
       if (fni_ != NULL) fni_->PushVariableName(name);
@@ -4009,6 +4039,7 @@ ObjectLiteral::Property* Parser::ParseObjectLiteralGetSet(bool is_getter,
     FunctionLiteral* value =
         ParseFunctionLiteral(name,
                              false,   // reserved words are allowed here
+                             false,   // not a generator
                              RelocInfo::kNoPosition,
                              FunctionLiteral::ANONYMOUS_EXPRESSION,
                              CHECK_OK);
@@ -4310,6 +4341,7 @@ class SingletonLogger : public ParserRecorder {
 
 FunctionLiteral* Parser::ParseFunctionLiteral(Handle<String> function_name,
                                               bool name_is_strict_reserved,
+                                              bool is_generator,
                                               int function_token_position,
                                               FunctionLiteral::Type type,
                                               bool* ok) {
@@ -4333,6 +4365,7 @@ FunctionLiteral* Parser::ParseFunctionLiteral(Handle<String> function_name,
   Scope* scope = (type == FunctionLiteral::DECLARATION && !is_extended_mode())
       ? NewScope(top_scope_->DeclarationScope(), FUNCTION_SCOPE)
       : NewScope(top_scope_, FUNCTION_SCOPE);
+  scope->set_inside_generator(is_generator);

[Michael Starzinger, 2013/03/14 22:48:23]

Actually, the more I think about it, is it really necessary that this
information is carried along with the scope? Wouldn't it be sufficient to store
this predicate as part of the FunctionState?

That way we wouldn't need to deserialize that information and scope
initialization would be unaffected. The predicate would be available via
current_function_state_ and I don't see the need to carry it along to other
units of compilation through the context chain. WDYT?

   ZoneList<Statement*>* body = NULL;
   int materialized_literal_count = -1;
   int expected_property_count = -1;
@@ -4344,6 +4377,9 @@ FunctionLiteral* Parser::ParseFunctionLiteral(Handle<String> function_name,
   FunctionLiteral::IsParenthesizedFlag parenthesized = parenthesized_function_
       ? FunctionLiteral::kIsParenthesized
       : FunctionLiteral::kNotParenthesized;
+  FunctionLiteral::IsGeneratorFlag generator = is_generator
+      ? FunctionLiteral::kIsGenerator
+      : FunctionLiteral::kNotGenerator;
   AstProperties ast_properties;
   // Parse function body.
   { FunctionState function_state(this, scope, isolate());
@@ -4584,7 +4620,8 @@ FunctionLiteral* Parser::ParseFunctionLiteral(Handle<String> function_name,
                                     duplicate_parameters,
                                     type,
                                     FunctionLiteral::kIsFunction,
-                                    parenthesized);
+                                    parenthesized,
+                                    generator);
   function_literal->set_function_token_position(function_token_position);
   function_literal->set_ast_properties(&ast_properties);
 
@@ -4610,6 +4647,7 @@ preparser::PreParser::PreParseResult Parser::LazyParseFunctionLiteral(
   }
   preparser::PreParser::PreParseResult result =
       reusable_preparser_->PreParseLazyFunction(top_scope_->language_mode(),
+                                                top_scope_->inside_generator(),
                                                 logger);
   return result;
 }
@@ -4672,7 +4710,8 @@ bool Parser::peek_any_identifier() {
   Token::Value next = peek();
   return next == Token::IDENTIFIER ||
          next == Token::FUTURE_RESERVED_WORD ||
-         next == Token::FUTURE_STRICT_RESERVED_WORD;
+         next == Token::FUTURE_STRICT_RESERVED_WORD ||
+         next == Token::YIELD;
 }
 
 
@@ -4744,13 +4783,17 @@ Literal* Parser::GetLiteralTheHole() {
 // Parses an identifier that is valid for the current scope, in particular it
 // fails on strict mode future reserved keywords in a strict scope.
 Handle<String> Parser::ParseIdentifier(bool* ok) {
-  if (!top_scope_->is_classic_mode()) {
-    Expect(Token::IDENTIFIER, ok);
-  } else if (!Check(Token::IDENTIFIER)) {
-    Expect(Token::FUTURE_STRICT_RESERVED_WORD, ok);
+  Token::Value next = Next();
+  if (next == Token::IDENTIFIER ||
+      (top_scope_->is_classic_mode() &&
+       (next == Token::FUTURE_STRICT_RESERVED_WORD ||
+        (next == Token::YIELD && !inside_generator())))) {

[Michael Starzinger, 2013/03/14 22:29:24]

I think it would be clearer here if we would call the is_generator() predicate
directly on top_scope_ instead of using a helper. That makes it explicit that
this information is carried along with the scope.

+    return GetSymbol(ok);
+  } else {
+    ReportUnexpectedToken(next);
+    *ok = false;
+    return Handle<String>();
   }
-  if (!*ok) return Handle<String>();
-  return GetSymbol(ok);
 }
 
 
@@ -4758,12 +4801,17 @@ Handle<String> Parser::ParseIdentifier(bool* ok) {
 // whether it is strict mode future reserved.
 Handle<String> Parser::ParseIdentifierOrStrictReservedWord(
     bool* is_strict_reserved, bool* ok) {
-  *is_strict_reserved = false;
-  if (!Check(Token::IDENTIFIER)) {
-    Expect(Token::FUTURE_STRICT_RESERVED_WORD, ok);
+  Token::Value next = Next();
+  if (next == Token::IDENTIFIER) {
+    *is_strict_reserved = false;
+  } else if (next == Token::FUTURE_STRICT_RESERVED_WORD ||
+             (next == Token::YIELD && !inside_generator())) {

[Michael Starzinger, 2013/03/14 22:29:24]

Likewise.

     *is_strict_reserved = true;
+  } else {
+    ReportUnexpectedToken(next);
+    *ok = false;
+    return Handle<String>();
   }
-  if (!*ok) return Handle<String>();
   return GetSymbol(ok);
 }