[parser] Refactor of Parse*Statement*, part 1

src/parsing/parser-base.h

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef V8_PARSING_PARSER_BASE_H
 #define V8_PARSING_PARSER_BASE_H
 
 #include "src/ast/scopes.h"
 #include "src/bailout-reason.h"
 #include "src/base/hashmap.h"
@@ skipping 168 matching lines @@
 //   // not related to pure parsing.
 //   typedef GeneratorVariable;
 //   // Return types for traversing functions.
 //   typedef Identifier;
 //   typedef Expression;
 //   typedef FunctionLiteral;
 //   typedef ObjectLiteralProperty;
 //   typedef ExpressionList;
 //   typedef PropertyList;
 //   typedef FormalParameters;
+//   typedef Statement;
 //   typedef StatementList;
 //   typedef Block;
 //   // For constructing objects returned by the traversing functions.
 //   typedef Factory;
+//   // For other implementation-specific tasks.
+//   typedef Target;
+//   typedef TargetScope;
 // };
 
 template <typename Impl>
 struct ParserTypes;
 
 template <typename Impl>
 class ParserBase {
  public:
   // Shorten type names defined by ParserTypes<Impl>.
   typedef ParserTypes<Impl> Types;
   typedef typename Types::Identifier IdentifierT;
   typedef typename Types::Expression ExpressionT;
   typedef typename Types::FunctionLiteral FunctionLiteralT;
   typedef typename Types::ObjectLiteralProperty ObjectLiteralPropertyT;
   typedef typename Types::ExpressionList ExpressionListT;
   typedef typename Types::PropertyList PropertyListT;
   typedef typename Types::FormalParameters FormalParametersT;
+  typedef typename Types::Statement StatementT;
   typedef typename Types::StatementList StatementListT;
   typedef typename Types::Block BlockT;
   typedef typename v8::internal::ExpressionClassifier<Types>
       ExpressionClassifier;
 
   // All implementation-specific methods must be called through this.
   Impl* impl() { return static_cast<Impl*>(this); }
   const Impl* impl() const { return static_cast<const Impl*>(this); }
 
   ParserBase(Zone* zone, Scanner* scanner, uintptr_t stack_limit,
@@ skipping 689 matching lines @@
       return 0;  // 0 precedence will terminate binary expression parsing
     return Token::Precedence(token);
   }
 
   typename Types::Factory* factory() { return &ast_node_factory_; }
 
   DeclarationScope* GetReceiverScope() const {
     return scope()->GetReceiverScope();
   }
   LanguageMode language_mode() { return scope()->language_mode(); }
+  void RaiseLanguageMode(LanguageMode mode) {
+    LanguageMode old = scope()->language_mode();
+    impl()->SetLanguageMode(scope(), old > mode ? old : mode);
+  }
   bool is_generator() const { return function_state_->is_generator(); }
   bool is_async_function() const {
     return function_state_->is_async_function();
   }
   bool is_resumable() const { return function_state_->is_resumable(); }
 
   // Report syntax errors.
   void ReportMessage(MessageTemplate::Template message, const char* arg = NULL,
                      ParseErrorType error_type = kSyntaxError) {
     Scanner::Location source_location = scanner()->location();
@@ skipping 243 matching lines @@
   void ParseFormalParameterList(FormalParametersT* parameters, bool* ok);
   void CheckArityRestrictions(int param_count, FunctionKind function_type,
                               bool has_rest, int formals_start_pos,
                               int formals_end_pos, bool* ok);
 
   BlockT ParseVariableDeclarations(VariableDeclarationContext var_context,
                                    DeclarationParsingResult* parsing_result,
                                    ZoneList<const AstRawString*>* names,
                                    bool* ok);
 
+  static const int kLazyParseTrialLimit = 200;

[marja, 2016/09/05 08:14:33]

This const declaration could use a short comment here...

[nickie, 2016/09/05 11:29:31]

Done.  It never had one, so I improvised.

+  V8_INLINE void ParseStatementList(StatementListT body, int end_token,
+                                    bool* ok) {
+    LazyParsingResult result = ParseStatementList(body, end_token, false, ok);
+    USE(result);  // The result is just used in debug modes.

[marja, 2016/09/05 08:14:33]

... this comment, on the other hand, is not very useful :)

[nickie, 2016/09/05 11:29:31]

Done.  Removed.

+    DCHECK_EQ(result, kLazyParsingComplete);
+  }
+  LazyParsingResult ParseStatementList(StatementListT body, int end_token,
+                                       bool may_abort, bool* ok);
+  StatementT ParseStatementListItem(bool* ok);
+  StatementT ParseStatement(ZoneList<const AstRawString*>* labels,
+                            AllowLabelledFunctionStatement allow_function,
+                            bool* ok);
+  StatementT ParseStatementAsUnlabelled(ZoneList<const AstRawString*>* labels,
+                                        bool* ok);
+
   bool IsNextLetKeyword();
   bool IsTrivialExpression();
 
   // Checks if the expression is a valid reference expression (e.g., on the
   // left-hand side of assignments). Although ruled out by ECMA as early errors,
   // we allow calls for web compatibility and rewrite them to a runtime throw.
   ExpressionT CheckAndRewriteReferenceExpression(
       ExpressionT expression, int beg_pos, int end_pos,
       MessageTemplate::Template message, bool* ok);
   ExpressionT CheckAndRewriteReferenceExpression(
@@ skipping 2714 matching lines @@
 void ParserBase<Impl>::CheckDestructuringElement(ExpressionT expression,
                                                  int begin, int end) {
   if (!IsValidPattern(expression) && !expression->IsAssignment() &&
       !IsValidReferenceExpression(expression)) {
     classifier()->RecordAssignmentPatternError(
         Scanner::Location(begin, end),
         MessageTemplate::kInvalidDestructuringTarget);
   }
 }
 
+// Redefinition of CHECK_OK for parsing statements.
+#undef CHECK_OK
+#define CHECK_OK CHECK_OK_CUSTOM(NullStatement)
+
+template <typename Impl>
+typename ParserBase<Impl>::LazyParsingResult
+ParserBase<Impl>::ParseStatementList(StatementListT body, int end_token,
+                                     bool may_abort, bool* ok) {
+  // StatementList ::
+  //   (StatementListItem)* <end_token>
+
+  // Allocate a target stack to use for this set of source
+  // elements. This way, all scripts and functions get their own
+  // target stack thus avoiding illegal breaks and continues across
+  // functions.
+  typename Types::TargetScope target_scope(this);
+  int count_statements = 0;
+
+  DCHECK(!impl()->IsNullStatementList(body));
+  bool directive_prologue = true;  // Parsing directive prologue.
+
+  while (peek() != end_token) {
+    if (directive_prologue && peek() != Token::STRING) {
+      directive_prologue = false;
+    }
+
+    bool starts_with_identifier = peek() == Token::IDENTIFIER;
+    Scanner::Location token_loc = scanner()->peek_location();
+    StatementT stat = impl()->ParseStatementListItem(
+        CHECK_OK_CUSTOM(Return, kLazyParsingComplete));
+
+    if (impl()->IsNullOrEmptyStatement(stat)) {
+      directive_prologue = false;  // End of directive prologue.
+      continue;
+    }
+
+    if (directive_prologue) {
+      // The length of the token is used to distinguish between strings literals
+      // that evaluate equal to directives but contain either escape sequences
+      // (e.g., "use \x73trict") or line continuations (e.g., "use \
+      // strict").
+      if (impl()->IsUseStrictDirective(stat) &&
+          token_loc.end_pos - token_loc.beg_pos == sizeof("use strict") + 1) {
+        // Directive "use strict" (ES5 14.1).
+        RaiseLanguageMode(STRICT);
+        if (!scope()->HasSimpleParameters()) {
+          // TC39 deemed "use strict" directives to be an error when occurring
+          // in the body of a function with non-simple parameter list, on
+          // 29/7/2015. https://goo.gl/ueA7Ln
+          impl()->ReportMessageAt(
+              token_loc, MessageTemplate::kIllegalLanguageModeDirective,
+              "use strict");
+          *ok = false;
+          return kLazyParsingComplete;
+        }
+        // Because declarations in strict eval code don't leak into the scope
+        // of the eval call, it is likely that functions declared in strict
+        // eval code will be used within the eval code, so lazy parsing is
+        // probably not a win.
+        if (scope()->is_eval_scope()) mode_ = PARSE_EAGERLY;
+      } else if (impl()->IsUseAsmDirective(stat) &&
+                 token_loc.end_pos - token_loc.beg_pos ==
+                     sizeof("use asm") + 1) {
+        // Directive "use asm".
+        impl()->SetAsmModule();
+      } else if (impl()->IsStringLiteral(stat)) {
+        // Possibly an unknown directive.
+        // TODO(nikolaos): Check if the following is really what we want!
+        // """Should not change mode, but will increment UseCounter
+        // if appropriate. Ditto usages below."""  ???
+        RaiseLanguageMode(SLOPPY);
+      } else {

[nickie, 2016/09/05 11:29:31]

These were copied from here [1] and history takes us back to littledan (7ff114e)
[2].  @Dan, can you check please if these make sense?  (Both the calls to
RaiseLanguageMode and the comments, here and a few lines below).

[1] https://cs.chromium.org/chromium/src/v8/src/parsing/parser.cc?rcl=0&l=1006
[2] https://codereview.chromium.org/1429173002

+        // End of the directive prologue.
+        directive_prologue = false;
+        // TODO(nikolaos): Check if the following is really what we want!
+        RaiseLanguageMode(SLOPPY);
+      }
+    } else {
+      // TODO(nikolaos): Check if the following is really what we want!
+      RaiseLanguageMode(SLOPPY);
+    }
+
+    // If we're allowed to reset to a bookmark, we will do so when we see a long
+    // and trivial function.
+    // Our current definition of 'long and trivial' is:
+    // - over 200 statements
+    // - all starting with an identifier (i.e., no if, for, while, etc.)
+    if (may_abort) {
+      if (!starts_with_identifier) {
+        may_abort = false;
+      } else if (++count_statements > kLazyParseTrialLimit) {
+        return kLazyParsingAborted;
+      }
+    }
+
+    body->Add(stat, zone());
+  }
+  return kLazyParsingComplete;
+}
+
+template <typename Impl>
+typename ParserBase<Impl>::StatementT ParserBase<Impl>::ParseStatementListItem(
+    bool* ok) {
+  // ECMA 262 6th Edition
+  // StatementListItem[Yield, Return] :
+  //   Statement[?Yield, ?Return]
+  //   Declaration[?Yield]
+  //
+  // Declaration[Yield] :
+  //   HoistableDeclaration[?Yield]
+  //   ClassDeclaration[?Yield]
+  //   LexicalDeclaration[In, ?Yield]
+  //
+  // HoistableDeclaration[Yield, Default] :
+  //   FunctionDeclaration[?Yield, ?Default]
+  //   GeneratorDeclaration[?Yield, ?Default]
+  //
+  // LexicalDeclaration[In, Yield] :
+  //   LetOrConst BindingList[?In, ?Yield] ;
+
+  switch (peek()) {
+    case Token::FUNCTION:
+      return impl()->ParseHoistableDeclaration(NULL, false, ok);

[marja, 2016/09/05 08:14:33]

Nit: NULL -> nullptr in several places

[nickie, 2016/09/05 11:29:31]

Done.

+    case Token::CLASS:
+      Consume(Token::CLASS);
+      return impl()->ParseClassDeclaration(NULL, false, ok);
+    case Token::VAR:
+    case Token::CONST:
+      return impl()->ParseVariableStatement(kStatementListItem, NULL, ok);
+    case Token::LET:
+      if (IsNextLetKeyword()) {
+        return impl()->ParseVariableStatement(kStatementListItem, NULL, ok);
+      }
+      break;
+    case Token::ASYNC:
+      if (allow_harmony_async_await() && PeekAhead() == Token::FUNCTION &&
+          !scanner()->HasAnyLineTerminatorAfterNext()) {
+        Consume(Token::ASYNC);
+        return impl()->ParseAsyncFunctionDeclaration(NULL, false, ok);
+      }
+    /* falls through */
+    default:
+      break;
+  }
+  return impl()->ParseStatement(NULL, kAllowLabelledFunctionStatement, ok);
+}
+
+template <typename Impl>
+typename ParserBase<Impl>::StatementT ParserBase<Impl>::ParseStatement(
+    ZoneList<const AstRawString*>* labels,
+    AllowLabelledFunctionStatement allow_function, bool* ok) {
+  // Statement ::
+  //   Block
+  //   VariableStatement
+  //   EmptyStatement
+  //   ExpressionStatement
+  //   IfStatement
+  //   IterationStatement
+  //   ContinueStatement
+  //   BreakStatement
+  //   ReturnStatement
+  //   WithStatement
+  //   LabelledStatement
+  //   SwitchStatement
+  //   ThrowStatement
+  //   TryStatement
+  //   DebuggerStatement
+
+  // Note: Since labels can only be used by 'break' and 'continue'
+  // statements, which themselves are only valid within blocks,
+  // iterations or 'switch' statements (i.e., BreakableStatements),
+  // labels can be simply ignored in all other cases; except for
+  // trivial labeled break statements 'label: break label' which is
+  // parsed into an empty statement.
+  switch (peek()) {
+    case Token::LBRACE:
+      return impl()->ParseBlock(labels, ok);
+    case Token::SEMICOLON:
+      Next();
+      return factory()->NewEmptyStatement(kNoSourcePosition);
+    case Token::IF:
+      return impl()->ParseIfStatement(labels, ok);
+    case Token::DO:
+      return impl()->ParseDoWhileStatement(labels, ok);
+    case Token::WHILE:
+      return impl()->ParseWhileStatement(labels, ok);
+    case Token::FOR:
+      return impl()->ParseForStatement(labels, ok);
+    case Token::CONTINUE:
+    case Token::BREAK:
+    case Token::RETURN:
+    case Token::THROW:
+    case Token::TRY: {
+      // These statements must have their labels preserved in an enclosing
+      // block
+      if (labels == nullptr) {
+        return ParseStatementAsUnlabelled(labels, ok);
+      } else {
+        BlockT result =
+            factory()->NewBlock(labels, 1, false, kNoSourcePosition);
+        typename Types::Target target(this, result);
+        StatementT statement = ParseStatementAsUnlabelled(labels, CHECK_OK);

[marja, 2016/09/05 08:14:33]

What does "ParseStatementAsUnlabelled" mean? You're not changing the name in
this CL but maybe you could add a one-liner comment.

[nickie, 2016/09/05 11:29:31]

I don't know what it means... :-)
(Offline discussion.)
I'm adding a few lines to the comment above and I'm adding a comment before the
definition of ParseStatementAsUnlabelled.

+        result->statements()->Add(statement, zone());
+        return result;
+      }
+    }
+    case Token::WITH:
+      return impl()->ParseWithStatement(labels, ok);
+    case Token::SWITCH:
+      return impl()->ParseSwitchStatement(labels, ok);
+    case Token::FUNCTION:
+      // FunctionDeclaration only allowed as a StatementListItem, not in
+      // an arbitrary Statement position. Exceptions such as
+      // ES#sec-functiondeclarations-in-ifstatement-statement-clauses
+      // are handled by calling ParseScopedStatement rather than
+      // ParseStatement directly.
+      impl()->ReportMessageAt(scanner()->peek_location(),
+                              is_strict(language_mode())
+                                  ? MessageTemplate::kStrictFunction
+                                  : MessageTemplate::kSloppyFunction);
+      *ok = false;
+      return impl()->NullStatement();
+    case Token::DEBUGGER:
+      return impl()->ParseDebuggerStatement(ok);
+    case Token::VAR:
+      return impl()->ParseVariableStatement(kStatement, nullptr, ok);
+    default:
+      return impl()->ParseExpressionOrLabelledStatement(labels, allow_function,
+                                                        ok);
+  }
+}
+
+template <typename Impl>
+typename ParserBase<Impl>::StatementT
+ParserBase<Impl>::ParseStatementAsUnlabelled(
+    ZoneList<const AstRawString*>* labels, bool* ok) {
+  switch (peek()) {
+    case Token::CONTINUE:
+      return impl()->ParseContinueStatement(ok);
+    case Token::BREAK:
+      return impl()->ParseBreakStatement(labels, ok);
+    case Token::RETURN:
+      return impl()->ParseReturnStatement(ok);
+    case Token::THROW:
+      return impl()->ParseThrowStatement(ok);
+    case Token::TRY:
+      return impl()->ParseTryStatement(ok);
+    default:
+      UNREACHABLE();
+      return impl()->NullStatement();
+  }
+}
+
 #undef CHECK_OK
 #undef CHECK_OK_CUSTOM
 
 template <typename Impl>
 void ParserBase<Impl>::ObjectLiteralChecker::CheckProperty(
     Token::Value property, PropertyKind type, MethodKind method_type,
     bool* ok) {
   DCHECK(!IsStaticMethod(method_type));
   DCHECK(!IsSpecialMethod(method_type) ||
          type == PropertyKind::kMethodProperty);
@@ skipping 45 matching lines @@
     has_seen_constructor_ = true;
     return;
   }
 }
 
 
 }  // namespace internal
 }  // namespace v8
 
 #endif  // V8_PARSING_PARSER_BASE_H