Implement ES6 Template Literals

src/preparser.h

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef V8_PREPARSER_H
 #define V8_PREPARSER_H
 
 #include "src/v8.h"
 
 #include "src/bailout-reason.h"
@@ skipping 84 matching lines @@
   bool allow_arrow_functions() const { return allow_arrow_functions_; }
   bool allow_modules() const { return scanner()->HarmonyModules(); }
   bool allow_harmony_scoping() const { return scanner()->HarmonyScoping(); }
   bool allow_harmony_numeric_literals() const {
     return scanner()->HarmonyNumericLiterals();
   }
   bool allow_classes() const { return scanner()->HarmonyClasses(); }
   bool allow_harmony_object_literals() const {
     return allow_harmony_object_literals_;
   }
+  bool allow_harmony_templates() const { return scanner()->HarmonyTemplates(); }
 
   // Setters that determine whether certain syntactical constructs are
   // allowed to be parsed by this instance of the parser.
   void set_allow_lazy(bool allow) { allow_lazy_ = allow; }
   void set_allow_natives_syntax(bool allow) { allow_natives_syntax_ = allow; }
   void set_allow_arrow_functions(bool allow) { allow_arrow_functions_ = allow; }
   void set_allow_modules(bool allow) { scanner()->SetHarmonyModules(allow); }
   void set_allow_harmony_scoping(bool allow) {
     scanner()->SetHarmonyScoping(allow);
   }
   void set_allow_harmony_numeric_literals(bool allow) {
     scanner()->SetHarmonyNumericLiterals(allow);
   }
   void set_allow_classes(bool allow) { scanner()->SetHarmonyClasses(allow); }
   void set_allow_harmony_object_literals(bool allow) {
     allow_harmony_object_literals_ = allow;
   }
+  void set_allow_harmony_templates(bool allow) {
+    scanner()->SetHarmonyTemplates(allow);
+  }
 
  protected:
   enum AllowEvalOrArgumentsAsIdentifier {
     kAllowEvalOrArguments,
     kDontAllowEvalOrArguments
   };
 
   enum Mode {
     PARSE_LAZILY,
     PARSE_EAGERLY
@@ skipping 355 matching lines @@
   ExpressionT ParseMemberWithNewPrefixesExpression(bool* ok);
   ExpressionT ParseMemberExpression(bool* ok);
   ExpressionT ParseMemberExpressionContinuation(ExpressionT expression,
                                                 bool* ok);
   ExpressionT ParseArrowFunctionLiteral(int start_pos, ExpressionT params_ast,
                                         bool* ok);
   ExpressionT ParseClassLiteral(IdentifierT name,
                                 Scanner::Location function_name_location,
                                 bool name_is_strict_reserved, int pos,
                                 bool* ok);
+  ExpressionT ParseTemplateLiteral(ExpressionT tag, int start, bool* ok);
+  void AddTemplateSpan();
+  void AddTemplateExpression(ExpressionT);
 
   // Checks if the expression is a valid reference expression (e.g., on the
   // left-hand side of assignments). Although ruled out by ECMA as early errors,
   // we allow calls for web compatibility and rewrite them to a runtime throw.
   ExpressionT CheckAndRewriteReferenceExpression(
       ExpressionT expression,
       Scanner::Location location, const char* message, bool* ok);
 
   // Used to detect duplicates in object literals. Each of the values
   // kGetterProperty, kSetterProperty and kValueProperty represents
@@ skipping 865 matching lines @@
   // Utility functions
   int DeclareArrowParametersFromExpression(PreParserExpression expression,
                                            PreParserScope* scope,
                                            Scanner::Location* dupe_loc,
                                            bool* ok) {
     // TODO(aperez): Detect duplicated identifiers in paramlists.
     *ok = expression.IsValidArrowParamList();
     return 0;
   }
 
+  struct TemplateLiteralState {};
+
+  TemplateLiteralState OpenTemplateLiteral(int pos) {
+    return TemplateLiteralState();
+  }
+  void AddTemplateSpan(TemplateLiteralState* state) { USE(state); }
+  void AddTemplateExpression(TemplateLiteralState* state,
+                             PreParserExpression expression) {
+    USE(state);
+    USE(expression);
+  }
+  PreParserExpression CloseTemplateLiteral(TemplateLiteralState* state,
+                                           int start, PreParserExpression tag) {
+    USE(state);
+    USE(start);
+    USE(tag);
+    return EmptyExpression();
+  }
+  PreParserExpression NoTemplateTag() {
+    return PreParserExpression::Default();
+  }
   static AstValueFactory* ast_value_factory() { return NULL; }
 
   void CheckConflictingVarDeclarations(PreParserScope scope, bool* ok) {}
 
   // Temporary glue; these functions will move to ParserBase.
   PreParserExpression ParseV8Intrinsic(bool* ok);
   PreParserExpression ParseFunctionLiteral(
       PreParserIdentifier name, Scanner::Location function_name_location,
       bool name_is_strict_reserved, FunctionKind kind,
       int function_token_position, FunctionLiteral::FunctionType type,
@@ skipping 350 matching lines @@
   //   'true'
   //   'false'
   //   Identifier
   //   Number
   //   String
   //   ArrayLiteral
   //   ObjectLiteral
   //   RegExpLiteral
   //   ClassLiteral
   //   '(' Expression ')'
+  //   TemplateLiteral
 
   int pos = peek_position();
   ExpressionT result = this->EmptyExpression();
   Token::Value token = peek();
   switch (token) {
     case Token::THIS: {
       Consume(Token::THIS);
       scope_->RecordThisUsage();
       result = this->ThisExpression(scope_, factory());
       break;
@@ skipping 68 matching lines @@
         name = ParseIdentifierOrStrictReservedWord(&is_strict_reserved_name,
                                                    CHECK_OK);
         class_name_location = scanner()->location();
       }
       result = this->ParseClassLiteral(name, class_name_location,
                                        is_strict_reserved_name,
                                        class_token_position, CHECK_OK);
       break;
     }
 
+    case Token::TEMPLATE_SPAN:
+    case Token::TEMPLATE_TAIL:
+      result = this->ParseTemplateLiteral(Traits::NoTemplateTag(), pos,
+                                          CHECK_OK);
+      break;
+
     case Token::MOD:
       if (allow_natives_syntax() || extension_ != NULL) {
         result = this->ParseV8Intrinsic(CHECK_OK);
         break;
       }
       // If we're not allowing special syntax we fall-through to the
       // default case.
 
     default: {
       Next();
@@ skipping 603 matching lines @@
         // direct eval calls. These calls are all of the form eval(...), with
         // no explicit receiver.
         // These calls are marked as potentially direct eval calls. Whether
         // they are actually direct calls to eval is determined at run time.
         this->CheckPossibleEvalCall(result, scope_);
         result = factory()->NewCall(result, args, pos);
         if (fni_ != NULL) fni_->RemoveLastFunction();
         break;
       }
 
+      case Token::TEMPLATE_SPAN:
+      case Token::TEMPLATE_TAIL: {
+        int pos;
+        if (scanner()->current_token() == Token::IDENTIFIER) {
+          pos = position();
+        } else {
+          pos = peek_position();
+          if (result->IsFunctionLiteral() && mode() == PARSE_EAGERLY) {
+            result->AsFunctionLiteral()->set_parenthesized();

[marja, 2014/11/10 15:32:02]

Why?

[caitp (gmail), 2014/11/10 15:43:56]

It's basically copy/pasted from the MemberExpression -> CallExpression path ---
so, presumably if we have an IIFE we need to do this, but I'm not sure what
difference it makes.

[marja, 2014/11/11 09:15:36]

I dug up a bit where this is coming from. In any case, it needs a comment.

So, the original code is in ParseLeftHandSideExpression, and afaics, it's meant
to recognize this pattern:

function foo() { ... } (
^ result is a FunctionLiteral               
                       ^  just saw the LPAREN

so in that case, that's a hint that the function will be called right away, and
set_parenthesized is called to force eager compilation. Fair enough.

So, here, the corresponding situation would be e.g.,

function foo() { ... } `tail`
^ result is a FunctionLiteral
                            ^ just saw the TEMPLATE_TAIL

And this should be explained in the comment.

+          }
+        }
+        result = ParseTemplateLiteral(result, pos, CHECK_OK);
+        break;
+      }
+
       case Token::PERIOD: {
         Consume(Token::PERIOD);
         int pos = position();
         IdentifierT name = ParseIdentifierName(CHECK_OK);
         result = factory()->NewProperty(
             result, factory()->NewStringLiteral(name, pos), pos);
         if (fni_ != NULL) this->PushLiteralName(fni_, name);
         break;
       }
 
@@ skipping 318 matching lines @@
   int end_pos = peek_position();
   Expect(Token::RBRACE, CHECK_OK);
 
   return this->ClassExpression(name, extends, constructor, properties, pos,
                                end_pos + 1, factory());
 }
 
 
 template <typename Traits>
 typename ParserBase<Traits>::ExpressionT
+ParserBase<Traits>::ParseTemplateLiteral(ExpressionT tag, int start, bool* ok) {

[marja, 2014/11/10 15:32:02]

Can you add here a comment about the parse rules which describe the syntax (like
the comments many other parsing functions have)?

+  CHECK(peek() == Token::TEMPLATE_SPAN || peek() == Token::TEMPLATE_TAIL);
+
+  // Add TV / TRV
+  if (peek() == Token::TEMPLATE_SPAN) {
+    Consume(Token::TEMPLATE_SPAN);
+    int pos = position();
+    typename Traits::TemplateLiteralState ts = Traits::OpenTemplateLiteral(pos);
+    Traits::AddTemplateSpan(&ts);
+
+    for (;;) {
+      Token::Value next = peek();
+      if (next < 0) {
+        ReportMessage("unterminated_template");
+        *ok = false;
+        return Traits::EmptyExpression();
+      }
+
+      // Parse an Expression
+      ExpressionT expression = this->ParseExpression(false, CHECK_OK);

[marja, 2014/11/10 15:32:02]

Why expression? Are we now inside the ${... or... what? No, not necessarily?
*Confused*

[caitp (gmail), 2014/11/10 15:43:56]

Yeah, if we get a TEMPLATE_SPAN token, then we just finished parsing `${`, and
are ready to parse an expression. I'll add a comment about this.

Do you think we could combine TEMPLATE_SPAN and TEMPLATE_TAIL into a single
token somehow? I'm not sure how that would work properly

+      Traits::AddTemplateExpression(&ts, expression);
+
+      // If we didn't die parsing that expression, our next token should be a
+      // TEMPLATE_SPAN or
+      // TEMPLATE_TAIL.
+      next = scanner()->ScanTemplateSpan();
+      Next();
+
+      if (next == Token::ILLEGAL || next < 0) {
+        ReportMessage("unterminated_template");
+        *ok = false;
+        return Traits::EmptyExpression();
+      }
+
+      CHECK(next == Token::TEMPLATE_SPAN || next == Token::TEMPLATE_TAIL);
+      Traits::AddTemplateSpan(&ts);
+
+      if (next == Token::TEMPLATE_TAIL) {
+        break;
+      }
+    }
+    return Traits::CloseTemplateLiteral(&ts, start, tag);
+  }
+  Consume(Token::TEMPLATE_TAIL);
+  int pos = position();
+  typename Traits::TemplateLiteralState ts = Traits::OpenTemplateLiteral(pos);
+  Traits::AddTemplateSpan(&ts);
+  return Traits::CloseTemplateLiteral(&ts, start, tag);
+}
+
+
+template <typename Traits>
+typename ParserBase<Traits>::ExpressionT
 ParserBase<Traits>::CheckAndRewriteReferenceExpression(
     ExpressionT expression,
     Scanner::Location location, const char* message, bool* ok) {
   if (strict_mode() == STRICT && this->IsIdentifier(expression) &&
       this->IsEvalOrArguments(this->AsIdentifier(expression))) {
     this->ReportMessageAt(location, "strict_eval_arguments", false);
     *ok = false;
     return this->EmptyExpression();
   } else if (expression->IsValidReferenceExpression()) {
     return expression;
@@ skipping 37 matching lines @@
       DCHECK(IsAccessorAccessorConflict(old_type, type));
       // Both accessors of the same type.
       parser()->ReportMessage("accessor_get_set");
     }
     *ok = false;
   }
 }
 } }  // v8::internal
 
 #endif  // V8_PREPARSER_H