Implement ES6 Template Literals

src/scanner.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // Features shared by parsing and pre-parsing scanners.
 
 #include <stdint.h>
 
 #include <cmath>
 
@@ skipping 36 matching lines @@
   // in object literals.
   Init();
   // Skip initial whitespace allowing HTML comment ends just like
   // after a newline and scan first token.
   has_line_terminator_before_next_ = true;
   SkipWhiteSpace();
   Scan();
 }
 
 
-uc32 Scanner::ScanHexNumber(int expected_length) {
+uc32 Scanner::ScanHexNumber(int expected_length, bool recordRaw) {
   DCHECK(expected_length <= 4);  // prevent overflow
 
   uc32 x = 0;
   for (int i = 0; i < expected_length; i++) {
     int d = HexValue(c0_);
     if (d < 0) {
       return -1;
     }
     x = x * 16 + d;
+    if (recordRaw) {
+      AddRawLiteralChar(c0_);
+    }
     Advance();
   }
 
   return x;
 }
 
 
 // Ensure that tokens can be stored in a byte.
 STATIC_ASSERT(Token::NUM_TOKENS <= 0x100);
 
@@ skipping 319 matching lines @@
     PushBack('-');  // undo Advance()
   }
   PushBack('!');  // undo Advance()
   DCHECK(c0_ == '!');
   return Token::LT;
 }
 
 
 void Scanner::Scan() {
   next_.literal_chars = NULL;
+  next_.raw_literal_chars = NULL;
   Token::Value token;
+
   do {
     // Remember the position of the next token
     next_.location.beg_pos = source_pos();
 
     switch (c0_) {
       case ' ':
       case '\t':
         Advance();
         token = Token::WHITESPACE;
         break;
@@ skipping 202 matching lines @@
         break;
 
       case '?':
         token = Select(Token::CONDITIONAL);
         break;
 
       case '~':
         token = Select(Token::BIT_NOT);
         break;
 
+      case '`':
+        if (HarmonyTemplates()) {
+          token = ScanTemplateSpan();
+          break;
+        }
+
       default:
         if (c0_ < 0) {
           token = Token::EOS;
         } else if (unicode_cache_->IsIdentifierStart(c0_)) {
           token = ScanIdentifierOrKeyword();
         } else if (IsDecimalDigit(c0_)) {
           token = ScanNumber(false);
         } else if (SkipWhiteSpace()) {
           token = Token::WHITESPACE;
         } else {
@@ skipping 25 matching lines @@
     // This function is only called to seek to the location
     // of the end of a function (at the "}" token). It doesn't matter
     // whether there was a line terminator in the part we skip.
     has_line_terminator_before_next_ = false;
     has_multiline_comment_before_next_ = false;
   }
   Scan();
 }
 
 
-bool Scanner::ScanEscape() {
+bool Scanner::ScanEscape(bool recordRaw) {
   uc32 c = c0_;
+  uc32 rc = c;
+  bool singleCharEscape = true;
   Advance();
 
   // Skip escaped newlines.
   if (c0_ >= 0 && unicode_cache_->IsLineTerminator(c)) {
     // Allow CR+LF newlines in multiline string literals.
     if (IsCarriageReturn(c) && IsLineFeed(c0_)) Advance();
     // Allow LF+CR newlines in multiline string literals.
     if (IsLineFeed(c) && IsCarriageReturn(c0_)) Advance();
     return true;
   }
 
   switch (c) {
     case '\'':  // fall through
     case '"' :  // fall through
     case '\\': break;
     case 'b' : c = '\b'; break;
     case 'f' : c = '\f'; break;
     case 'n' : c = '\n'; break;
     case 'r' : c = '\r'; break;
     case 't' : c = '\t'; break;
     case 'u' : {
-      c = ScanHexNumber(4);
+      if (recordRaw) AddRawLiteralChar('u');
+      singleCharEscape = false;
+      c = ScanHexNumber(4, recordRaw);
       if (c < 0) return false;
       break;
     }
     case 'v' : c = '\v'; break;
     case 'x' : {
-      c = ScanHexNumber(2);
+      if (recordRaw) AddRawLiteralChar('x');
+      singleCharEscape = false;
+      c = ScanHexNumber(2, recordRaw);
       if (c < 0) return false;
       break;
     }
     case '0' :  // fall through
     case '1' :  // fall through
     case '2' :  // fall through
     case '3' :  // fall through
     case '4' :  // fall through
     case '5' :  // fall through
     case '6' :  // fall through
-    case '7' : c = ScanOctalEscape(c, 2); break;
+    case '7':
+      singleCharEscape = false;
+      c = ScanOctalEscape(c, 2, recordRaw);
+      break;
   }
 
   // According to ECMA-262, section 7.8.4, characters not covered by the
   // above cases should be illegal, but they are commonly handled as
   // non-escaped characters by JS VMs.
+  if (singleCharEscape && recordRaw) AddRawLiteralChar(rc);
   AddLiteralChar(c);
   return true;
 }
 
 
 // Octal escapes of the forms '\0xx' and '\xxx' are not a part of
 // ECMA-262. Other JS VMs support them.
-uc32 Scanner::ScanOctalEscape(uc32 c, int length) {
+uc32 Scanner::ScanOctalEscape(uc32 c, int length, bool recordRaw) {
   uc32 x = c - '0';
   int i = 0;
   for (; i < length; i++) {
     int d = c0_ - '0';
     if (d < 0 || d > 7) break;
     int nx = x * 8 + d;
     if (nx >= 256) break;
     x = nx;
+    if (recordRaw) {
+      AddRawLiteralChar(c0_);
+    }
     Advance();
   }
   // Anything except '\0' is an octal escape sequence, illegal in strict mode.
   // Remember the position of octal escape sequences so that an error
   // can be reported later (in strict mode).
   // We don't report the error immediately, because the octal escape can
   // occur before the "use strict" directive.
   if (c != '0' || i > 0) {
     octal_pos_ = Location(source_pos() - i - 1, source_pos() - 1);
   }
@@ skipping 17 matching lines @@
     }
   }
   if (c0_ != quote) return Token::ILLEGAL;
   literal.Complete();
 
   Advance();  // consume quote
   return Token::STRING;
 }
 
 
+Token::Value Scanner::ScanTemplateSpan() {
+  // When scanning a TemplateSpan, we are looking for the following construct:
+  // TEMPLATE_SPAN ::
+  //     ` LiteralChars* ${
+  //   | } LiteralChars* ${
+  //
+  // TEMPLATE_TAIL ::
+  //     ` LiteralChars* `
+  //   | } LiteralChar* `
+  //
+  // A TEMPLATE_SPAN should always be followed by an Expression, while a
+  // TEMPLATE_TAIL terminates a TemplateLiteral and does not need to be
+  // followed by an Expression.
+  //
+  // raw_literal_chars_ represents TRV or the raw value of the template span,
+  // per the spec, while literal_chars_ represents TV or the cooked value of
+  // the template span.
+  //
+  // TODO(caitp): Do not store a separate literal buffer for the span TRV.
+  //
+
+  if (next_.token == Token::RBRACE) {
+    PushBack('}');

[marja, 2014/11/11 09:47:24]

Hmm, when does this happen and why do we PushBack?

[caitp (gmail), 2014/11/11 13:59:29]

After parsing an expression, the scanner ends up with a peeked RBRACE and we're
at the wrong position in source, I'm not sure if this is the right way to deal
with that but it seems to work.

Without this, it if you have a template like `${a}`, it misses the ending
backtick and breaks, which is bad

[marja, 2014/11/11 15:01:56]

But after this, we anyway do Advance() right away. Would the code be clearer if
it was restructured to be PushBack-free?

+  }
+  next_.location.beg_pos = source_pos();
+  Token::Value result = Token::ILLEGAL;
+  DCHECK(c0_ == '`' || c0_ == '}');
+  Advance();  // Consume ` or }
+
+  LiteralScope literal(this);
+  while (true) {
+    uc32 c = c0_;
+    Advance();
+    if (c == '`') {
+      result = Token::TEMPLATE_TAIL;
+      break;
+    } else if (c == '$' && c0_ == '{') {
+      Advance();  // Consume '{'
+      result = Token::TEMPLATE_SPAN;
+      break;
+    } else if (c == '\\') {
+      AddRawLiteralChar('\\');
+      if (unicode_cache_->IsLineTerminator(c0_)) {
+        // The TV of LineContinuation :: \ LineTerminatorSequence is the empty
+        // code unit sequence.
+        do {
+          uc32 lastChar = c0_;
+          Advance();
+          if (lastChar == '\r' && c0_ == '\n') Advance();
+          AddRawLiteralChar('\n');
+        } while (unicode_cache_->IsLineTerminator(c0_));
+      } else if (c0_ == '0') {
+        Advance();
+        AddRawLiteralChar('0');
+        AddLiteralChar('0');
+      } else {
+        ScanEscape(true);
+      }
+    } else if (c < 0) {
+      // Unterminated template literal
+      PushBack(c);
+      break;
+    } else {
+      // The TRV of LineTerminatorSequence :: <CR> is the CV 0x000A.
+      // The TRV of LineTerminatorSequence :: <CR><LF> is the sequence
+      // consisting of the CV 0x000A.
+      if (c == '\r') {
+        if (c0_ == '\n') Advance();
+        c = '\n';
+      }
+      AddLiteralChar(c);
+      AddRawLiteralChar(c);
+    }
+  }
+  literal.Complete();
+  next_.location.end_pos = source_pos();
+  next_.token = result;
+  return result;
+}
+
+
 void Scanner::ScanDecimalDigits() {
   while (IsDecimalDigit(c0_))
     AddLiteralCharAdvance();
 }
 
 
 Token::Value Scanner::ScanNumber(bool seen_period) {
   DCHECK(IsDecimalDigit(c0_));  // the first digit of the number or the fraction
 
   enum { DECIMAL, HEX, OCTAL, IMPLICIT_OCTAL, BINARY } kind = DECIMAL;
@@ skipping 373 matching lines @@
 
 
 const AstRawString* Scanner::CurrentSymbol(AstValueFactory* ast_value_factory) {
   if (is_literal_one_byte()) {
     return ast_value_factory->GetOneByteString(literal_one_byte_string());
   }
   return ast_value_factory->GetTwoByteString(literal_two_byte_string());
 }
 
 
+const AstRawString* Scanner::CurrentRawSymbol(
+    AstValueFactory* ast_value_factory) {
+  if (is_raw_one_byte()) {
+    return ast_value_factory->GetOneByteString(raw_one_byte_string());
+  }
+  return ast_value_factory->GetTwoByteString(raw_two_byte_string());
+}
+
+
 const AstRawString* Scanner::NextSymbol(AstValueFactory* ast_value_factory) {
   if (is_next_literal_one_byte()) {
     return ast_value_factory->GetOneByteString(next_literal_one_byte_string());
   }
   return ast_value_factory->GetTwoByteString(next_literal_two_byte_string());
 }
 
 
+const AstRawString* Scanner::NextRawSymbol(AstValueFactory* ast_value_factory) {
+  if (is_next_raw_one_byte()) {
+    return ast_value_factory->GetOneByteString(next_raw_one_byte_string());
+  }
+  return ast_value_factory->GetTwoByteString(next_raw_two_byte_string());
+}
+
+
 double Scanner::DoubleValue() {
   DCHECK(is_literal_one_byte());
   return StringToDouble(
       unicode_cache_,
       literal_one_byte_string(),
       ALLOW_HEX | ALLOW_OCTAL | ALLOW_IMPLICIT_OCTAL | ALLOW_BINARY);
 }
 
 
 int Scanner::FindNumber(DuplicateFinder* finder, int value) {
@@ skipping 142 matching lines @@
     }
     backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u));
   }
   backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f));
 
   backing_store_.AddBlock(bytes);
   return backing_store_.EndSequence().start();
 }
 
 } }  // namespace v8::internal