Implement ES6 Template Literals

src/scanner.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // Features shared by parsing and pre-parsing scanners.
 
 #include <stdint.h>
 
 #include <cmath>
 
@@ skipping 36 matching lines @@
   // in object literals.
   Init();
   // Skip initial whitespace allowing HTML comment ends just like
   // after a newline and scan first token.
   has_line_terminator_before_next_ = true;
   SkipWhiteSpace();
   Scan();
 }
 
 
-uc32 Scanner::ScanHexNumber(int expected_length) {
+uc32 Scanner::ScanHexNumber(int expected_length, bool recordRaw) {
   DCHECK(expected_length <= 4);  // prevent overflow
 
   uc32 x = 0;
   for (int i = 0; i < expected_length; i++) {
     int d = HexValue(c0_);
     if (d < 0) {
       return -1;
     }
     x = x * 16 + d;
+    if (recordRaw) {
+      AddRawLiteralChar(c0_);
+    }
     Advance();
   }
 
   return x;
 }
 
 
 // Ensure that tokens can be stored in a byte.
 STATIC_ASSERT(Token::NUM_TOKENS <= 0x100);
 
@@ skipping 319 matching lines @@
     PushBack('-');  // undo Advance()
   }
   PushBack('!');  // undo Advance()
   DCHECK(c0_ == '!');
   return Token::LT;
 }
 
 
 void Scanner::Scan() {
   next_.literal_chars = NULL;
+  next_.raw_literal_chars = NULL;
   Token::Value token;
+
   do {
     // Remember the position of the next token
     next_.location.beg_pos = source_pos();
 
     switch (c0_) {
       case ' ':
       case '\t':
         Advance();
         token = Token::WHITESPACE;
         break;
@@ skipping 202 matching lines @@
         break;
 
       case '?':
         token = Select(Token::CONDITIONAL);
         break;
 
       case '~':
         token = Select(Token::BIT_NOT);
         break;
 
+      case '`':
+        if (HarmonyTemplates()) {
+          token = ScanTemplateSpan();
+          break;
+        }
+
       default:
         if (c0_ < 0) {
           token = Token::EOS;
         } else if (unicode_cache_->IsIdentifierStart(c0_)) {
           token = ScanIdentifierOrKeyword();
         } else if (IsDecimalDigit(c0_)) {
           token = ScanNumber(false);
         } else if (SkipWhiteSpace()) {
           token = Token::WHITESPACE;
         } else {
@@ skipping 25 matching lines @@
     // This function is only called to seek to the location
     // of the end of a function (at the "}" token). It doesn't matter
     // whether there was a line terminator in the part we skip.
     has_line_terminator_before_next_ = false;
     has_multiline_comment_before_next_ = false;
   }
   Scan();
 }
 
 
-bool Scanner::ScanEscape() {
+bool Scanner::ScanEscape(bool recordRaw) {
   uc32 c = c0_;
+  uc32 rc = c;
+  bool singleCharEscape = true;
   Advance();
 
   // Skip escaped newlines.
   if (c0_ >= 0 && unicode_cache_->IsLineTerminator(c)) {
     // Allow CR+LF newlines in multiline string literals.
     if (IsCarriageReturn(c) && IsLineFeed(c0_)) Advance();
     // Allow LF+CR newlines in multiline string literals.
     if (IsLineFeed(c) && IsCarriageReturn(c0_)) Advance();
     return true;
   }
 
   switch (c) {
     case '\'':  // fall through
     case '"' :  // fall through
     case '\\': break;
     case 'b' : c = '\b'; break;
     case 'f' : c = '\f'; break;
     case 'n' : c = '\n'; break;
     case 'r' : c = '\r'; break;
     case 't' : c = '\t'; break;
     case 'u' : {
-      c = ScanHexNumber(4);
+      if (recordRaw) AddRawLiteralChar('u');
+      singleCharEscape = false;
+      c = ScanHexNumber(4, recordRaw);
       if (c < 0) return false;
       break;
     }
     case 'v' : c = '\v'; break;
     case 'x' : {
-      c = ScanHexNumber(2);
+      if (recordRaw) AddRawLiteralChar('x');
+      singleCharEscape = false;
+      c = ScanHexNumber(2, recordRaw);
       if (c < 0) return false;
       break;
     }
     case '0' :  // fall through
     case '1' :  // fall through
     case '2' :  // fall through
     case '3' :  // fall through
     case '4' :  // fall through
     case '5' :  // fall through
     case '6' :  // fall through
-    case '7' : c = ScanOctalEscape(c, 2); break;
+    case '7':
+      singleCharEscape = false;
+      c = ScanOctalEscape(c, 2, recordRaw);
+      break;
   }
 
   // According to ECMA-262, section 7.8.4, characters not covered by the
   // above cases should be illegal, but they are commonly handled as
   // non-escaped characters by JS VMs.
+  if (singleCharEscape && recordRaw) AddRawLiteralChar(rc);
   AddLiteralChar(c);
   return true;
 }
 
 
 // Octal escapes of the forms '\0xx' and '\xxx' are not a part of
 // ECMA-262. Other JS VMs support them.
-uc32 Scanner::ScanOctalEscape(uc32 c, int length) {
+uc32 Scanner::ScanOctalEscape(uc32 c, int length, bool recordRaw) {
   uc32 x = c - '0';
   int i = 0;
   for (; i < length; i++) {
     int d = c0_ - '0';
     if (d < 0 || d > 7) break;
     int nx = x * 8 + d;
     if (nx >= 256) break;
     x = nx;
+    if (recordRaw) {
+      AddRawLiteralChar(c0_);
+    }
     Advance();
   }
   // Anything except '\0' is an octal escape sequence, illegal in strict mode.
   // Remember the position of octal escape sequences so that an error
   // can be reported later (in strict mode).
   // We don't report the error immediately, because the octal escape can
   // occur before the "use strict" directive.
   if (c != '0' || i > 0) {
     octal_pos_ = Location(source_pos() - i - 1, source_pos() - 1);
   }
@@ skipping 17 matching lines @@
     }
   }
   if (c0_ != quote) return Token::ILLEGAL;
   literal.Complete();
 
   Advance();  // consume quote
   return Token::STRING;
 }
 
 
+Token::Value Scanner::ScanTemplateSpan() {

[marja, 2014/11/10 15:32:02]

Hmm, I don't fully understand this function.

Pls add comments:

1) about how literal_chars_ and raw_literal_chars_ are used and why (I was
briefly confused when I read that part at first...)
2) about which part exactly this function scans; what's the returned token and
what's in the literal buffer...

I'm also confused because the terminology doesn't seem to match the spec
draft... what's "TEMPLATE_SPAN"? (The spec has stuff like template head,
template middle, template tail..)

[caitp (gmail), 2014/11/10 15:43:56]

literal_chars_ are being used as the TV or cooked value of the template (escaped
characters -> unescaped form, basically), raw_literal_chars_ are being used as
the TRV or raw value of the template (escaped characters as they appear in the
source, basically).

arv@ has suggested that we do this without keeping a separate string for the raw
values, but I don't know how it will actually work in all cases, since raw
values need to normalize line endings anyways.

Ideas on that are welcome.

So the spec has concept of template head -> template middle -> template end ->
etc, but in practice I think there are only 2 cases that matter: the case where
you have a literal span followed by an expression, and the case where you have
just a literal span.

TEMPLATE_SPAN -> always followed by an expression (ends with ${), while
TEMPLATE_TAIL -> always followed by ` or the end of the template literal (if
not, then it's a bad token).

Basically, all of the NoSubstTemplate productions become TEMPLATE_TAIL, and all
of the other template productions become TEMPLATE_SPAN, I think it's simpler
this way

+  if (next_.token == Token::RBRACE) {
+    PushBack('}');
+  }
+  next_.location.beg_pos = source_pos();
+  Token::Value result = Token::ILLEGAL;
+  DCHECK(c0_ == '`' || c0_ == '}');
+  Advance();  // Consume ` or }
+
+  LiteralScope literal(this);
+  while (true) {
+    uc32 c = c0_;
+    Advance();
+    if (c == '`') {
+      result = Token::TEMPLATE_TAIL;
+      break;
+    } else if (c == '$' && c0_ == '{') {
+      Advance();  // Consume '{'
+      result = Token::TEMPLATE_SPAN;
+      break;
+    } else if (c == '\\') {
+      AddRawLiteralChar('\\');
+      if (unicode_cache_->IsLineTerminator(c0_)) {
+        // The TV of LineContinuation :: \ LineTerminatorSequence is the empty
+        // code unit sequence.
+        do {
+          uc32 lastChar = c0_;
+          Advance();
+          if (lastChar == '\r' && c0_ == '\n') Advance();
+          AddRawLiteralChar('\n');
+        } while (unicode_cache_->IsLineTerminator(c0_));
+      } else if (c0_ == '0') {
+        Advance();
+        AddRawLiteralChar('0');
+        AddLiteralChar('0');
+      } else {
+        ScanEscape(true);
+      }
+    } else if (c < 0) {
+      // Unterminated template literal
+      PushBack(c);
+      break;
+    } else {
+      // The TRV of LineTerminatorSequence :: <CR> is the CV 0x000A.
+      // The TRV of LineTerminatorSequence :: <CR><LF> is the sequence
+      // consisting of the CV 0x000A.
+      if (c == '\r') {
+        if (c0_ == '\n') Advance();
+        c = '\n';
+      }
+      AddLiteralChar(c);
+      AddRawLiteralChar(c);
+    }
+  }
+  literal.Complete();
+  next_.location.end_pos = source_pos();
+  next_.token = result;
+  return result;
+}
+
+
 void Scanner::ScanDecimalDigits() {
   while (IsDecimalDigit(c0_))
     AddLiteralCharAdvance();
 }
 
 
 Token::Value Scanner::ScanNumber(bool seen_period) {
   DCHECK(IsDecimalDigit(c0_));  // the first digit of the number or the fraction
 
   enum { DECIMAL, HEX, OCTAL, IMPLICIT_OCTAL, BINARY } kind = DECIMAL;
@@ skipping 394 matching lines @@
 
 
 const AstRawString* Scanner::CurrentSymbol(AstValueFactory* ast_value_factory) {
   if (is_literal_one_byte()) {
     return ast_value_factory->GetOneByteString(literal_one_byte_string());
   }
   return ast_value_factory->GetTwoByteString(literal_two_byte_string());
 }
 
 
+const AstRawString* Scanner::CurrentRawSymbol(
+    AstValueFactory* ast_value_factory) {
+  if (is_raw_one_byte()) {
+    return ast_value_factory->GetOneByteString(raw_one_byte_string());
+  }
+  return ast_value_factory->GetTwoByteString(raw_two_byte_string());
+}
+
+
 const AstRawString* Scanner::NextSymbol(AstValueFactory* ast_value_factory) {
   if (is_next_literal_one_byte()) {
     return ast_value_factory->GetOneByteString(next_literal_one_byte_string());
   }
   return ast_value_factory->GetTwoByteString(next_literal_two_byte_string());
 }
 
 
+const AstRawString* Scanner::NextRawSymbol(AstValueFactory* ast_value_factory) {
+  if (is_next_raw_one_byte()) {
+    return ast_value_factory->GetOneByteString(next_raw_one_byte_string());
+  }
+  return ast_value_factory->GetTwoByteString(next_raw_two_byte_string());
+}
+
+
 double Scanner::DoubleValue() {
   DCHECK(is_literal_one_byte());
   return StringToDouble(
       unicode_cache_,
       literal_one_byte_string(),
       ALLOW_HEX | ALLOW_OCTAL | ALLOW_IMPLICIT_OCTAL | ALLOW_BINARY);
 }
 
 
 int Scanner::FindNumber(DuplicateFinder* finder, int value) {
@@ skipping 142 matching lines @@
     }
     backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u));
   }
   backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f));
 
   backing_store_.AddBlock(bytes);
   return backing_store_.EndSequence().start();
 }
 
 } }  // namespace v8::internal