Implement ES6 Template Literals

src/scanner.cc

Index: src/scanner.cc
diff --git a/src/scanner.cc b/src/scanner.cc
index e63239d6eb0d3afed82d7af59498cc6c9a1cbfe4..c5d8e98980b39785a162fdc7a3dcab3212ef9be7 100644
--- a/src/scanner.cc
+++ b/src/scanner.cc
@@ -54,7 +54,7 @@ void Scanner::Initialize(Utf16CharacterStream* source) {
 }
 
 
-uc32 Scanner::ScanHexNumber(int expected_length) {
+uc32 Scanner::ScanHexNumber(int expected_length, bool recordRaw) {
   DCHECK(expected_length <= 4);  // prevent overflow
 
   uc32 x = 0;
@@ -64,6 +64,9 @@ uc32 Scanner::ScanHexNumber(int expected_length) {
       return -1;
     }
     x = x * 16 + d;
+    if (recordRaw) {
+      AddRawLiteralChar(c0_);
+    }
     Advance();
   }
 
@@ -403,7 +406,9 @@ Token::Value Scanner::ScanHtmlComment() {
 
 void Scanner::Scan() {
   next_.literal_chars = NULL;
+  next_.raw_literal_chars = NULL;
   Token::Value token;
+
   do {
     // Remember the position of the next token
     next_.location.beg_pos = source_pos();
@@ -626,6 +631,12 @@ void Scanner::Scan() {
         token = Select(Token::BIT_NOT);
         break;
 
+      case '`':
+        if (HarmonyTemplates()) {
+          token = ScanTemplateSpan();
+          break;
+        }
+
       default:
         if (c0_ < 0) {
           token = Token::EOS;
@@ -671,8 +682,10 @@ void Scanner::SeekForward(int pos) {
 }
 
 
-bool Scanner::ScanEscape() {
+bool Scanner::ScanEscape(bool recordRaw) {
   uc32 c = c0_;
+  uc32 rc = c;
+  bool singleCharEscape = true;
   Advance();
 
   // Skip escaped newlines.
@@ -694,13 +707,17 @@ bool Scanner::ScanEscape() {
     case 'r' : c = '\r'; break;
     case 't' : c = '\t'; break;
     case 'u' : {
-      c = ScanHexNumber(4);
+      if (recordRaw) AddRawLiteralChar('u');
+      singleCharEscape = false;
+      c = ScanHexNumber(4, recordRaw);
       if (c < 0) return false;
       break;
     }
     case 'v' : c = '\v'; break;
     case 'x' : {
-      c = ScanHexNumber(2);
+      if (recordRaw) AddRawLiteralChar('x');
+      singleCharEscape = false;
+      c = ScanHexNumber(2, recordRaw);
       if (c < 0) return false;
       break;
     }
@@ -711,12 +728,16 @@ bool Scanner::ScanEscape() {
     case '4' :  // fall through
     case '5' :  // fall through
     case '6' :  // fall through
-    case '7' : c = ScanOctalEscape(c, 2); break;
+    case '7':
+      singleCharEscape = false;
+      c = ScanOctalEscape(c, 2, recordRaw);
+      break;
   }
 
   // According to ECMA-262, section 7.8.4, characters not covered by the
   // above cases should be illegal, but they are commonly handled as
   // non-escaped characters by JS VMs.
+  if (singleCharEscape && recordRaw) AddRawLiteralChar(rc);
   AddLiteralChar(c);
   return true;
 }
@@ -724,7 +745,7 @@ bool Scanner::ScanEscape() {
 
 // Octal escapes of the forms '\0xx' and '\xxx' are not a part of
 // ECMA-262. Other JS VMs support them.
-uc32 Scanner::ScanOctalEscape(uc32 c, int length) {
+uc32 Scanner::ScanOctalEscape(uc32 c, int length, bool recordRaw) {
   uc32 x = c - '0';
   int i = 0;
   for (; i < length; i++) {
@@ -733,6 +754,9 @@ uc32 Scanner::ScanOctalEscape(uc32 c, int length) {
     int nx = x * 8 + d;
     if (nx >= 256) break;
     x = nx;
+    if (recordRaw) {
+      AddRawLiteralChar(c0_);
+    }
     Advance();
   }
   // Anything except '\0' is an octal escape sequence, illegal in strict mode.
@@ -770,6 +794,67 @@ Token::Value Scanner::ScanString() {
 }
 
 
+Token::Value Scanner::ScanTemplateSpan() {

[marja, 2014/11/10 15:32:02]

Hmm, I don't fully understand this function.

Pls add comments:

1) about how literal_chars_ and raw_literal_chars_ are used and why (I was
briefly confused when I read that part at first...)
2) about which part exactly this function scans; what's the returned token and
what's in the literal buffer...

I'm also confused because the terminology doesn't seem to match the spec
draft... what's "TEMPLATE_SPAN"? (The spec has stuff like template head,
template middle, template tail..)

[caitp (gmail), 2014/11/10 15:43:56]

literal_chars_ are being used as the TV or cooked value of the template (escaped
characters -> unescaped form, basically), raw_literal_chars_ are being used as
the TRV or raw value of the template (escaped characters as they appear in the
source, basically).

arv@ has suggested that we do this without keeping a separate string for the raw
values, but I don't know how it will actually work in all cases, since raw
values need to normalize line endings anyways.

Ideas on that are welcome.

So the spec has concept of template head -> template middle -> template end ->
etc, but in practice I think there are only 2 cases that matter: the case where
you have a literal span followed by an expression, and the case where you have
just a literal span.

TEMPLATE_SPAN -> always followed by an expression (ends with ${), while
TEMPLATE_TAIL -> always followed by ` or the end of the template literal (if
not, then it's a bad token).

Basically, all of the NoSubstTemplate productions become TEMPLATE_TAIL, and all
of the other template productions become TEMPLATE_SPAN, I think it's simpler
this way

+  if (next_.token == Token::RBRACE) {
+    PushBack('}');
+  }
+  next_.location.beg_pos = source_pos();
+  Token::Value result = Token::ILLEGAL;
+  DCHECK(c0_ == '`' || c0_ == '}');
+  Advance();  // Consume ` or }
+
+  LiteralScope literal(this);
+  while (true) {
+    uc32 c = c0_;
+    Advance();
+    if (c == '`') {
+      result = Token::TEMPLATE_TAIL;
+      break;
+    } else if (c == '$' && c0_ == '{') {
+      Advance();  // Consume '{'
+      result = Token::TEMPLATE_SPAN;
+      break;
+    } else if (c == '\\') {
+      AddRawLiteralChar('\\');
+      if (unicode_cache_->IsLineTerminator(c0_)) {
+        // The TV of LineContinuation :: \ LineTerminatorSequence is the empty
+        // code unit sequence.
+        do {
+          uc32 lastChar = c0_;
+          Advance();
+          if (lastChar == '\r' && c0_ == '\n') Advance();
+          AddRawLiteralChar('\n');
+        } while (unicode_cache_->IsLineTerminator(c0_));
+      } else if (c0_ == '0') {
+        Advance();
+        AddRawLiteralChar('0');
+        AddLiteralChar('0');
+      } else {
+        ScanEscape(true);
+      }
+    } else if (c < 0) {
+      // Unterminated template literal
+      PushBack(c);
+      break;
+    } else {
+      // The TRV of LineTerminatorSequence :: <CR> is the CV 0x000A.
+      // The TRV of LineTerminatorSequence :: <CR><LF> is the sequence
+      // consisting of the CV 0x000A.
+      if (c == '\r') {
+        if (c0_ == '\n') Advance();
+        c = '\n';
+      }
+      AddLiteralChar(c);
+      AddRawLiteralChar(c);
+    }
+  }
+  literal.Complete();
+  next_.location.end_pos = source_pos();
+  next_.token = result;
+  return result;
+}
+
+
 void Scanner::ScanDecimalDigits() {
   while (IsDecimalDigit(c0_))
     AddLiteralCharAdvance();
@@ -1184,6 +1269,15 @@ const AstRawString* Scanner::CurrentSymbol(AstValueFactory* ast_value_factory) {
 }
 
 
+const AstRawString* Scanner::CurrentRawSymbol(
+    AstValueFactory* ast_value_factory) {
+  if (is_raw_one_byte()) {
+    return ast_value_factory->GetOneByteString(raw_one_byte_string());
+  }
+  return ast_value_factory->GetTwoByteString(raw_two_byte_string());
+}
+
+
 const AstRawString* Scanner::NextSymbol(AstValueFactory* ast_value_factory) {
   if (is_next_literal_one_byte()) {
     return ast_value_factory->GetOneByteString(next_literal_one_byte_string());
@@ -1192,6 +1286,14 @@ const AstRawString* Scanner::NextSymbol(AstValueFactory* ast_value_factory) {
 }
 
 
+const AstRawString* Scanner::NextRawSymbol(AstValueFactory* ast_value_factory) {
+  if (is_next_raw_one_byte()) {
+    return ast_value_factory->GetOneByteString(next_raw_one_byte_string());
+  }
+  return ast_value_factory->GetTwoByteString(next_raw_two_byte_string());
+}
+
+
 double Scanner::DoubleValue() {
   DCHECK(is_literal_one_byte());
   return StringToDouble(