Implement ES6 Template Literals

src/scanner.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // Features shared by parsing and pre-parsing scanners.
 
 #include <stdint.h>
 
 #include <cmath>
 
@@ skipping 36 matching lines @@
   // in object literals.
   Init();
   // Skip initial whitespace allowing HTML comment ends just like
   // after a newline and scan first token.
   has_line_terminator_before_next_ = true;
   SkipWhiteSpace();
   Scan();
 }
 
 
-uc32 Scanner::ScanHexNumber(int expected_length) {
+uc32 Scanner::ScanHexNumber(int expected_length, bool recordRaw) {
   DCHECK(expected_length <= 4);  // prevent overflow
 
   uc32 digits[4] = { 0, 0, 0, 0 };
   uc32 x = 0;
   for (int i = 0; i < expected_length; i++) {
     digits[i] = c0_;
     int d = HexValue(c0_);
     if (d < 0) {
       // According to ECMA-262, 3rd, 7.8.4, page 18, these hex escapes
       // should be illegal, but other JS VMs just return the
       // non-escaped version of the original character.
 
       // Push back digits that we have advanced past.
       for (int j = i-1; j >= 0; j--) {
         PushBack(digits[j]);
       }
       return -1;
     }
     x = x * 16 + d;
+    if (recordRaw) {
+      AddRawLiteralChar(c0_);
+    }
     Advance();
   }
 
   return x;
 }
 
 
 // Ensure that tokens can be stored in a byte.
 STATIC_ASSERT(Token::NUM_TOKENS <= 0x100);
 
@@ skipping 123 matching lines @@
   Token::ILLEGAL,
   Token::ILLEGAL,
   Token::LBRACE,       // 0x7b
   Token::ILLEGAL,
   Token::RBRACE,       // 0x7d
   Token::BIT_NOT,      // 0x7e
   Token::ILLEGAL
 };
 
 
-Token::Value Scanner::Next() {
+Token::Value Scanner::Next(Mode mode) {
   current_ = next_;
   has_line_terminator_before_next_ = false;
   has_multiline_comment_before_next_ = false;
-  if (static_cast<unsigned>(c0_) <= 0x7f) {
+  if (mode != TemplateLiteral && static_cast<unsigned>(c0_) <= 0x7f) {
     Token::Value token = static_cast<Token::Value>(one_char_tokens[c0_]);
     if (token != Token::ILLEGAL) {
       int pos = source_pos();
       next_.token = token;
       next_.location.beg_pos = pos;
       next_.location.end_pos = pos + 1;
       Advance();
       return current_.token;
     }
   }
-  Scan();
+  Scan(mode);
+  if (mode == TemplateLiteral && current_.token == Token::RBRACE) {
+    // The current token is now invalid
+    return Next();
+  }
   return current_.token;
 }
 
 
 // TODO(yangguo): check whether this is actually necessary.
 static inline bool IsLittleEndianByteOrderMark(uc32 c) {
   // The Unicode value U+FFFE is guaranteed never to be assigned as a
   // Unicode character; this implies that in a Unicode context the
   // 0xFF, 0xFE byte pattern can only be interpreted as the U+FEFF
   // character expressed in little-endian byte order (since it could
@@ skipping 156 matching lines @@
     Advance();
     if (c0_ == '-') return SkipSingleLineComment();
     PushBack('-');  // undo Advance()
   }
   PushBack('!');  // undo Advance()
   DCHECK(c0_ == '!');
   return Token::LT;
 }
 
 
-void Scanner::Scan() {
+void Scanner::Scan(Scanner::Mode mode) {
   next_.literal_chars = NULL;
+  next_.raw_literal_chars = NULL;
   Token::Value token;
+
+  if (mode == TemplateLiteral) {
+    CHECK(HarmonyTemplates());
+
+    // If we have an RBRACE next, PushBack a `}`, this should be the start of a
+    // TemplateMiddle span
+    if (peek() == Token::RBRACE) {
+      PushBack('}');
+    }
+
+    token = ScanTemplateSpan();
+    next_.location.end_pos = source_pos();
+    next_.token = token;
+    return;
+  }
+
   do {
     // Remember the position of the next token
     next_.location.beg_pos = source_pos();
 
     switch (c0_) {
       case ' ':
       case '\t':
         Advance();
         token = Token::WHITESPACE;
         break;
@@ skipping 202 matching lines @@
         break;
 
       case '?':
         token = Select(Token::CONDITIONAL);
         break;
 
       case '~':
         token = Select(Token::BIT_NOT);
         break;
 
+      case '`':
+        if (HarmonyTemplates()) {
+          token = ScanTemplateSpan();
+          break;
+        }
+
       default:
         if (unicode_cache_->IsIdentifierStart(c0_)) {
           token = ScanIdentifierOrKeyword();
         } else if (IsDecimalDigit(c0_)) {
           token = ScanNumber(false);
         } else if (SkipWhiteSpace()) {
           token = Token::WHITESPACE;
         } else if (c0_ < 0) {
           token = Token::EOS;
         } else {
@@ skipping 25 matching lines @@
     // This function is only called to seek to the location
     // of the end of a function (at the "}" token). It doesn't matter
     // whether there was a line terminator in the part we skip.
     has_line_terminator_before_next_ = false;
     has_multiline_comment_before_next_ = false;
   }
   Scan();
 }
 
 
-bool Scanner::ScanEscape() {
+bool Scanner::ScanEscape(bool recordRaw) {
   uc32 c = c0_;
   Advance();
 
   // Skip escaped newlines.
   if (unicode_cache_->IsLineTerminator(c)) {
     // Allow CR+LF newlines in multiline string literals.
     if (IsCarriageReturn(c) && IsLineFeed(c0_)) Advance();
     // Allow LF+CR newlines in multiline string literals.
     if (IsLineFeed(c) && IsCarriageReturn(c0_)) Advance();
     return true;
   }
 
   switch (c) {
     case '\'':  // fall through
     case '"' :  // fall through
     case '\\': break;
     case 'b' : c = '\b'; break;
     case 'f' : c = '\f'; break;
     case 'n' : c = '\n'; break;
     case 'r' : c = '\r'; break;
     case 't' : c = '\t'; break;
     case 'u' : {
-      c = ScanHexNumber(4);
+      c = ScanHexNumber(4, recordRaw);
       if (c < 0) return false;
       break;
     }
     case 'v' : c = '\v'; break;
     case 'x' : {
-      c = ScanHexNumber(2);
+      c = ScanHexNumber(2, recordRaw);
       if (c < 0) return false;
       break;
     }
     case '0' :  // fall through
     case '1' :  // fall through
     case '2' :  // fall through
     case '3' :  // fall through
     case '4' :  // fall through
     case '5' :  // fall through
     case '6' :  // fall through
-    case '7' : c = ScanOctalEscape(c, 2); break;
+    case '7':
+      c = ScanOctalEscape(c, 2, recordRaw);
+      break;
   }
 
   // According to ECMA-262, section 7.8.4, characters not covered by the
   // above cases should be illegal, but they are commonly handled as
   // non-escaped characters by JS VMs.
   AddLiteralChar(c);
   return true;
 }
 
 
 // Octal escapes of the forms '\0xx' and '\xxx' are not a part of
 // ECMA-262. Other JS VMs support them.
-uc32 Scanner::ScanOctalEscape(uc32 c, int length) {
+uc32 Scanner::ScanOctalEscape(uc32 c, int length, bool recordRaw) {
   uc32 x = c - '0';
   int i = 0;
   for (; i < length; i++) {
     int d = c0_ - '0';
     if (d < 0 || d > 7) break;
     int nx = x * 8 + d;
     if (nx >= 256) break;
     x = nx;
+    if (recordRaw) {
+      AddRawLiteralChar(c0_);
+    }
     Advance();
   }
   // Anything except '\0' is an octal escape sequence, illegal in strict mode.
   // Remember the position of octal escape sequences so that an error
   // can be reported later (in strict mode).
   // We don't report the error immediately, because the octal escape can
   // occur before the "use strict" directive.
   if (c != '0' || i > 0) {
     octal_pos_ = Location(source_pos() - i - 1, source_pos() - 1);
   }
@@ skipping 17 matching lines @@
     }
   }
   if (c0_ != quote) return Token::ILLEGAL;
   literal.Complete();
 
   Advance();  // consume quote
   return Token::STRING;
 }
 
 
+Token::Value Scanner::ScanTemplateSpan() {
+  DCHECK(c0_ == '`' || c0_ == '}');
+  Advance();  // Consume ` or }
+  LiteralScope literal(this);
+  while (true) {
+    uc32 c = c0_;
+    Advance();
+    if (c == '`') {
+      literal.Complete();
+      return Token::TEMPLATE_TAIL;
+    } else if (c == '$' && c0_ == '{') {
+      Advance();  // Consume '{'
+      literal.Complete();
+      return Token::TEMPLATE_SPAN;
+    } else if (c == '\\') {
+      if (unicode_cache_->IsLineTerminator(c0_)) {
+        // The TV of LineContinuation :: \ LineTerminatorSequence is the empty
+        // code unit sequence.
+        do {
+          uc32 lastChar = c0_;
+          Advance();
+          if (lastChar == '\r' && c0_ == '\n') Advance();
+          AddRawLiteralChar('\n');
+        } while (unicode_cache_->IsLineTerminator(c0_));
+      } else {
+        ScanEscape();
+      }
+    } else if (c < 0) {
+      // Unterminated template literal
+      literal.Complete();
+      PushBack(c);
+      return Token::ILLEGAL;
+    } else {
+      AddLiteralChar(c);
+      AddRawLiteralChar(c);
+    }
+  }
+}
+
+
 void Scanner::ScanDecimalDigits() {
   while (IsDecimalDigit(c0_))
     AddLiteralCharAdvance();
 }
 
 
 Token::Value Scanner::ScanNumber(bool seen_period) {
   DCHECK(IsDecimalDigit(c0_));  // the first digit of the number or the fraction
 
   enum { DECIMAL, HEX, OCTAL, IMPLICIT_OCTAL, BINARY } kind = DECIMAL;
@@ skipping 403 matching lines @@
 
 
 const AstRawString* Scanner::CurrentSymbol(AstValueFactory* ast_value_factory) {
   if (is_literal_one_byte()) {
     return ast_value_factory->GetOneByteString(literal_one_byte_string());
   }
   return ast_value_factory->GetTwoByteString(literal_two_byte_string());
 }
 
 
+const AstRawString* Scanner::CurrentRawSymbol(
+    AstValueFactory* ast_value_factory) {
+  if (is_raw_one_byte()) {
+    return ast_value_factory->GetOneByteString(raw_one_byte_string());
+  }
+  return ast_value_factory->GetTwoByteString(raw_two_byte_string());
+}
+
+
 const AstRawString* Scanner::NextSymbol(AstValueFactory* ast_value_factory) {
   if (is_next_literal_one_byte()) {
     return ast_value_factory->GetOneByteString(next_literal_one_byte_string());
   }
   return ast_value_factory->GetTwoByteString(next_literal_two_byte_string());
 }
 
 
+const AstRawString* Scanner::NextRawSymbol(AstValueFactory* ast_value_factory) {
+  if (is_next_raw_one_byte()) {
+    return ast_value_factory->GetOneByteString(next_raw_one_byte_string());
+  }
+  return ast_value_factory->GetTwoByteString(next_raw_two_byte_string());
+}
+
+
 double Scanner::DoubleValue() {
   DCHECK(is_literal_one_byte());
   return StringToDouble(
       unicode_cache_,
       literal_one_byte_string(),
       ALLOW_HEX | ALLOW_OCTAL | ALLOW_IMPLICIT_OCTAL | ALLOW_BINARY);
 }
 
 
 int Scanner::FindNumber(DuplicateFinder* finder, int value) {
@@ skipping 142 matching lines @@
     }
     backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u));
   }
   backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f));
 
   backing_store_.AddBlock(bytes);
   return backing_store_.EndSequence().start();
 }
 
 } }  // namespace v8::internal