Implement ES6 Template Literals

src/scanner.h

Index: src/scanner.h
diff --git a/src/scanner.h b/src/scanner.h
index 387d3319c167c014177499e90698047e2b4364ae..d02909e213616af30eedbc46e016078ee2dca4b8 100644
--- a/src/scanner.h
+++ b/src/scanner.h
@@ -350,6 +350,8 @@ class Scanner {
     int end_pos;
   };
 
+  enum Mode { None, TemplateLiteral };
+
   // -1 is outside of the range of any real source code.
   static const int kNoOctalLocation = -1;
 
@@ -358,7 +360,7 @@ class Scanner {
   void Initialize(Utf16CharacterStream* source);
 
   // Returns the next token and advances input.
-  Token::Value Next();
+  Token::Value Next(Mode mode = None);

[arv (Not doing code reviews), 2014/10/27 18:14:06]

I'm not sure you want to use a mode here. Once you are in a template literal you
can just call the right function, similar to how we handle regular expressions.

[caitp (gmail), 2014/10/27 20:22:36]

Done.

SpiderMonkey uses the mode strategy for this, but it's true that it's not really
needed, so it's gone.

   // Returns the current token again.
   Token::Value current_token() { return current_.token; }
   // Returns the location information for the current token
@@ -391,7 +393,9 @@ class Scanner {
   }
 
   const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory);
+  const AstRawString* CurrentRawSymbol(AstValueFactory* ast_value_factory);
   const AstRawString* NextSymbol(AstValueFactory* ast_value_factory);
+  const AstRawString* NextRawSymbol(AstValueFactory* ast_value_factory);
 
   double DoubleValue();
   bool LiteralMatches(const char* data, int length, bool allow_escapes = true) {
@@ -458,6 +462,8 @@ class Scanner {
   void SetHarmonyClasses(bool classes) {
     harmony_classes_ = classes;
   }
+  bool HarmonyTemplates() const { return harmony_templates_; }
+  void SetHarmonyTemplates(bool templates) { harmony_templates_ = templates; }
 
   // Returns true if there was a line terminator before the peek'ed token,
   // possibly inside a multi-line comment.
@@ -486,12 +492,13 @@ class Scanner {
     Token::Value token;
     Location location;
     LiteralBuffer* literal_chars;
+    LiteralBuffer* raw_literal_chars;
   };
 
   static const int kCharacterLookaheadBufferSize = 1;
 
   // Scans octal escape sequence. Also accepts "\0" decimal escape sequence.
-  uc32 ScanOctalEscape(uc32 c, int length);
+  uc32 ScanOctalEscape(uc32 c, int length, bool recordRaw = false);
 
   // Call this after setting source_ to the input.
   void Init() {
@@ -500,14 +507,19 @@ class Scanner {
     Advance();
     // Initialize current_ to not refer to a literal.
     current_.literal_chars = NULL;
+    current_.raw_literal_chars = NULL;
   }
 
   // Literal buffer support
   inline void StartLiteral() {
     LiteralBuffer* free_buffer = (current_.literal_chars == &literal_buffer1_) ?
             &literal_buffer2_ : &literal_buffer1_;
+    LiteralBuffer* raw_buffer = (current_.raw_literal_chars == &raw_buffer1_)
+                                    ? &raw_buffer2_
+                                    : &raw_buffer1_;
     free_buffer->Reset();
     next_.literal_chars = free_buffer;
+    next_.raw_literal_chars = raw_buffer;
   }
 
   INLINE(void AddLiteralChar(uc32 c)) {
@@ -515,6 +527,11 @@ class Scanner {
     next_.literal_chars->AddChar(c);
   }
 
+  INLINE(void AddRawLiteralChar(uc32 c)) {
+    DCHECK_NOT_NULL(next_.raw_literal_chars);
+    next_.raw_literal_chars->AddChar(c);
+  }
+
   // Complete scanning of a literal.
   inline void TerminateLiteral() {
     // Does nothing in the current implementation.
@@ -583,10 +600,22 @@ class Scanner {
     DCHECK_NOT_NULL(current_.literal_chars);
     return current_.literal_chars->two_byte_literal();
   }
+  Vector<const uint8_t> raw_one_byte_string() {
+    DCHECK_NOT_NULL(current_.raw_literal_chars);
+    return current_.raw_literal_chars->one_byte_literal();
+  }
+  Vector<const uint16_t> raw_two_byte_string() {
+    DCHECK_NOT_NULL(current_.raw_literal_chars);
+    return current_.raw_literal_chars->two_byte_literal();
+  }
   bool is_literal_one_byte() {
     DCHECK_NOT_NULL(current_.literal_chars);
     return current_.literal_chars->is_one_byte();
   }
+  bool is_raw_one_byte() {
+    DCHECK_NOT_NULL(current_.raw_literal_chars);
+    return current_.raw_literal_chars->is_one_byte();
+  }
   int literal_length() const {
     DCHECK_NOT_NULL(current_.literal_chars);
     return current_.literal_chars->length();
@@ -601,19 +630,31 @@ class Scanner {
     DCHECK_NOT_NULL(next_.literal_chars);
     return next_.literal_chars->two_byte_literal();
   }
+  Vector<const uint8_t> next_raw_one_byte_string() {
+    DCHECK_NOT_NULL(next_.raw_literal_chars);
+    return next_.raw_literal_chars->one_byte_literal();
+  }
+  Vector<const uint16_t> next_raw_two_byte_string() {
+    DCHECK_NOT_NULL(next_.raw_literal_chars);
+    return next_.raw_literal_chars->two_byte_literal();
+  }
   bool is_next_literal_one_byte() {
     DCHECK_NOT_NULL(next_.literal_chars);
     return next_.literal_chars->is_one_byte();
   }
+  bool is_next_raw_one_byte() {
+    DCHECK_NOT_NULL(next_.raw_literal_chars);
+    return next_.raw_literal_chars->is_one_byte();
+  }
   int next_literal_length() const {
     DCHECK_NOT_NULL(next_.literal_chars);
     return next_.literal_chars->length();
   }
 
-  uc32 ScanHexNumber(int expected_length);
+  uc32 ScanHexNumber(int expected_length, bool recordRaw = false);
 
   // Scans a single JavaScript token.
-  void Scan();
+  void Scan(Mode mode = None);
 
   bool SkipWhiteSpace();
   Token::Value SkipSingleLineComment();
@@ -629,11 +670,12 @@ class Scanner {
   Token::Value ScanIdentifierSuffix(LiteralScope* literal);
 
   Token::Value ScanString();
+  Token::Value ScanTemplateSpan();
 
   // Scans an escape-sequence which is part of a string and adds the
   // decoded character to the current literal. Returns true if a pattern
   // is scanned.
-  bool ScanEscape();
+  bool ScanEscape(bool recordRaw = false);
   // Decodes a Unicode escape-sequence which is part of an identifier.
   // If the escape sequence cannot be decoded the result is kBadChar.
   uc32 ScanIdentifierUnicodeEscape();
@@ -653,6 +695,10 @@ class Scanner {
   LiteralBuffer literal_buffer1_;
   LiteralBuffer literal_buffer2_;
 
+  // Buffer to store raw string values
+  LiteralBuffer raw_buffer1_;
+  LiteralBuffer raw_buffer2_;
+
   // Values parsed from magic comments.
   LiteralBuffer source_url_;
   LiteralBuffer source_mapping_url_;
@@ -685,6 +731,8 @@ class Scanner {
   bool harmony_numeric_literals_;
   // Whether we scan 'class', 'extends', 'static' and 'super' as keywords.
   bool harmony_classes_;
+  // Whether we scan TEMPLATE_SPAN and TEMPLATE_TAIL
+  bool harmony_templates_;
 };
 
 } }  // namespace v8::internal