Implement ES6 Template Literals

src/scanner.h

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // Features shared by parsing and pre-parsing scanners.
 
 #ifndef V8_SCANNER_H_
 #define V8_SCANNER_H_
 
 #include "src/allocation.h"
@@ skipping 332 matching lines @@
     bool IsValid() const {
       return beg_pos >= 0 && end_pos >= beg_pos;
     }
 
     static Location invalid() { return Location(-1, -1); }
 
     int beg_pos;
     int end_pos;
   };
 
+  enum Mode { None, TemplateLiteral };
+
   // -1 is outside of the range of any real source code.
   static const int kNoOctalLocation = -1;
 
   explicit Scanner(UnicodeCache* scanner_contants);
 
   void Initialize(Utf16CharacterStream* source);
 
   // Returns the next token and advances input.
-  Token::Value Next();
+  Token::Value Next(Mode mode = None);

[arv (Not doing code reviews), 2014/10/27 18:14:06]

I'm not sure you want to use a mode here. Once you are in a template literal you
can just call the right function, similar to how we handle regular expressions.

[caitp (gmail), 2014/10/27 20:22:36]

Done.

SpiderMonkey uses the mode strategy for this, but it's true that it's not really
needed, so it's gone.

   // Returns the current token again.
   Token::Value current_token() { return current_.token; }
   // Returns the location information for the current token
   // (the token last returned by Next()).
   Location location() const { return current_.location; }
 
   // Similar functions for the upcoming token.
 
   // One token look-ahead (past the token returned by Next()).
   Token::Value peek() const { return next_.token; }
@@ skipping 12 matching lines @@
   bool is_literal_contextual_keyword(Vector<const char> keyword) {
     DCHECK_NOT_NULL(current_.literal_chars);
     return current_.literal_chars->is_contextual_keyword(keyword);
   }
   bool is_next_contextual_keyword(Vector<const char> keyword) {
     DCHECK_NOT_NULL(next_.literal_chars);
     return next_.literal_chars->is_contextual_keyword(keyword);
   }
 
   const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory);
+  const AstRawString* CurrentRawSymbol(AstValueFactory* ast_value_factory);
   const AstRawString* NextSymbol(AstValueFactory* ast_value_factory);
+  const AstRawString* NextRawSymbol(AstValueFactory* ast_value_factory);
 
   double DoubleValue();
   bool LiteralMatches(const char* data, int length, bool allow_escapes = true) {
     if (is_literal_one_byte() &&
         literal_length() == length &&
         (allow_escapes || !literal_contains_escapes())) {
       const char* token =
           reinterpret_cast<const char*>(literal_one_byte_string().start());
       return !strncmp(token, data, length);
     }
@@ skipping 46 matching lines @@
   }
   void SetHarmonyNumericLiterals(bool numeric_literals) {
     harmony_numeric_literals_ = numeric_literals;
   }
   bool HarmonyClasses() const {
     return harmony_classes_;
   }
   void SetHarmonyClasses(bool classes) {
     harmony_classes_ = classes;
   }
+  bool HarmonyTemplates() const { return harmony_templates_; }
+  void SetHarmonyTemplates(bool templates) { harmony_templates_ = templates; }
 
   // Returns true if there was a line terminator before the peek'ed token,
   // possibly inside a multi-line comment.
   bool HasAnyLineTerminatorBeforeNext() const {
     return has_line_terminator_before_next_ ||
            has_multiline_comment_before_next_;
   }
 
   // Scans the input as a regular expression pattern, previous
   // character(s) must be /(=). Returns true if a pattern is scanned.
   bool ScanRegExpPattern(bool seen_equal);
   // Returns true if regexp flags are scanned (always since flags can
   // be empty).
   bool ScanRegExpFlags();
 
   const LiteralBuffer* source_url() const { return &source_url_; }
   const LiteralBuffer* source_mapping_url() const {
     return &source_mapping_url_;
   }
 
   bool IdentifierIsFutureStrictReserved(const AstRawString* string) const;
 
  private:
   // The current and look-ahead token.
   struct TokenDesc {
     Token::Value token;
     Location location;
     LiteralBuffer* literal_chars;
+    LiteralBuffer* raw_literal_chars;
   };
 
   static const int kCharacterLookaheadBufferSize = 1;
 
   // Scans octal escape sequence. Also accepts "\0" decimal escape sequence.
-  uc32 ScanOctalEscape(uc32 c, int length);
+  uc32 ScanOctalEscape(uc32 c, int length, bool recordRaw = false);
 
   // Call this after setting source_ to the input.
   void Init() {
     // Set c0_ (one character ahead)
     STATIC_ASSERT(kCharacterLookaheadBufferSize == 1);
     Advance();
     // Initialize current_ to not refer to a literal.
     current_.literal_chars = NULL;
+    current_.raw_literal_chars = NULL;
   }
 
   // Literal buffer support
   inline void StartLiteral() {
     LiteralBuffer* free_buffer = (current_.literal_chars == &literal_buffer1_) ?
             &literal_buffer2_ : &literal_buffer1_;
+    LiteralBuffer* raw_buffer = (current_.raw_literal_chars == &raw_buffer1_)
+                                    ? &raw_buffer2_
+                                    : &raw_buffer1_;
     free_buffer->Reset();
     next_.literal_chars = free_buffer;
+    next_.raw_literal_chars = raw_buffer;
   }
 
   INLINE(void AddLiteralChar(uc32 c)) {
     DCHECK_NOT_NULL(next_.literal_chars);
     next_.literal_chars->AddChar(c);
   }
 
+  INLINE(void AddRawLiteralChar(uc32 c)) {
+    DCHECK_NOT_NULL(next_.raw_literal_chars);
+    next_.raw_literal_chars->AddChar(c);
+  }
+
   // Complete scanning of a literal.
   inline void TerminateLiteral() {
     // Does nothing in the current implementation.
   }
 
   // Stops scanning of a literal and drop the collected characters,
   // e.g., due to an encountered error.
   inline void DropLiteral() {
     next_.literal_chars = NULL;
   }
@@ skipping 48 matching lines @@
   // These functions only give the correct result if the literal
   // was scanned between calls to StartLiteral() and TerminateLiteral().
   Vector<const uint8_t> literal_one_byte_string() {
     DCHECK_NOT_NULL(current_.literal_chars);
     return current_.literal_chars->one_byte_literal();
   }
   Vector<const uint16_t> literal_two_byte_string() {
     DCHECK_NOT_NULL(current_.literal_chars);
     return current_.literal_chars->two_byte_literal();
   }
+  Vector<const uint8_t> raw_one_byte_string() {
+    DCHECK_NOT_NULL(current_.raw_literal_chars);
+    return current_.raw_literal_chars->one_byte_literal();
+  }
+  Vector<const uint16_t> raw_two_byte_string() {
+    DCHECK_NOT_NULL(current_.raw_literal_chars);
+    return current_.raw_literal_chars->two_byte_literal();
+  }
   bool is_literal_one_byte() {
     DCHECK_NOT_NULL(current_.literal_chars);
     return current_.literal_chars->is_one_byte();
   }
+  bool is_raw_one_byte() {
+    DCHECK_NOT_NULL(current_.raw_literal_chars);
+    return current_.raw_literal_chars->is_one_byte();
+  }
   int literal_length() const {
     DCHECK_NOT_NULL(current_.literal_chars);
     return current_.literal_chars->length();
   }
   // Returns the literal string for the next token (the token that
   // would be returned if Next() were called).
   Vector<const uint8_t> next_literal_one_byte_string() {
     DCHECK_NOT_NULL(next_.literal_chars);
     return next_.literal_chars->one_byte_literal();
   }
   Vector<const uint16_t> next_literal_two_byte_string() {
     DCHECK_NOT_NULL(next_.literal_chars);
     return next_.literal_chars->two_byte_literal();
   }
+  Vector<const uint8_t> next_raw_one_byte_string() {
+    DCHECK_NOT_NULL(next_.raw_literal_chars);
+    return next_.raw_literal_chars->one_byte_literal();
+  }
+  Vector<const uint16_t> next_raw_two_byte_string() {
+    DCHECK_NOT_NULL(next_.raw_literal_chars);
+    return next_.raw_literal_chars->two_byte_literal();
+  }
   bool is_next_literal_one_byte() {
     DCHECK_NOT_NULL(next_.literal_chars);
     return next_.literal_chars->is_one_byte();
   }
+  bool is_next_raw_one_byte() {
+    DCHECK_NOT_NULL(next_.raw_literal_chars);
+    return next_.raw_literal_chars->is_one_byte();
+  }
   int next_literal_length() const {
     DCHECK_NOT_NULL(next_.literal_chars);
     return next_.literal_chars->length();
   }
 
-  uc32 ScanHexNumber(int expected_length);
+  uc32 ScanHexNumber(int expected_length, bool recordRaw = false);
 
   // Scans a single JavaScript token.
-  void Scan();
+  void Scan(Mode mode = None);
 
   bool SkipWhiteSpace();
   Token::Value SkipSingleLineComment();
   Token::Value SkipSourceURLComment();
   void TryToParseSourceURLComment();
   Token::Value SkipMultiLineComment();
   // Scans a possible HTML comment -- begins with '<!'.
   Token::Value ScanHtmlComment();
 
   void ScanDecimalDigits();
   Token::Value ScanNumber(bool seen_period);
   Token::Value ScanIdentifierOrKeyword();
   Token::Value ScanIdentifierSuffix(LiteralScope* literal);
 
   Token::Value ScanString();
+  Token::Value ScanTemplateSpan();
 
   // Scans an escape-sequence which is part of a string and adds the
   // decoded character to the current literal. Returns true if a pattern
   // is scanned.
-  bool ScanEscape();
+  bool ScanEscape(bool recordRaw = false);
   // Decodes a Unicode escape-sequence which is part of an identifier.
   // If the escape sequence cannot be decoded the result is kBadChar.
   uc32 ScanIdentifierUnicodeEscape();
   // Scans a Unicode escape-sequence and adds its characters,
   // uninterpreted, to the current literal. Used for parsing RegExp
   // flags.
   bool ScanLiteralUnicodeEscape();
 
   // Return the current source position.
   int source_pos() {
     return source_->pos() - kCharacterLookaheadBufferSize;
   }
 
   UnicodeCache* unicode_cache_;
 
   // Buffers collecting literal strings, numbers, etc.
   LiteralBuffer literal_buffer1_;
   LiteralBuffer literal_buffer2_;
 
+  // Buffer to store raw string values
+  LiteralBuffer raw_buffer1_;
+  LiteralBuffer raw_buffer2_;
+
   // Values parsed from magic comments.
   LiteralBuffer source_url_;
   LiteralBuffer source_mapping_url_;
 
   TokenDesc current_;  // desc for current token (as returned by Next())
   TokenDesc next_;     // desc for next token (one token look-ahead)
 
   // Input stream. Must be initialized to an Utf16CharacterStream.
   Utf16CharacterStream* source_;
 
@@ skipping 12 matching lines @@
   // line-terminator after the current token, and before the next.
   bool has_multiline_comment_before_next_;
   // Whether we scan 'let' as a keyword for harmony block-scoped let bindings.
   bool harmony_scoping_;
   // Whether we scan 'module', 'import', 'export' as keywords.
   bool harmony_modules_;
   // Whether we scan 0o777 and 0b111 as numbers.
   bool harmony_numeric_literals_;
   // Whether we scan 'class', 'extends', 'static' and 'super' as keywords.
   bool harmony_classes_;
+  // Whether we scan TEMPLATE_SPAN and TEMPLATE_TAIL
+  bool harmony_templates_;
 };
 
 } }  // namespace v8::internal
 
 #endif  // V8_SCANNER_H_