Simplify template literal raw string creation

src/scanner.h

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // Features shared by parsing and pre-parsing scanners.
 
 #ifndef V8_SCANNER_H_
 #define V8_SCANNER_H_
 
 #include "src/allocation.h"
@@ skipping 234 matching lines @@
     DCHECK(is_one_byte_);
     return Vector<const uint8_t>(
         reinterpret_cast<const uint8_t*>(backing_store_.start()),
         position_);
   }
 
   int length() const {
     return is_one_byte_ ? position_ : (position_ >> 1);
   }
 
+  void ReduceLength(int delta) {
+    position_ -= delta * (is_one_byte_ ? kOneByteSize : kUC16Size);
+  }
+
   void Reset() {
     position_ = 0;
     is_one_byte_ = true;
   }
 
   Handle<String> Internalize(Isolate* isolate) const;
 
  private:
   static const int kInitialCapacity = 16;
   static const int kGrowthFactory = 4;
@@ skipping 46 matching lines @@
 
 // ----------------------------------------------------------------------------
 // JavaScript Scanner.
 
 class Scanner {
  public:
   // Scoped helper for literal recording. Automatically drops the literal
   // if aborting the scanning before it's complete.
   class LiteralScope {
    public:
-    explicit LiteralScope(Scanner* self)
+    explicit LiteralScope(Scanner* self, bool capture_raw = false)
         : scanner_(self), complete_(false) {
       scanner_->StartLiteral();
+      if (capture_raw) scanner_->StartRawLiteral();
     }
      ~LiteralScope() {
        if (!complete_) scanner_->DropLiteral();
      }
     void Complete() {
       scanner_->TerminateLiteral();
       complete_ = true;
     }
 
    private:
@@ skipping 51 matching lines @@
     DCHECK_NOT_NULL(current_.literal_chars);
     return current_.literal_chars->is_contextual_keyword(keyword);
   }
   bool is_next_contextual_keyword(Vector<const char> keyword) {
     DCHECK_NOT_NULL(next_.literal_chars);
     return next_.literal_chars->is_contextual_keyword(keyword);
   }
 
   const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory);
   const AstRawString* NextSymbol(AstValueFactory* ast_value_factory);
+  const AstRawString* CurrentRawSymbol(AstValueFactory* ast_value_factory);
 
   double DoubleValue();
   bool LiteralMatches(const char* data, int length, bool allow_escapes = true) {
     if (is_literal_one_byte() &&
         literal_length() == length &&
         (allow_escapes || !literal_contains_escapes())) {
       const char* token =
           reinterpret_cast<const char*>(literal_one_byte_string().start());
       return !strncmp(token, data, length);
     }
@@ skipping 79 matching lines @@
   }
 
   bool IdentifierIsFutureStrictReserved(const AstRawString* string) const;
 
  private:
   // The current and look-ahead token.
   struct TokenDesc {
     Token::Value token;
     Location location;
     LiteralBuffer* literal_chars;
+    LiteralBuffer* raw_literal_chars;
   };
 
   static const int kCharacterLookaheadBufferSize = 1;
 
   // Scans octal escape sequence. Also accepts "\0" decimal escape sequence.
   uc32 ScanOctalEscape(uc32 c, int length);
 
   // Call this after setting source_ to the input.
   void Init() {
     // Set c0_ (one character ahead)
     STATIC_ASSERT(kCharacterLookaheadBufferSize == 1);
     Advance();
     // Initialize current_ to not refer to a literal.
     current_.literal_chars = NULL;
+    current_.raw_literal_chars = NULL;
   }
 
   // Literal buffer support
   inline void StartLiteral() {
     LiteralBuffer* free_buffer = (current_.literal_chars == &literal_buffer1_) ?
             &literal_buffer2_ : &literal_buffer1_;
     free_buffer->Reset();
     next_.literal_chars = free_buffer;
   }
 
+  inline void StartRawLiteral() {
+    raw_literal_buffer_.Reset();
+    next_.raw_literal_chars = &raw_literal_buffer_;
+    capturing_raw_literal_ = true;
+  }
+
   INLINE(void AddLiteralChar(uc32 c)) {
     DCHECK_NOT_NULL(next_.literal_chars);
     next_.literal_chars->AddChar(c);
   }
 
+  INLINE(void AddRawLiteralChar(uc32 c)) {
+    DCHECK(capturing_raw_literal_);
+    DCHECK_NOT_NULL(next_.raw_literal_chars);
+    next_.raw_literal_chars->AddChar(c);
+  }
+
+  INLINE(void ReduceRawLiteralLength(int delta)) {
+    DCHECK(capturing_raw_literal_);
+    DCHECK_NOT_NULL(next_.raw_literal_chars);
+    next_.raw_literal_chars->ReduceLength(delta);
+  }
+
   // Complete scanning of a literal.
-  inline void TerminateLiteral() {
-    // Does nothing in the current implementation.
-  }
+  inline void TerminateLiteral() { capturing_raw_literal_ = false; }
 
   // Stops scanning of a literal and drop the collected characters,
   // e.g., due to an encountered error.
   inline void DropLiteral() {
     next_.literal_chars = NULL;
+    next_.raw_literal_chars = NULL;
+    capturing_raw_literal_ = false;
   }
 
   inline void AddLiteralCharAdvance() {
     AddLiteralChar(c0_);
     Advance();
   }
 
   // Low-level scanning support.
   void Advance() {
+    if (capturing_raw_literal_) {
+      AddRawLiteralChar(c0_);
+    }
     c0_ = source_->Advance();
     if (unibrow::Utf16::IsLeadSurrogate(c0_)) {
       uc32 c1 = source_->Advance();
       if (!unibrow::Utf16::IsTrailSurrogate(c1)) {
         source_->PushBack(c1);
       } else {
         c0_ = unibrow::Utf16::CombineSurrogatePair(c0_, c1);
       }
     }
   }
 
   void PushBack(uc32 ch) {
+    DCHECK(ch < 0 || !capturing_raw_literal_);

[caitp (gmail), 2014/12/02 21:37:35]

I feel like PushBack() is okay --- it's more just that you want to make sure
PushBack() is closely followed by a ReduceRawLiteralLength() if raw literals are
being captured, am I wrong about that?

     if (ch > static_cast<uc32>(unibrow::Utf16::kMaxNonSurrogateCharCode)) {
       source_->PushBack(unibrow::Utf16::TrailSurrogate(c0_));
       source_->PushBack(unibrow::Utf16::LeadSurrogate(c0_));
     } else {
       source_->PushBack(c0_);
     }
     c0_ = ch;
   }
 
   inline Token::Value Select(Token::Value tok) {
     Advance();
     return tok;
   }
 
   inline Token::Value Select(uc32 next, Token::Value then, Token::Value else_) {
     Advance();
     if (c0_ == next) {
       Advance();
       return then;
     } else {
       return else_;
     }
   }
 
   // Returns the literal string, if any, for the current token (the
   // token last returned by Next()). The string is 0-terminated.
-  // Literal strings are collected for identifiers, strings, and
-  // numbers.
+  // Literal strings are collected for identifiers, strings, numbers as well
+  // as for template literals. For template literals we also collect the raw
+  // form.
   // These functions only give the correct result if the literal
   // was scanned between calls to StartLiteral() and TerminateLiteral().
   Vector<const uint8_t> literal_one_byte_string() {
     DCHECK_NOT_NULL(current_.literal_chars);
     return current_.literal_chars->one_byte_literal();
   }
   Vector<const uint16_t> literal_two_byte_string() {
     DCHECK_NOT_NULL(current_.literal_chars);
     return current_.literal_chars->two_byte_literal();
   }
@@ skipping 12 matching lines @@
     return next_.literal_chars->one_byte_literal();
   }
   Vector<const uint16_t> next_literal_two_byte_string() {
     DCHECK_NOT_NULL(next_.literal_chars);
     return next_.literal_chars->two_byte_literal();
   }
   bool is_next_literal_one_byte() {
     DCHECK_NOT_NULL(next_.literal_chars);
     return next_.literal_chars->is_one_byte();
   }
-  int next_literal_length() const {
-    DCHECK_NOT_NULL(next_.literal_chars);
-    return next_.literal_chars->length();
+  Vector<const uint8_t> raw_literal_one_byte_string() {
+    DCHECK_NOT_NULL(current_.raw_literal_chars);
+    return current_.raw_literal_chars->one_byte_literal();
   }
+  Vector<const uint16_t> raw_literal_two_byte_string() {
+    DCHECK_NOT_NULL(current_.raw_literal_chars);
+    return current_.raw_literal_chars->two_byte_literal();
+  }
+  bool is_raw_literal_one_byte() {
+    DCHECK_NOT_NULL(current_.raw_literal_chars);
+    return current_.raw_literal_chars->is_one_byte();
+  }
+
 
   uc32 ScanHexNumber(int expected_length);
 
   // Scans a single JavaScript token.
   void Scan();
 
   bool SkipWhiteSpace();
   Token::Value SkipSingleLineComment();
   Token::Value SkipSourceURLComment();
   void TryToParseSourceURLComment();
@@ skipping 24 matching lines @@
   UnicodeCache* unicode_cache_;
 
   // Buffers collecting literal strings, numbers, etc.
   LiteralBuffer literal_buffer1_;
   LiteralBuffer literal_buffer2_;
 
   // Values parsed from magic comments.
   LiteralBuffer source_url_;
   LiteralBuffer source_mapping_url_;
 
+  // Buffer to store raw string values
+  LiteralBuffer raw_literal_buffer_;
+
+  // We only need to capture the raw literal when we are scanning template
+  // literal spans.
+  bool capturing_raw_literal_;
+
   TokenDesc current_;  // desc for current token (as returned by Next())
   TokenDesc next_;     // desc for next token (one token look-ahead)
 
   // Input stream. Must be initialized to an Utf16CharacterStream.
   Utf16CharacterStream* source_;
 
 
   // Start position of the octal literal last scanned.
   Location octal_pos_;
 
@@ skipping 15 matching lines @@
   bool harmony_numeric_literals_;
   // Whether we scan 'class', 'extends', 'static' and 'super' as keywords.
   bool harmony_classes_;
   // Whether we scan TEMPLATE_SPAN and TEMPLATE_TAIL
   bool harmony_templates_;
 };
 
 } }  // namespace v8::internal
 
 #endif  // V8_SCANNER_H_