Fix raw TemplateLiteral spans with non-ascii characters

src/parser.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/v8.h"
 
 #include "src/api.h"
 #include "src/ast.h"
 #include "src/bailout-reason.h"
 #include "src/base/platform/platform.h"
@@ skipping 5318 matching lines @@
       if (ch == '\r') {
         ch = '\n';
         if (from_index + 1 < length && raw_chars[from_index + 1] == '\n') {
           ++from_index;
         }
       }
       hash_string[num_hash_chars++] = ch;
       raw_chars[to_index++] = ch;
     }
 
-    const AstRawString* raw_str = ast_value_factory()->GetOneByteString(
-        OneByteVector(raw_chars.get(), to_index));
-    Literal* raw_lit = factory()->NewStringLiteral(raw_str, span_start - 1);
+    Access<UnicodeCache::Utf8Decoder>
+        decoder(isolate()->unicode_cache()->utf8_decoder());
+    decoder->Reset(raw_chars.get(), to_index);

[marja, 2014/12/01 15:53:59]

Since we're now doing parsing on a background thread in some use cases, Parser
should remain independent of the Isolate. It gets passed a ParseInfo which
contains the UnicodeCache to use, you can use it here (via Scanner I guess).

There are guards in place to make sure the background parsing cases don't mess
with the heap, but sadly there are no guards to ensure we don't access the
Isolate in non-thread-safe ways. :/

[Dmitry Lomov (no reviews), 2014/12/01 16:09:48]

Oops, sorry, forgot about that! Good catch!

+    int utf16_length = decoder->Utf16Length();
+    Literal* raw_lit = NULL;
+    if (utf16_length > 0) {
+      uc16* utf16_buffer = zone()->NewArray<uc16>(utf16_length);
+      to_index = decoder->WriteUtf16(utf16_buffer, utf16_length);
+      const uint16_t* data = reinterpret_cast<const uint16_t*>(utf16_buffer);
+      const AstRawString* raw_str = ast_value_factory()->GetTwoByteString(
+          Vector<const uint16_t>(data, to_index));
+      raw_lit = factory()->NewStringLiteral(raw_str, span_start - 1);
+    } else {
+      raw_lit = factory()->NewStringLiteral(
+          ast_value_factory()->empty_string(), span_start - 1);
+    }
+    DCHECK_NOT_NULL(raw_lit);
     raw_strings->Add(raw_lit, zone());
   }
 
   hash_string.Truncate(num_hash_chars);
   int utf16_length;
   *hash = StringHasher::ComputeUtf8Hash(Vector<const char>::cast(hash_string),
       num_hash_chars, &utf16_length);
   hash_string.Dispose();
 
   return raw_strings;
 }
 } }  // namespace v8::internal