Fix raw TemplateLiteral spans with non-ascii characters

src/parser.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/v8.h"
 
 #include "src/api.h"
 #include "src/ast.h"
 #include "src/bailout-reason.h"
 #include "src/base/platform/platform.h"
@@ skipping 5285 matching lines @@
   Handle<String> source(String::cast(script()->source()));
 
   raw_strings = new (zone()) ZoneList<Expression*>(total, zone());
 
   int num_hash_chars = (total - 1) * 3;
   for (int index = 0; index < total; ++index) {
     // Allow about length * 4 to handle most UTF8 sequences.
     num_hash_chars += lengths->at(index) * 4;
   }
 
   Vector<uint8_t> hash_string = Vector<uint8_t>::New(num_hash_chars);

[arv (Not doing code reviews), 2014/11/21 06:12:49]

One thing that seems strange to me is that we have to assemble a large string to
compute the hash. The hash  function just iterates over the characters in that
string.

Conceptually we should be able to compute the hash span by span. Each span is
either a one byte string or a two byte string. We can compute the hash for these
using a rolling hash.

Looking into the source code here (this is new territory for me too) it looks
like we can use a StringHasher and then call AddCharacters on it (which has one
version for one byte string and one for two byte strings) and then finally get
the hash code out.

Take a look at
https://code.google.com/p/chromium/codesearch#chromium/src/v8/src/objects-inl...
for inspiration.

   num_hash_chars = 0;
 
   for (int index = 0; index < total; ++index) {
     int span_start = cooked_strings->at(index)->position() + 1;
     int span_end = lengths->at(index) - 1;
     int length;
     int to_index = 0;
 
     if (index) {
       hash_string[num_hash_chars++] = '$';
@@ skipping 12 matching lines @@
       if (ch == '\r') {
         ch = '\n';
         if (from_index + 1 < length && raw_chars[from_index + 1] == '\n') {
           ++from_index;
         }
       }
       hash_string[num_hash_chars++] = ch;
       raw_chars[to_index++] = ch;
     }
 
-    const AstRawString* raw_str = ast_value_factory()->GetOneByteString(
-        OneByteVector(raw_chars.get(), to_index));
+    unibrow::Utf8Decoder<256> decoder(raw_chars.get(), to_index);

[Dmitry Lomov (no reviews), 2014/11/25 13:16:21]

Use cached decoder from isoalte()->unicode_cache()->utf8_decoder()

See:
https://code.google.com/p/chromium/codesearch#chromium/src/v8/src/factory.cc&...

[caitp (gmail), 2014/11/26 14:12:01]

Should this be cached outside of the loop? But, done and done!

+    int utf16_length = decoder.Utf16Length();
+    uc16* utf16_buffer = zone()->NewArray<uc16>(utf16_length);
+    to_index = decoder.WriteUtf16(utf16_buffer, utf16_length);
+    const AstRawString* raw_str = ast_value_factory()->GetTwoByteString(
+        Vector<const uint16_t>(reinterpret_cast<const uint16_t*>(utf16_buffer),
+            to_index));
     Literal* raw_lit = factory()->NewStringLiteral(raw_str, span_start - 1);
     raw_strings->Add(raw_lit, zone());
   }
 
   hash_string.Truncate(num_hash_chars);
   int utf16_length;
   *hash = StringHasher::ComputeUtf8Hash(Vector<const char>::cast(hash_string),
       num_hash_chars, &utf16_length);
   hash_string.Dispose();
 
   return raw_strings;
 }
 } }  // namespace v8::internal