Script streaming: more UTF-8 handing fixes (again).

test/cctest/test-api.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 23618 matching lines @@
         "  function foo() { return ";
     char chunk2[] = "  13; }\n";
     const char* chunks[] = {chunk1, chunk2, "foo();", NULL};
 
     RunStreamingTest(chunks);
   }
 }
 
 
 TEST(StreamingUtf8Script) {
-  // We'd want to write \uc481 instead of \xeb\x91\x80, but Windows compilers
+  // We'd want to write \uc481 instead of \xec\x92\x81, but Windows compilers
   // don't like it.
   const char* chunk1 =
       "function foo() {\n"
       "  // This function will contain an UTF-8 character which is not in\n"
       "  // ASCII.\n"
-      "  var foob\xeb\x91\x80r = 13;\n"
-      "  return foob\xeb\x91\x80r;\n"
+      "  var foob\xec\x92\x81r = 13;\n"
+      "  return foob\xec\x92\x81r;\n"
       "}\n";
   const char* chunks[] = {chunk1, "foo(); ", NULL};
   RunStreamingTest(chunks, v8::ScriptCompiler::StreamedSource::UTF8);
 }
 
 
 TEST(StreamingUtf8ScriptWithSplitCharactersSanityCheck) {
   // A sanity check to prove that the approach of splitting UTF-8
   // characters is correct. Here is an UTF-8 character which will take three
   // bytes.
-  const char* reference = "\xeb\x91\x80";
+  const char* reference = "\xec\x92\x81";
   CHECK(3u == strlen(reference));  // NOLINT - no CHECK_EQ for unsigned.
 
   char chunk1[] =
       "function foo() {\n"
       "  // This function will contain an UTF-8 character which is not in\n"
       "  // ASCII.\n"
       "  var foob";
   char chunk2[] =
       "XXXr = 13;\n"
-      "  return foob\xeb\x91\x80r;\n"
+      "  return foob\xec\x92\x81r;\n"
       "}\n";
   for (int i = 0; i < 3; ++i) {
     chunk2[i] = reference[i];
   }
   const char* chunks[] = {chunk1, chunk2, "foo();", NULL};
   RunStreamingTest(chunks, v8::ScriptCompiler::StreamedSource::UTF8);
 }
 
 
 TEST(StreamingUtf8ScriptWithSplitCharacters) {
   // Stream data where a multi-byte UTF-8 character is split between two data
   // chunks.
-  const char* reference = "\xeb\x91\x80";
+  const char* reference = "\xec\x92\x81";
   char chunk1[] =
       "function foo() {\n"
       "  // This function will contain an UTF-8 character which is not in\n"
       "  // ASCII.\n"
       "  var foobX";
   char chunk2[] =
       "XXr = 13;\n"
-      "  return foob\xeb\x91\x80r;\n"
+      "  return foob\xec\x92\x81r;\n"
       "}\n";
   chunk1[strlen(chunk1) - 1] = reference[0];
   chunk2[0] = reference[1];
   chunk2[1] = reference[2];
   const char* chunks[] = {chunk1, chunk2, "foo();", NULL};
   RunStreamingTest(chunks, v8::ScriptCompiler::StreamedSource::UTF8);
 }
 
 
 TEST(StreamingUtf8ScriptWithSplitCharactersValidEdgeCases) {
   // Tests edge cases which should still be decoded correctly.
 
   // Case 1: a chunk contains only bytes for a split character (and no other
   // data). This kind of a chunk would be exceptionally small, but we should
   // still decode it correctly.
-  const char* reference = "\xeb\x91\x80";
+  const char* reference = "\xec\x92\x81";
   // The small chunk is at the beginning of the split character
   {
     char chunk1[] =
         "function foo() {\n"
         "  // This function will contain an UTF-8 character which is not in\n"
         "  // ASCII.\n"
         "  var foob";
     char chunk2[] = "XX";
     char chunk3[] =
         "Xr = 13;\n"
-        "  return foob\xeb\x91\x80r;\n"
+        "  return foob\xec\x92\x81r;\n"
         "}\n";
     chunk2[0] = reference[0];
     chunk2[1] = reference[1];
     chunk3[0] = reference[2];
     const char* chunks[] = {chunk1, chunk2, chunk3, "foo();", NULL};
     RunStreamingTest(chunks, v8::ScriptCompiler::StreamedSource::UTF8);
   }
   // The small chunk is at the end of a character
   {
     char chunk1[] =
         "function foo() {\n"
         "  // This function will contain an UTF-8 character which is not in\n"
         "  // ASCII.\n"
         "  var foobX";
     char chunk2[] = "XX";
     char chunk3[] =
         "r = 13;\n"
-        "  return foob\xeb\x91\x80r;\n"
+        "  return foob\xec\x92\x81r;\n"
         "}\n";
     chunk1[strlen(chunk1) - 1] = reference[0];
     chunk2[0] = reference[1];
     chunk2[1] = reference[2];
     const char* chunks[] = {chunk1, chunk2, chunk3, "foo();", NULL};
     RunStreamingTest(chunks, v8::ScriptCompiler::StreamedSource::UTF8);
   }
   // Case 2: the script ends with a multi-byte character. Make sure that it's
   // decoded correctly and not just ignored.
   {
     char chunk1[] =
-        "var foob\xeb\x91\x80 = 13;\n"
-        "foob\xeb\x91\x80";
+        "var foob\xec\x92\x81 = 13;\n"
+        "foob\xec\x92\x81";
     const char* chunks[] = {chunk1, NULL};
     RunStreamingTest(chunks, v8::ScriptCompiler::StreamedSource::UTF8);
   }
 }
 
 
 TEST(StreamingUtf8ScriptWithSplitCharactersInvalidEdgeCases) {
   // Test cases where a UTF-8 character is split over several chunks. Those
   // cases are not supported (the embedder should give the data in big enough
   // chunks), but we shouldn't crash, just produce a parse error.
-  const char* reference = "\xeb\x91\x80";
+  const char* reference = "\xec\x92\x81";
   char chunk1[] =
       "function foo() {\n"
       "  // This function will contain an UTF-8 character which is not in\n"
       "  // ASCII.\n"
       "  var foobX";
   char chunk2[] = "X";
   char chunk3[] =
       "Xr = 13;\n"
-      "  return foob\xeb\x91\x80r;\n"
+      "  return foob\xec\x92\x81r;\n"
       "}\n";
   chunk1[strlen(chunk1) - 1] = reference[0];
   chunk2[0] = reference[1];
   chunk3[0] = reference[2];
   const char* chunks[] = {chunk1, chunk2, chunk3, "foo();", NULL};
 
   RunStreamingTest(chunks, v8::ScriptCompiler::StreamedSource::UTF8, false);
 }
 
 
@@ skipping 21 matching lines @@
   const v8::ScriptCompiler::CachedData* cached_data = source.GetCachedData();
   CHECK(cached_data != NULL);
   CHECK(cached_data->data != NULL);
   CHECK_GT(cached_data->length, 0);
 }
 
 
 TEST(StreamingScriptWithInvalidUtf8) {
   // Regression test for a crash: test that invalid UTF-8 bytes in the end of a
   // chunk don't produce a crash.
-  const char* reference = "\xeb\x91\x80\x80\x80";
+  const char* reference = "\xec\x92\x81\x80\x80";
   char chunk1[] =
       "function foo() {\n"
       "  // This function will contain an UTF-8 character which is not in\n"
       "  // ASCII.\n"
       "  var foobXXXXX";  // Too many bytes which look like incomplete chars!
   char chunk2[] =
       "r = 13;\n"
-      "  return foob\xeb\x91\x80\x80\x80r;\n"
+      "  return foob\xec\x92\x81\x80\x80r;\n"
       "}\n";
   for (int i = 0; i < 5; ++i) chunk1[strlen(chunk1) - 5 + i] = reference[i];
 
   const char* chunks[] = {chunk1, chunk2, "foo();", NULL};
   RunStreamingTest(chunks, v8::ScriptCompiler::StreamedSource::UTF8, false);
 }
 
 
 TEST(StreamingUtf8ScriptWithMultipleMultibyteCharactersSomeSplit) {
   // Regression test: Stream data where there are several multi-byte UTF-8
   // characters in a sequence and one of them is split between two data chunks.
-  const char* reference = "\xeb\x91\x80";
+  const char* reference = "\xec\x92\x81";
   char chunk1[] =
       "function foo() {\n"
       "  // This function will contain an UTF-8 character which is not in\n"
       "  // ASCII.\n"
-      "  var foob\xeb\x91\x80X";
+      "  var foob\xec\x92\x81X";
   char chunk2[] =
       "XXr = 13;\n"
-      "  return foob\xeb\x91\x80\xeb\x91\x80r;\n"
+      "  return foob\xec\x92\x81\xec\x92\x81r;\n"
       "}\n";
   chunk1[strlen(chunk1) - 1] = reference[0];
   chunk2[0] = reference[1];
+  chunk2[1] = reference[2];
+  const char* chunks[] = {chunk1, chunk2, "foo();", NULL};
+  RunStreamingTest(chunks, v8::ScriptCompiler::StreamedSource::UTF8);
+}
+
+
+TEST(StreamingUtf8ScriptWithMultipleMultibyteCharactersSomeSplit2) {
+  // Another regression test, similar to the previous one. The difference is
+  // that the split character is not the last one in the sequence.
+  const char* reference = "\xec\x92\x81";
+  char chunk1[] =
+      "function foo() {\n"
+      "  // This function will contain an UTF-8 character which is not in\n"
+      "  // ASCII.\n"
+      "  var foobX";
+  char chunk2[] =
+      "XX\xec\x92\x81r = 13;\n"
+      "  return foob\xec\x92\x81\xec\x92\x81r;\n"
+      "}\n";
+  chunk1[strlen(chunk1) - 1] = reference[0];
+  chunk2[0] = reference[1];
   chunk2[1] = reference[2];
   const char* chunks[] = {chunk1, chunk2, "foo();", NULL};
   RunStreamingTest(chunks, v8::ScriptCompiler::StreamedSource::UTF8);
 }