Streaming API: detect UTF-8 BOM.

test/cctest/test-api.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 23830 matching lines @@
     memcpy(copy, chunks_[index_], len);
     *src = copy;
     ++index_;
     return len;
   }
 
   // Helper for constructing a string from chunks (the compilation needs it
   // too).
   static char* FullSourceString(const char** chunks) {
     size_t total_len = 0;
+    bool has_bom = false;
     for (size_t i = 0; chunks[i] != NULL; ++i) {
       total_len += strlen(chunks[i]);
+
+      // Remove BOM when constructing the full source string; this simluates
+      // what the embedder does.
+      if (i == 0 && strlen(chunks[i]) >= 3 &&
+          chunks[i][0] == static_cast<char>(0xef) &&
+          chunks[i][1] == static_cast<char>(0xbb) &&
+          chunks[i][2] == static_cast<char>(0xbf)) {
+        total_len -= 3;
+        has_bom = true;
+      }
     }
     char* full_string = new char[total_len + 1];
     size_t offset = 0;
     for (size_t i = 0; chunks[i] != NULL; ++i) {
       size_t len = strlen(chunks[i]);
-      memcpy(full_string + offset, chunks[i], len);
-      offset += len;
+      if (has_bom) {
+        memcpy(full_string + offset, chunks[i] + 3, len - 3);
+        offset += len - 3;
+      } else {
+        memcpy(full_string + offset, chunks[i], len);
+        offset += len;
+      }
     }
     full_string[total_len] = 0;
     return full_string;
   }
 
  private:
   const char** chunks_;
   unsigned index_;
 };
 
@@ skipping 23 matching lines @@
 
   // The possible errors are only produced while compiling.
   CHECK_EQ(false, try_catch.HasCaught());
 
   v8::Handle<Script> script = v8::ScriptCompiler::Compile(
       isolate, &source, v8_str(full_source), origin);
   if (expected_success) {
     CHECK(!script.IsEmpty());
     v8::Handle<Value> result(script->Run());
     // All scripts are supposed to return the fixed value 13 when ran.
+    CHECK_EQ(false, try_catch.HasCaught());
     CHECK_EQ(13, result->Int32Value());
   } else {
     CHECK(script.IsEmpty());
     CHECK(try_catch.HasCaught());
   }
   delete[] full_source;
 }
 
 
 TEST(StreamingSimpleScript) {
@@ skipping 271 matching lines @@
   char chunk2[] =
       "XX\xec\x92\x81r = 13;\n"
       "  return foob\xec\x92\x81\xec\x92\x81r;\n"
       "}\n";
   chunk1[strlen(chunk1) - 1] = reference[0];
   chunk2[0] = reference[1];
   chunk2[1] = reference[2];
   const char* chunks[] = {chunk1, chunk2, "foo();", NULL};
   RunStreamingTest(chunks, v8::ScriptCompiler::StreamedSource::UTF8);
 }
+
+
+TEST(StreamingUtf8ScriptWithByteOrderMark) {
+  // Some Windows editors add the "UTF-8 BOM". If we don't handle it properly,
+  // some scripts might be parsed successfully but but the positions of lazy
+  // functions will be off.
+  i::FLAG_min_preparse_length = 0;
+  i::FLAG_lazy = true;
+  // Note that in this case the byte order mark doesn't actually cause a parse
+  // error, since it's a UTF-8 character followed by a comment.
+  char chunk1[] =
+      "XXX// That's the BOM.\n"
+      "function this_is_lazy() {\n"
+      "  return 13;\n"
+      "}\n";
+  chunk1[0] = static_cast<char>(0xef);
+  chunk1[1] = static_cast<char>(0xbb);
+  chunk1[2] = static_cast<char>(0xbf);
+  const char* chunks[] = {chunk1, "this_is_lazy();", NULL};
+  RunStreamingTest(chunks, v8::ScriptCompiler::StreamedSource::UTF8);
+}