Script streaming: remove byte order marks, detect encoding based on them.

Source/bindings/core/v8/ScriptStreamerTest.cpp

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 
 #include "config.h"
 #include "bindings/core/v8/ScriptStreamer.h"
 
 #include "bindings/core/v8/ScriptSourceCode.h"
 #include "bindings/core/v8/ScriptStreamerThread.h"
@@ skipping 68 matching lines @@
 
     ScriptState* scriptState() const { return m_scope.scriptState(); }
     v8::Isolate* isolate() const { return m_scope.isolate(); }
 
     PendingScript& pendingScript() const { return m_pendingScript->get(); }
 
 protected:
     void appendData(const char* data)
     {
         m_resource->appendData(data, strlen(data));
-        // Yield control to the background thread, so that V8 gets a change to
+        // Yield control to the background thread, so that V8 gets a chance to
         // process the data before the main thread adds more. Note that we
         // cannot fully control in what kind of chunks the data is passed to V8
-        // (if the V8 is not requesting more data between two appendData calls,
-        // V8 will get both chunks together).
+        // (if V8 is not requesting more data between two appendData calls, it
+        // will get both chunks together).
         Platform::current()->yieldCurrentThread();
     }
 
     void appendPadding()
     {
         for (int i = 0; i < 10; ++i) {
             appendData(" /* this is padding to make the script long enough, so "
                 "that V8's buffer gets filled and it starts processing "
                 "the data */ ");
         }
@@ skipping 253 matching lines @@
     bool errorOccurred = false;
     ScriptSourceCode sourceCode = pendingScript().getSource(KURL(), errorOccurred);
     EXPECT_FALSE(errorOccurred);
     EXPECT_TRUE(sourceCode.streamer());
     v8::TryCatch tryCatch;
     v8::Handle<v8::Script> script = V8ScriptRunner::compileScript(sourceCode, isolate());
     EXPECT_FALSE(script.IsEmpty());
     EXPECT_FALSE(tryCatch.HasCaught());
 }
 
+
+TEST_P(ScriptStreamingTest, EncodingFromBOM)
+{
+    // Byte order marks should be removed before giving the data to V8. They
+    // will also affect encoding detection.
+    m_resource->setEncoding("windows-1252"); // This encoding is wrong on purpose.
+
+    ScriptStreamer::startStreaming(pendingScript(), m_settings.get(), m_scope.scriptState(), PendingScript::ParsingBlocking);
+    TestScriptResourceClient client;
+    pendingScript().watchForLoad(&client);
+
+    // \xef\xbb\xbf is the UTF-8 byte order mark. \xec\x92\x81 are the raw bytes
+    // for \uc481.
+    appendData("\xef\xbb\xbf function foo() { var foob\xec\x92\x81r = 13; return foob\xec\x92\x81r; } foo();");
+
+    finish();
+    processTasksUntilStreamingComplete();
+    EXPECT_TRUE(client.finished());
+    bool errorOccurred = false;
+    ScriptSourceCode sourceCode = pendingScript().getSource(KURL(), errorOccurred);
+    EXPECT_FALSE(errorOccurred);
+    EXPECT_TRUE(sourceCode.streamer());
+    v8::TryCatch tryCatch;
+    v8::Handle<v8::Script> script = V8ScriptRunner::compileScript(sourceCode, isolate());
+    EXPECT_FALSE(script.IsEmpty());
+    EXPECT_FALSE(tryCatch.HasCaught());
+}
+
 INSTANTIATE_TEST_CASE_P(ScriptStreamingInstantiation, ScriptStreamingTest, ::testing::Values(false, true));
 
 } // namespace
 
 } // namespace blink