Script streaming: remove byte order marks, detect encoding based on them.

Source/bindings/core/v8/ScriptStreamer.cpp

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "config.h"
 #include "bindings/core/v8/ScriptStreamer.h"
 
 #include "bindings/core/v8/ScriptStreamerThread.h"
 #include "bindings/core/v8/V8ScriptRunner.h"
 #include "core/dom/Document.h"
 #include "core/dom/Element.h"
 #include "core/dom/PendingScript.h"
 #include "core/fetch/ScriptResource.h"
 #include "core/frame/Settings.h"
+#include "core/html/parser/TextResourceDecoder.h"
 #include "platform/SharedBuffer.h"
 #include "platform/TraceEvent.h"
 #include "public/platform/Platform.h"
 #include "wtf/MainThread.h"
 #include "wtf/text/TextEncodingRegistry.h"
 
 namespace blink {
 
 // For passing data between the main thread (producer) and the streamer thread
 // (consumer). The main thread prepares the data (copies it from Resource) and
@@ skipping 91 matching lines @@
         }
         return length;
     }
 
     void didFinishLoading()
     {
         ASSERT(isMainThread());
         m_dataQueue.finish();
     }
 
-    void didReceiveData()
+    void didReceiveData(size_t lengthOfBOM)
     {
         ASSERT(isMainThread());
-        prepareDataOnMainThread();
+        prepareDataOnMainThread(lengthOfBOM);
     }
 
     void cancel()
     {
         ASSERT(isMainThread());
         // The script is no longer needed by the upper layers. Stop streaming
         // it. The next time GetMoreData is called (or woken up), it will return
         // 0, which will be interpreted as EOS by V8 and the parsing will
         // fail. ScriptStreamer::streamingComplete will be called, and at that
         // point we will release the references to SourceStream.
         {
             MutexLocker locker(m_mutex);
             m_cancelled = true;
         }
         m_dataQueue.finish();
     }
 
 private:
-    void prepareDataOnMainThread()
+    void prepareDataOnMainThread(size_t lengthOfBOM)
     {
         ASSERT(isMainThread());
         // The Resource must still be alive; otherwise we should've cancelled
         // the streaming (if we have cancelled, the background thread is not
         // waiting).
         ASSERT(m_streamer->resource());
 
+        // BOM can only occur at the beginning of the data.
+        ASSERT(lengthOfBOM == 0 || m_dataPosition == 0);
+
         if (m_streamer->resource()->cachedMetadata(V8ScriptRunner::tagForCodeCache())) {
             // The resource has a code cache, so it's unnecessary to stream and
             // parse the code. Cancel the streaming and resume the non-streaming
             // code path.
             m_streamer->suppressStreaming();
             {
                 MutexLocker locker(m_mutex);
                 m_cancelled = true;
             }
             m_dataQueue.finish();
             return;
         }
 
         if (!m_resourceBuffer) {
             // We don't have a buffer yet. Try to get it from the resource.
             SharedBuffer* buffer = m_streamer->resource()->resourceBuffer();
-            if (!buffer)
-                return;
             m_resourceBuffer = RefPtr<SharedBuffer>(buffer);
         }
 
         // Get as much data from the ResourceBuffer as we can.
         const char* data = 0;
         Vector<const char*> chunks;
         Vector<unsigned> chunkLengths;
         size_t dataLength = 0;
         while (unsigned length = m_resourceBuffer->getSomeData(data, m_dataPosition)) {
             // FIXME: Here we can limit based on the total length, if it turns
             // out that we don't want to give all the data we have (memory
             // vs. speed).
             chunks.append(data);
             chunkLengths.append(length);
             dataLength += length;
             m_dataPosition += length;
         }
         // Copy the data chunks into a new buffer, since we're going to give the
         // data to a background thread.
-        if (dataLength > 0) {
+        if (dataLength > lengthOfBOM) {
+            dataLength -= lengthOfBOM;
             uint8_t* copiedData = new uint8_t[dataLength];
             unsigned offset = 0;
             for (size_t i = 0; i < chunks.size(); ++i) {
-                memcpy(copiedData + offset, chunks[i], chunkLengths[i]);
-                offset += chunkLengths[i];
+                memcpy(copiedData + offset, chunks[i] + lengthOfBOM, chunkLengths[i] - lengthOfBOM);
+                offset += chunkLengths[i] - lengthOfBOM;
+                // BOM is only in the first chunk
+                lengthOfBOM = 0;
             }
             m_dataQueue.produce(copiedData, dataLength);
         }
     }
 
     ScriptStreamer* m_streamer;
 
     // For coordinating between the main thread and background thread tasks.
     // Guarded by m_mutex.
     bool m_cancelled;
@@ skipping 70 matching lines @@
 
 void ScriptStreamer::notifyAppendData(ScriptResource* resource)
 {
     ASSERT(isMainThread());
     ASSERT(m_resource == resource);
     {
         MutexLocker locker(m_mutex);
         if (m_streamingSuppressed)
             return;
     }
+    size_t lengthOfBOM = 0;
     if (!m_haveEnoughDataForStreaming) {
         // Even if the first data chunk is small, the script can still be big
         // enough - wait until the next data chunk comes before deciding whether
         // to start the streaming.
-        if (resource->resourceBuffer()->size() < kSmallScriptThreshold) {
+        ASSERT(resource->resourceBuffer());
+        if (resource->resourceBuffer()->size() < kSmallScriptThreshold)
             return;
-        }
         m_haveEnoughDataForStreaming = true;
         const char* histogramName = startedStreamingHistogramName(m_scriptType);
 
         // Encoding should be detected only when we have some data. It's
         // possible that resource->encoding() returns a different encoding
-        // before the loading has started and after we got some data.
-        WTF::TextEncoding textEncoding(resource->encoding());
-        const char* encodingName = textEncoding.name();
+        // before the loading has started and after we got some data.  In
+        // addition, check for byte order marks. Note that checking the byte
+        // order mark might change the encoding. We cannot decode the full text
+        // here, because it might contain incomplete UTF-8 characters. Also note
+        // that have at least kSmallScriptThreshold worth of data, which is more
+        // than enough for detecting a BOM.
+        const char* data = 0;
+        unsigned length = resource->resourceBuffer()->getSomeData(data, 0);
+
+        OwnPtr<TextResourceDecoder> decoder(TextResourceDecoder::create("application/javascript", resource->encoding()));
+        lengthOfBOM = decoder->checkForBOM(data, length);
+
+        // Maybe the encoding changed because we saw the BOM; get the encoding
+        // from the decoder.
+        const char* encodingName = decoder->encoding().name();
 
         // Here's a list of encodings we can use for streaming. These are
         // the canonical names.
         v8::ScriptCompiler::StreamedSource::Encoding encoding;
         if (strcmp(encodingName, "windows-1252") == 0
             || strcmp(encodingName, "ISO-8859-1") == 0
             || strcmp(encodingName, "US-ASCII") == 0) {
             encoding = v8::ScriptCompiler::StreamedSource::ONE_BYTE;
         } else if (strcmp(encodingName, "UTF-8") == 0) {
             encoding = v8::ScriptCompiler::StreamedSource::UTF8;
         } else {
             // We don't stream other encodings; especially we don't stream two
-            // byte scripts to avoid the handling of byte order marks. Most
-            // scripts are Latin1 or UTF-8 anyway, so this should be enough for
-            // most real world purposes.
+            // byte scripts to avoid the handling of endianness. Most scripts
+            // are Latin1 or UTF-8 anyway, so this should be enough for most
+            // real world purposes.
             suppressStreaming();
             blink::Platform::current()->histogramEnumeration(histogramName, 0, 2);
             return;
         }
         if (ScriptStreamerThread::shared()->isRunningTask()) {
             // At the moment we only have one thread for running the tasks. A
             // new task shouldn't be queued before the running task completes,
             // because the running task can block and wait for data from the
             // network.
             suppressStreaming();
@@ skipping 27 matching lines @@
         // ScriptStreamer needs to stay alive as long as the background task is
         // running. This is taken care of with a manual ref() & deref() pair;
         // the corresponding deref() is in streamingComplete or in
         // notifyFinished.
         ref();
         ScriptStreamingTask* task = new ScriptStreamingTask(scriptStreamingTask.release(), this);
         ScriptStreamerThread::shared()->postTask(task);
         blink::Platform::current()->histogramEnumeration(histogramName, 1, 2);
     }
     if (m_stream)
-        m_stream->didReceiveData();
+        m_stream->didReceiveData(lengthOfBOM);
 }
 
 void ScriptStreamer::notifyFinished(Resource* resource)
 {
     ASSERT(isMainThread());
     ASSERT(m_resource == resource);
     // A special case: empty and small scripts. We didn't receive enough data to
     // start the streaming before this notification. In that case, there won't
     // be a "parsing complete" notification either, and we should not wait for
     // it.
@@ skipping 155 matching lines @@
 
     // The Resource might go out of scope if the script is no longer
     // needed. This makes PendingScript notify the ScriptStreamer when it is
     // destroyed.
     script.setStreamer(adoptRef(new ScriptStreamer(resource, scriptType, settings->v8ScriptStreamingMode(), scriptState, compileOption)));
 
     return true;
 }
 
 } // namespace blink