Merge "Pass more hints to encoding detector." to M57 branch

third_party/WebKit/Source/core/html/parser/TextResourceDecoder.cpp

 /*
     Copyright (C) 1999 Lars Knoll (knoll@mpi-hd.mpg.de)
     Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2012 Apple Inc. All
     rights reserved.
     Copyright (C) 2005, 2006, 2007 Alexey Proskuryakov (ap@nypop.com)
 
     This library is free software; you can redistribute it and/or
     modify it under the terms of the GNU Library General Public
     License as published by the Free Software Foundation; either
     version 2 of the License, or (at your option) any later version.
 
     This library is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     Library General Public License for more details.
 
     You should have received a copy of the GNU Library General Public License
     along with this library; see the file COPYING.LIB.  If not, write to
     the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
     Boston, MA 02110-1301, USA.
 */
 
 #include "core/html/parser/TextResourceDecoder.h"
 
 #include "core/HTMLNames.h"
 #include "core/dom/DOMImplementation.h"
 #include "core/html/parser/HTMLMetaCharsetParser.h"
+#include "platform/Language.h"
 #include "platform/text/TextEncodingDetector.h"
 #include "wtf/StringExtras.h"
 #include "wtf/text/TextCodec.h"
 #include "wtf/text/TextEncodingRegistry.h"
 
 using namespace WTF;
 
 namespace blink {
 
 using namespace HTMLNames;
@@ skipping 97 matching lines @@
   if (contentType == XMLContent)
     return UTF8Encoding();
   if (!specifiedDefaultEncoding.isValid())
     return Latin1Encoding();
   return specifiedDefaultEncoding;
 }
 
 TextResourceDecoder::TextResourceDecoder(
     const String& mimeType,
     const WTF::TextEncoding& specifiedDefaultEncoding,
-    EncodingDetectionOption encodingDetectionOption)
+    EncodingDetectionOption encodingDetectionOption,
+    const String& url)
     : m_contentType(determineContentType(mimeType)),
       m_encoding(defaultEncoding(m_contentType, specifiedDefaultEncoding)),
       m_source(DefaultEncoding),
       m_hintEncoding(0),
+      m_hintUrl(url.utf8()),
       m_checkedForBOM(false),
       m_checkedForCSSCharset(false),
       m_checkedForXMLCharset(false),
       m_checkedForMetaCharset(false),
       m_useLenientXMLDecoding(false),
       m_sawError(false),
       m_encodingDetectionOption(encodingDetectionOption) {
-  if (m_encodingDetectionOption == AlwaysUseUTF8ForText)
+  m_hintLanguage[0] = 0;
+  if (m_encodingDetectionOption == AlwaysUseUTF8ForText) {
     ASSERT(m_contentType == PlainTextContent && m_encoding == UTF8Encoding());
+  } else if (m_encodingDetectionOption == UseAllAutoDetection) {
+    // Checking empty URL helps unit testing. Providing defaultLanguage() is
+    // sometimes difficult in tests.
+    if (!url.isEmpty()) {
+      // This object is created in the main thread, but used in another thread.
+      // We should not share an AtomicString.
+      AtomicString locale = defaultLanguage();
+      if (locale.length() >= 2) {
+        // defaultLanguage() is always an ASCII string.
+        m_hintLanguage[0] = static_cast<char>(locale[0]);
+        m_hintLanguage[1] = static_cast<char>(locale[1]);
+        m_hintLanguage[2] = 0;
+      }
+    }
+  }
 }
 
 TextResourceDecoder::~TextResourceDecoder() {}
 
 void TextResourceDecoder::setEncoding(const WTF::TextEncoding& encoding,
                                       EncodingSource source) {
   // In case the encoding didn't exist, we keep the old one (helps some sites
   // specifying invalid encodings).
   if (!encoding.isValid())
     return;
@@ skipping 279 matching lines @@
 
     dataForDecode = m_buffer.data() + lengthOfBOM;
     lengthForDecode = m_buffer.size() - lengthOfBOM;
   }
 
   if (m_contentType == HTMLContent && !m_checkedForMetaCharset)
     checkForMetaCharset(dataForDecode, lengthForDecode);
 
   if (shouldAutoDetect()) {
     WTF::TextEncoding detectedEncoding;
-    if (detectTextEncoding(data, len, m_hintEncoding, &detectedEncoding))
+    if (detectTextEncoding(data, len, m_hintEncoding, m_hintUrl.data(),
+                           m_hintLanguage, &detectedEncoding))
       setEncoding(detectedEncoding, EncodingFromContentSniffing);
   }
 
   ASSERT(m_encoding.isValid());
 
   if (!m_codec)
     m_codec = newTextCodec(m_encoding);
 
   String result = m_codec->decode(
       dataForDecode, lengthForDecode, DoNotFlush,
       m_contentType == XMLContent && !m_useLenientXMLDecoding, m_sawError);
 
   m_buffer.clear();
   return result;
 }
 
 String TextResourceDecoder::flush() {
   // If we can not identify the encoding even after a document is completely
   // loaded, we need to detect the encoding if other conditions for
   // autodetection is satisfied.
   if (m_buffer.size() && shouldAutoDetect() &&
       ((!m_checkedForXMLCharset &&
         (m_contentType == HTMLContent || m_contentType == XMLContent)) ||
        (!m_checkedForCSSCharset && (m_contentType == CSSContent)))) {
     WTF::TextEncoding detectedEncoding;
     if (detectTextEncoding(m_buffer.data(), m_buffer.size(), m_hintEncoding,
-                           &detectedEncoding))
+                           m_hintUrl.data(), m_hintLanguage, &detectedEncoding))
       setEncoding(detectedEncoding, EncodingFromContentSniffing);
   }
 
   if (!m_codec)
     m_codec = newTextCodec(m_encoding);
 
   String result = m_codec->decode(
       m_buffer.data(), m_buffer.size(), FetchEOF,
       m_contentType == XMLContent && !m_useLenientXMLDecoding, m_sawError);
   m_buffer.clear();
   m_codec.reset();
   m_checkedForBOM = false;  // Skip BOM again when re-decoding.
   return result;
 }
 
 }  // namespace blink