Avoid using language hint in encoding detection

third_party/WebKit/Source/core/html/parser/TextResourceDecoder.cpp

 /*
     Copyright (C) 1999 Lars Knoll (knoll@mpi-hd.mpg.de)
     Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2012 Apple Inc. All
     rights reserved.
     Copyright (C) 2005, 2006, 2007 Alexey Proskuryakov (ap@nypop.com)
 
     This library is free software; you can redistribute it and/or
     modify it under the terms of the GNU Library General Public
     License as published by the Free Software Foundation; either
     version 2 of the License, or (at your option) any later version.
@@ skipping 141 matching lines @@
       m_checkedForCSSCharset(false),
       m_checkedForXMLCharset(false),
       m_checkedForMetaCharset(false),
       m_useLenientXMLDecoding(false),
       m_sawError(false),
       m_encodingDetectionOption(encodingDetectionOption) {
   m_hintLanguage[0] = 0;
   if (m_encodingDetectionOption == AlwaysUseUTF8ForText) {
     DCHECK(m_contentType == PlainTextContent && m_encoding == UTF8Encoding());
   } else if (m_encodingDetectionOption == UseAllAutoDetection) {
-    // Checking empty URL helps unit testing. Providing defaultLanguage() is
+    // Obtain language hint from system locale:
+    // 1) In general, do not use language hint. This helps get more
+    // deterministic encoding detection results across devices. Note that local
+    // file resources can still benefit from the hint.
+    // 2) Checking empty URL helps unit testing. Providing defaultLanguage() is
     // sometimes difficult in tests.
-    if (!hintUrl.isEmpty()) {
+    if (hintUrl.protocol() == "file" || !hintUrl.isEmpty()) {

[tkent, 2017/04/06 07:29:13]

The code is incorrect.  http:// URLs are not isEmpty(). So any URL passes this
condition.

BTW, I'd like to add file:// check to detectTextEncoding() rather than
TextResourceDecoder.  It would make unit testing easy.

[Jinsuk Kim, 2017/04/06 08:10:25]

My bad.

Moved to TextEncodingDetector and updated tests.

       // This object is created in the main thread, but used in another thread.
       // We should not share an AtomicString.
       AtomicString locale = defaultLanguage();
       if (locale.length() >= 2) {
         // defaultLanguage() is always an ASCII string.
         m_hintLanguage[0] = static_cast<char>(locale[0]);
         m_hintLanguage[1] = static_cast<char>(locale[1]);
         m_hintLanguage[2] = 0;
       }
     }
@@ skipping 335 matching lines @@
   String result = m_codec->decode(
       m_buffer.data(), m_buffer.size(), WTF::FetchEOF,
       m_contentType == XMLContent && !m_useLenientXMLDecoding, m_sawError);
   m_buffer.clear();
   m_codec.reset();
   m_checkedForBOM = false;  // Skip BOM again when re-decoding.
   return result;
 }
 
 }  // namespace blink