UTF-8 detector for pages missing encoding info

third_party/WebKit/Source/core/html/parser/TextResourceDecoder.cpp

 /*
     Copyright (C) 1999 Lars Knoll (knoll@mpi-hd.mpg.de)
     Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2012 Apple Inc. All rights reserved.
     Copyright (C) 2005, 2006, 2007 Alexey Proskuryakov (ap@nypop.com)
 
     This library is free software; you can redistribute it and/or
     modify it under the terms of the GNU Library General Public
     License as published by the Free Software Foundation; either
     version 2 of the License, or (at your option) any later version.
 
@@ skipping 388 matching lines @@
         lengthForDecode = m_buffer.size() - lengthOfBOM;
     }
 
     if (m_contentType == HTMLContent && !m_checkedForMetaCharset)
         checkForMetaCharset(dataForDecode, lengthForDecode);
 
     if (shouldAutoDetect()) {
         WTF::TextEncoding detectedEncoding;
         if (detectTextEncoding(data, len, m_hintEncoding, &detectedEncoding))
             setEncoding(detectedEncoding, EncodingFromContentSniffing);
+    } else if ((m_source == DefaultEncoding || (m_source == EncodingFromParentFrame && m_hintEncoding)) && isUTF8Encoded(data, len)) {

[aelias_OOO_until_Jul13, 2016/02/24 04:37:53]

This if statement is duplicative with the other and also with
shouldAutodetect(), could you create a small method for it?

[Jinsuk Kim, 2016/02/24 06:54:54]

Done.

+        setEncoding(UTF8Encoding(), EncodingFromContentSniffing);
     }
 
     ASSERT(m_encoding.isValid());
 
     if (!m_codec)
         m_codec = newTextCodec(m_encoding);
 
     String result = m_codec->decode(dataForDecode, lengthForDecode, DoNotFlush, m_contentType == XMLContent && !m_useLenientXMLDecoding, m_sawError);
 
     m_buffer.clear();
     return result;
 }
 
 String TextResourceDecoder::flush()
 {
     // If we can not identify the encoding even after a document is completely
     // loaded, we need to detect the encoding if other conditions for
     // autodetection is satisfied.
-    if (m_buffer.size() && shouldAutoDetect()
+    if (m_buffer.size()
         && ((!m_checkedForXMLCharset && (m_contentType == HTMLContent || m_contentType == XMLContent)) || (!m_checkedForCSSCharset && (m_contentType == CSSContent)))) {
-        WTF::TextEncoding detectedEncoding;
-        if (detectTextEncoding(m_buffer.data(), m_buffer.size(), m_hintEncoding, &detectedEncoding))
-            setEncoding(detectedEncoding, EncodingFromContentSniffing);
+        if (shouldAutoDetect()) {
+            WTF::TextEncoding detectedEncoding;
+            if (detectTextEncoding(m_buffer.data(), m_buffer.size(), m_hintEncoding, &detectedEncoding))
+                setEncoding(detectedEncoding, EncodingFromContentSniffing);
+        } else if ((m_source == DefaultEncoding || (m_source == EncodingFromParentFrame && m_hintEncoding)) && isUTF8Encoded(m_buffer.data(), m_buffer.size())) {
+            setEncoding(UTF8Encoding(), EncodingFromContentSniffing);
+        }
     }
 
     if (!m_codec)
         m_codec = newTextCodec(m_encoding);
 
     String result = m_codec->decode(m_buffer.data(), m_buffer.size(), FetchEOF, m_contentType == XMLContent && !m_useLenientXMLDecoding, m_sawError);
     m_buffer.clear();
     m_codec.clear();
     m_checkedForBOM = false; // Skip BOM again when re-decoding.
     return result;
 }
 
 } // namespace blink