UTF-8 detector for pages missing encoding info

third_party/WebKit/Source/core/html/parser/TextResourceDecoder.cpp

 /*
     Copyright (C) 1999 Lars Knoll (knoll@mpi-hd.mpg.de)
     Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2012 Apple Inc. All rights reserved.
     Copyright (C) 2005, 2006, 2007 Alexey Proskuryakov (ap@nypop.com)
 
     This library is free software; you can redistribute it and/or
     modify it under the terms of the GNU Library General Public
     License as published by the Free Software Foundation; either
     version 2 of the License, or (at your option) any later version.
 
@@ skipping 10 matching lines @@
 
 #include "core/html/parser/TextResourceDecoder.h"
 
 #include "core/HTMLNames.h"
 #include "core/dom/DOMImplementation.h"
 #include "core/html/parser/HTMLMetaCharsetParser.h"
 #include "platform/text/TextEncodingDetector.h"
 #include "wtf/StringExtras.h"
 #include "wtf/text/TextCodec.h"
 #include "wtf/text/TextEncodingRegistry.h"
+#include "wtf/text/UTF8.h"
 
 using namespace WTF;
 
 namespace blink {
 
 using namespace HTMLNames;
 
 const int minimumLengthOfXMLDeclaration = 8;
 
 static inline bool bytesEqual(const char* p, char b0, char b1, char b2, char b3, char b4)
@@ skipping 301 matching lines @@
 
     if (!m_charsetParser->checkForMetaCharset(data, length))
         return;
 
     setEncoding(m_charsetParser->encoding(), EncodingFromMetaTag);
     m_charsetParser.clear();
     m_checkedForMetaCharset = true;
     return;
 }
 
-// We use the encoding detector in two cases:
-//   1. Encoding detector is turned ON and no other encoding source is
-//      available (that is, it's DefaultEncoding).
-//   2. Encoding detector is turned ON and the encoding is set to
-//      the encoding of the parent frame, which is also auto-detected.
-//   Note that condition #2 is NOT satisfied unless parent-child frame
-//   relationship is compliant to the same-origin policy. If they're from
-//   different domains, |m_source| would not be set to EncodingFromParentFrame
-//   in the first place.
-bool TextResourceDecoder::shouldAutoDetect() const
-{
-    // Just checking m_hintEncoding suffices here because it's only set
-    // in setHintEncoding when the source is AutoDetectedEncoding.
-    return m_encodingDetectionOption == UseAllAutoDetection
-        && (m_source == DefaultEncoding || (m_source == EncodingFromParentFrame && m_hintEncoding));
-}
-
 String TextResourceDecoder::decode(const char* data, size_t len)
 {
     size_t lengthOfBOM = 0;
     if (!m_checkedForBOM)
         lengthOfBOM = checkForBOM(data, len);
 
     bool movedDataToBuffer = false;
 
     if (m_contentType == CSSContent && !m_checkedForCSSCharset) {
         if (!checkForCSSCharset(data, len, movedDataToBuffer))
@@ skipping 16 matching lines @@
             memcpy(m_buffer.data() + oldSize, data, len);
         }
 
         dataForDecode = m_buffer.data() + lengthOfBOM;
         lengthForDecode = m_buffer.size() - lengthOfBOM;
     }
 
     if (m_contentType == HTMLContent && !m_checkedForMetaCharset)
         checkForMetaCharset(dataForDecode, lengthForDecode);
 
-    if (shouldAutoDetect()) {
-        WTF::TextEncoding detectedEncoding;
-        if (detectTextEncoding(data, len, m_hintEncoding, &detectedEncoding))
-            setEncoding(detectedEncoding, EncodingFromContentSniffing);
-    }
+    detectTextEncoding(data, len);
 
     ASSERT(m_encoding.isValid());
 
     if (!m_codec)
         m_codec = newTextCodec(m_encoding);
 
     String result = m_codec->decode(dataForDecode, lengthForDecode, DoNotFlush, m_contentType == XMLContent && !m_useLenientXMLDecoding, m_sawError);
 
     m_buffer.clear();
     return result;
 }
 
+// We use the encoding detector in following cases:
+//   1. Encoding detector is turned ON and no other encoding source is
+//      available (that is, it's DefaultEncoding).
+//   2. Encoding detector is turned ON and the encoding is set to
+//      the encoding of the parent frame, which is also auto-detected.
+//   Note that condition #2 is NOT satisfied unless parent-child frame
+//   relationship is compliant to the same-origin policy. If they're from
+//   different domains, |m_source| would not be set to EncodingFromParentFrame
+//   in the first place.
+void TextResourceDecoder::detectTextEncoding(const char* data, size_t len)
+{
+    if (!shouldDetectEncoding())
+        return;
+
+    if (WTF::Unicode::isUTF8andNotASCII(data, len)) {
+        setEncoding(UTF8Encoding(), EncodingFromContentSniffing);
+        return;
+    }
+    if (m_encodingDetectionOption == UseAllAutoDetection) {
+        WTF::TextEncoding detectedEncoding;
+        if (detectTextEncodingUniversal(data, len, m_hintEncoding, &detectedEncoding))
+            setEncoding(detectedEncoding, EncodingFromContentSniffing);
+    }
+}
+
+bool TextResourceDecoder::shouldDetectEncoding() const
+{
+    // Just checking m_hintEncoding suffices here because it's only set
+    // in setHintEncoding when the source is AutoDetectedEncoding.
+    return m_source == DefaultEncoding || (m_source == EncodingFromParentFrame && m_hintEncoding);
+}
+
 String TextResourceDecoder::flush()
 {
     // If we can not identify the encoding even after a document is completely
     // loaded, we need to detect the encoding if other conditions for
     // autodetection is satisfied.
-    if (m_buffer.size() && shouldAutoDetect()
+    if (m_buffer.size()
         && ((!m_checkedForXMLCharset && (m_contentType == HTMLContent || m_contentType == XMLContent)) || (!m_checkedForCSSCharset && (m_contentType == CSSContent)))) {
-        WTF::TextEncoding detectedEncoding;
-        if (detectTextEncoding(m_buffer.data(), m_buffer.size(), m_hintEncoding, &detectedEncoding))
-            setEncoding(detectedEncoding, EncodingFromContentSniffing);
+        detectTextEncoding(m_buffer.data(), m_buffer.size());
     }
 
     if (!m_codec)
         m_codec = newTextCodec(m_encoding);
 
     String result = m_codec->decode(m_buffer.data(), m_buffer.size(), FetchEOF, m_contentType == XMLContent && !m_useLenientXMLDecoding, m_sawError);
     m_buffer.clear();
     m_codec.clear();
     m_checkedForBOM = false; // Skip BOM again when re-decoding.
     return result;
 }
 
 } // namespace blink