[Fetch] Always use utf-8 for decoding in text()

third_party/WebKit/Source/core/html/parser/TextResourceDecoder.cpp

 /*
     Copyright (C) 1999 Lars Knoll (knoll@mpi-hd.mpg.de)
     Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2012 Apple Inc. All rights reserved.
     Copyright (C) 2005, 2006, 2007 Alexey Proskuryakov (ap@nypop.com)
 
     This library is free software; you can redistribute it and/or
     modify it under the terms of the GNU Library General Public
     License as published by the Free Software Foundation; either
     version 2 of the License, or (at your option) any later version.
 
@@ skipping 94 matching lines @@
 {
     // Despite 8.5 "Text/xml with Omitted Charset" of RFC 3023, we assume UTF-8 instead of US-ASCII
     // for text/xml. This matches Firefox.
     if (contentType == XMLContent)
         return UTF8Encoding();
     if (!specifiedDefaultEncoding.isValid())
         return Latin1Encoding();
     return specifiedDefaultEncoding;
 }
 
-TextResourceDecoder::TextResourceDecoder(const String& mimeType, const WTF::TextEncoding& specifiedDefaultEncoding, bool usesEncodingDetector)
+TextResourceDecoder::TextResourceDecoder(const String& mimeType, const WTF::TextEncoding& specifiedDefaultEncoding, EncodingDetectionOption encodingDetectionOption)
     : m_contentType(determineContentType(mimeType))
     , m_encoding(defaultEncoding(m_contentType, specifiedDefaultEncoding))
     , m_source(DefaultEncoding)
     , m_hintEncoding(0)
     , m_checkedForBOM(false)
     , m_checkedForCSSCharset(false)
     , m_checkedForXMLCharset(false)
     , m_checkedForMetaCharset(false)
     , m_useLenientXMLDecoding(false)
     , m_sawError(false)
-    , m_usesEncodingDetector(usesEncodingDetector)
+    , m_encodingDetectionOption(encodingDetectionOption)
 {
+    ASSERT(!(m_encodingDetectionOption == AlwaysUseUTF8ForText && (m_contentType != PlainTextContent || m_encoding != UTF8Encoding())));

[kouhei (in TOK), 2015/12/14 02:00:22]

Optional nit: Would you split this ASSERT stmt?
if (m_encodingDetectionOption == AlwaysUseUTF8ForText)
    ASSERT(m_contentType != PlainTextContent || m_encoding != UTF8Encoding());

[hiroshige, 2015/12/14 05:44:02]

Done.

 }
 
 TextResourceDecoder::~TextResourceDecoder()
 {
 }
 
 void TextResourceDecoder::setEncoding(const WTF::TextEncoding& encoding, EncodingSource source)
 {
     // In case the encoding didn't exist, we keep the old one (helps some sites specifying invalid encodings).
     if (!encoding.isValid())
@@ skipping 65 matching lines @@
     size_t buf1Len = bufferLength;
     size_t buf2Len = len;
     const unsigned char* buf1 = reinterpret_cast<const unsigned char*>(m_buffer.data());
     const unsigned char* buf2 = reinterpret_cast<const unsigned char*>(data);
     unsigned char c1 = buf1Len ? (--buf1Len, *buf1++) : buf2Len ? (--buf2Len, *buf2++) : 0;
     unsigned char c2 = buf1Len ? (--buf1Len, *buf1++) : buf2Len ? (--buf2Len, *buf2++) : 0;
     unsigned char c3 = buf1Len ? (--buf1Len, *buf1++) : buf2Len ? (--buf2Len, *buf2++) : 0;
     unsigned char c4 = buf2Len ? (--buf2Len, *buf2++) : 0;
 
     // Check for the BOM.
-    if (c1 == 0xFF && c2 == 0xFE) {
+    if (m_encodingDetectionOption != AlwaysUseUTF8ForText && c1 == 0xFF && c2 == 0xFE) {

[kouhei (in TOK), 2015/12/14 02:00:22]

Can we reorder the if stmts here?

if (c1 == 0xEF && c2 == 0xBB && c3 == 0xBF) {

} else if (m_encodingDetectionOption != AlwaysUseUTF8ForText) {
  if (c1 == 0xFF && c2 == 0xFE) {

  } else if (c1 == 0xFE && c2 == 0xFF) {

  } else if (...) {

  }
}

[hiroshige, 2015/12/14 05:44:02]

Done.

         if (c3 || c4) {
             setEncoding(UTF16LittleEndianEncoding(), AutoDetectedEncoding);
             lengthOfBOM = 2;
         } else {
             setEncoding(UTF32LittleEndianEncoding(), AutoDetectedEncoding);
             lengthOfBOM = 4;
         }
     } else if (c1 == 0xEF && c2 == 0xBB && c3 == 0xBF) {
         setEncoding(UTF8Encoding(), AutoDetectedEncoding);
         lengthOfBOM = 3;
-    } else if (c1 == 0xFE && c2 == 0xFF) {
+    } else if (m_encodingDetectionOption != AlwaysUseUTF8ForText && c1 == 0xFE && c2 == 0xFF) {
         setEncoding(UTF16BigEndianEncoding(), AutoDetectedEncoding);
         lengthOfBOM = 2;
-    } else if (!c1 && !c2 && c3 == 0xFE && c4 == 0xFF) {
+    } else if (m_encodingDetectionOption != AlwaysUseUTF8ForText && !c1 && !c2 && c3 == 0xFE && c4 == 0xFF) {
         setEncoding(UTF32BigEndianEncoding(), AutoDetectedEncoding);
         lengthOfBOM = 4;
     }
 
     if (lengthOfBOM || bufferLength + len >= 4)
         m_checkedForBOM = true;
 
     return lengthOfBOM;
 }
 
@@ skipping 115 matching lines @@
 //   2. Encoding detector is turned ON and the encoding is set to
 //      the encoding of the parent frame, which is also auto-detected.
 //   Note that condition #2 is NOT satisfied unless parent-child frame
 //   relationship is compliant to the same-origin policy. If they're from
 //   different domains, |m_source| would not be set to EncodingFromParentFrame
 //   in the first place.
 bool TextResourceDecoder::shouldAutoDetect() const
 {
     // Just checking m_hintEncoding suffices here because it's only set
     // in setHintEncoding when the source is AutoDetectedEncoding.
-    return m_usesEncodingDetector
+    return m_encodingDetectionOption == UseAllAutoDetection
         && (m_source == DefaultEncoding || (m_source == EncodingFromParentFrame && m_hintEncoding));
 }
 
 String TextResourceDecoder::decode(const char* data, size_t len)
 {
     size_t lengthOfBOM = 0;
     if (!m_checkedForBOM)
         lengthOfBOM = checkForBOM(data, len);
 
     bool movedDataToBuffer = false;
@@ skipping 59 matching lines @@
         m_codec = newTextCodec(m_encoding);
 
     String result = m_codec->decode(m_buffer.data(), m_buffer.size(), FetchEOF, m_contentType == XMLContent && !m_useLenientXMLDecoding, m_sawError);
     m_buffer.clear();
     m_codec.clear();
     m_checkedForBOM = false; // Skip BOM again when re-decoding.
     return result;
 }
 
 }