[Fetch] Always use utf-8 for decoding in text()

third_party/WebKit/Source/core/html/parser/TextResourceDecoder.cpp

 /*
     Copyright (C) 1999 Lars Knoll (knoll@mpi-hd.mpg.de)
     Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2012 Apple Inc. All rights reserved.
     Copyright (C) 2005, 2006, 2007 Alexey Proskuryakov (ap@nypop.com)
 
     This library is free software; you can redistribute it and/or
     modify it under the terms of the GNU Library General Public
     License as published by the Free Software Foundation; either
     version 2 of the License, or (at your option) any later version.
 
@@ skipping 94 matching lines @@
 {
     // Despite 8.5 "Text/xml with Omitted Charset" of RFC 3023, we assume UTF-8 instead of US-ASCII
     // for text/xml. This matches Firefox.
     if (contentType == XMLContent)
         return UTF8Encoding();
     if (!specifiedDefaultEncoding.isValid())
         return Latin1Encoding();
     return specifiedDefaultEncoding;
 }
 
-TextResourceDecoder::TextResourceDecoder(const String& mimeType, const WTF::TextEncoding& specifiedDefaultEncoding, bool usesEncodingDetector)
+TextResourceDecoder::TextResourceDecoder(const String& mimeType, const WTF::TextEncoding& specifiedDefaultEncoding, bool usesEncodingDetector, BOMCheckOptions bomCheckOptions)
     : m_contentType(determineContentType(mimeType))
     , m_encoding(defaultEncoding(m_contentType, specifiedDefaultEncoding))
     , m_source(DefaultEncoding)
     , m_hintEncoding(0)
     , m_checkedForBOM(false)
     , m_checkedForCSSCharset(false)
     , m_checkedForXMLCharset(false)
     , m_checkedForMetaCharset(false)
     , m_useLenientXMLDecoding(false)
     , m_sawError(false)
     , m_usesEncodingDetector(usesEncodingDetector)
+    , m_bomCheckOptions(bomCheckOptions)
 {
 }
 
 TextResourceDecoder::~TextResourceDecoder()
 {
 }
 
 void TextResourceDecoder::setEncoding(const WTF::TextEncoding& encoding, EncodingSource source)
 {
     // In case the encoding didn't exist, we keep the old one (helps some sites specifying invalid encodings).
@@ skipping 49 matching lines @@
     if (end >= len)
         return -1;
 
     encodingLength = end - pos;
     return pos;
 }
 
 size_t TextResourceDecoder::checkForBOM(const char* data, size_t len)
 {
     // Check for UTF-16/32 or UTF-8 BOM mark at the beginning, which is a sure sign of a Unicode encoding.
     // We let it override even a user-chosen encoding.

[yhirano, 2015/11/24 02:06:40]

This is the problem, right?
Rather than introducing a new parameter, I think it is good to expand
|usesEncodingDetector| constructor parameter so that a user can choose one of

 - UseAutoDetection (<= usesEncodingDetector == true)
 - UseBOMBasedDetectionOnly  (<= usesEncodingDetector == false, default value)
 - DoNotUseAutoDetection (<= Response.text())

. What do you think?

[hiroshige, 2015/11/30 09:50:01]

How about Patch Set 8:
  |usesEncodingDetector| can be:
  - UseAllAutoDetection (<= usesEncodingDetector == true)
  - UseContentAndBOMBasedDetection  (<= usesEncodingDetector == false, default
value)
  - AlwaysUseUTF8ForText (<= Response.text())
  and when it is AlwaysUseUTF8ForText, |defaultEncoding| should be utf-8, and
|mimeType| should be "text/plain".
This is not orthogonal, but implementing DoNotUseAutoDetection for all encodings
and non-text contents complicates the code:
- Even when we don't detect encoding, we should handle utf-8 BOMs in
AlwaysUseUTF8ForText, i.e. we should skip utf-8 BOMs (but not other BOMs such as
utf-16be BOMs) when we decode the content as utf-8.
  Supporting this BOM skipping for utf-16/32 makes code more complicated.
- In non-text contents (i.e. CSS/XML/HTML), TextResourceDecoder handles
encodings explicitly specified in those files (e.g. <meta> tag) and its
detection logic is built into other logics in the decoder.
  Disabling such logic makes code and state management of TextResourceDecoder
more complicated.
- And anyway we don't have such use cases.

     ASSERT(!m_checkedForBOM);
 
     size_t lengthOfBOM = 0;
 
     size_t bufferLength = m_buffer.size();
 
     size_t buf1Len = bufferLength;
     size_t buf2Len = len;
     const unsigned char* buf1 = reinterpret_cast<const unsigned char*>(m_buffer.data());
     const unsigned char* buf2 = reinterpret_cast<const unsigned char*>(data);
     unsigned char c1 = buf1Len ? (--buf1Len, *buf1++) : buf2Len ? (--buf2Len, *buf2++) : 0;
     unsigned char c2 = buf1Len ? (--buf1Len, *buf1++) : buf2Len ? (--buf2Len, *buf2++) : 0;
     unsigned char c3 = buf1Len ? (--buf1Len, *buf1++) : buf2Len ? (--buf2Len, *buf2++) : 0;
     unsigned char c4 = buf2Len ? (--buf2Len, *buf2++) : 0;
 
     // Check for the BOM.
-    if (c1 == 0xFF && c2 == 0xFE) {
+    if (m_bomCheckOptions == CheckForAllBOM && c1 == 0xFF && c2 == 0xFE) {
         if (c3 || c4) {
             setEncoding(UTF16LittleEndianEncoding(), AutoDetectedEncoding);
             lengthOfBOM = 2;
         } else {
             setEncoding(UTF32LittleEndianEncoding(), AutoDetectedEncoding);
             lengthOfBOM = 4;
         }
     } else if (c1 == 0xEF && c2 == 0xBB && c3 == 0xBF) {
         setEncoding(UTF8Encoding(), AutoDetectedEncoding);
         lengthOfBOM = 3;
-    } else if (c1 == 0xFE && c2 == 0xFF) {
+    } else if (m_bomCheckOptions == CheckForAllBOM && c1 == 0xFE && c2 == 0xFF) {
         setEncoding(UTF16BigEndianEncoding(), AutoDetectedEncoding);
         lengthOfBOM = 2;
-    } else if (!c1 && !c2 && c3 == 0xFE && c4 == 0xFF) {
+    } else if (m_bomCheckOptions == CheckForAllBOM && !c1 && !c2 && c3 == 0xFE && c4 == 0xFF) {
         setEncoding(UTF32BigEndianEncoding(), AutoDetectedEncoding);
         lengthOfBOM = 4;
     }
 
     if (lengthOfBOM || bufferLength + len >= 4)
         m_checkedForBOM = true;
 
     return lengthOfBOM;
 }
 
@@ skipping 195 matching lines @@
         m_codec = newTextCodec(m_encoding);
 
     String result = m_codec->decode(m_buffer.data(), m_buffer.size(), FetchEOF, m_contentType == XMLContent && !m_useLenientXMLDecoding, m_sawError);
     m_buffer.clear();
     m_codec.clear();
     m_checkedForBOM = false; // Skip BOM again when re-decoding.
     return result;
 }
 
 }