| OLD | NEW |
|---|
| (Empty) |
| 1 /* | |
| 2 Copyright (C) 1999 Lars Knoll (knoll@mpi-hd.mpg.de) | |
| 3 Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2012 Apple Inc. All
rights reserved. | |
| 4 Copyright (C) 2005, 2006, 2007 Alexey Proskuryakov (ap@nypop.com) | |
| 5 | |
| 6 This library is free software; you can redistribute it and/or | |
| 7 modify it under the terms of the GNU Library General Public | |
| 8 License as published by the Free Software Foundation; either | |
| 9 version 2 of the License, or (at your option) any later version. | |
| 10 | |
| 11 This library is distributed in the hope that it will be useful, | |
| 12 but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| 13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
| 14 Library General Public License for more details. | |
| 15 | |
| 16 You should have received a copy of the GNU Library General Public License | |
| 17 along with this library; see the file COPYING.LIB. If not, write to | |
| 18 the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, | |
| 19 Boston, MA 02110-1301, USA. | |
| 20 */ | |
| 21 | |
| 22 | |
| 23 #include "config.h" | |
| 24 #include "core/fetch/TextResourceDecoder.h" | |
| 25 | |
| 26 #include "HTMLNames.h" | |
| 27 #include "core/dom/DOMImplementation.h" | |
| 28 #include "core/html/parser/HTMLMetaCharsetParser.h" | |
| 29 #include "platform/text/TextEncodingDetector.h" | |
| 30 #include "wtf/StringExtras.h" | |
| 31 #include "wtf/text/TextCodec.h" | |
| 32 #include "wtf/text/TextEncoding.h" | |
| 33 #include "wtf/text/TextEncodingRegistry.h" | |
| 34 | |
| 35 using namespace WTF; | |
| 36 | |
| 37 namespace WebCore { | |
| 38 | |
| 39 using namespace HTMLNames; | |
| 40 | |
| 41 static inline bool bytesEqual(const char* p, char b0, char b1, char b2, char b3,
char b4) | |
| 42 { | |
| 43 return p[0] == b0 && p[1] == b1 && p[2] == b2 && p[3] == b3 && p[4] == b4; | |
| 44 } | |
| 45 | |
| 46 static inline bool bytesEqual(const char* p, char b0, char b1, char b2, char b3,
char b4, char b5) | |
| 47 { | |
| 48 return p[0] == b0 && p[1] == b1 && p[2] == b2 && p[3] == b3 && p[4] == b4 &&
p[5] == b5; | |
| 49 } | |
| 50 | |
| 51 static inline bool bytesEqual(const char* p, char b0, char b1, char b2, char b3,
char b4, char b5, char b6, char b7) | |
| 52 { | |
| 53 return p[0] == b0 && p[1] == b1 && p[2] == b2 && p[3] == b3 && p[4] == b4 &&
p[5] == b5 && p[6] == b6 && p[7] == b7; | |
| 54 } | |
| 55 | |
| 56 static inline bool bytesEqual(const char* p, char b0, char b1, char b2, char b3,
char b4, char b5, char b6, char b7, char b8, char b9) | |
| 57 { | |
| 58 return p[0] == b0 && p[1] == b1 && p[2] == b2 && p[3] == b3 && p[4] == b4 &&
p[5] == b5 && p[6] == b6 && p[7] == b7 && p[8] == b8 && p[9] == b9; | |
| 59 } | |
| 60 | |
| 61 // You might think we should put these find functions elsewhere, perhaps with th
e | |
| 62 // similar functions that operate on UChar, but arguably only the decoder has | |
| 63 // a reason to process strings of char rather than UChar. | |
| 64 | |
| 65 static int find(const char* subject, size_t subjectLength, const char* target) | |
| 66 { | |
| 67 size_t targetLength = strlen(target); | |
| 68 if (targetLength > subjectLength) | |
| 69 return -1; | |
| 70 for (size_t i = 0; i <= subjectLength - targetLength; ++i) { | |
| 71 bool match = true; | |
| 72 for (size_t j = 0; j < targetLength; ++j) { | |
| 73 if (subject[i + j] != target[j]) { | |
| 74 match = false; | |
| 75 break; | |
| 76 } | |
| 77 } | |
| 78 if (match) | |
| 79 return i; | |
| 80 } | |
| 81 return -1; | |
| 82 } | |
| 83 | |
| 84 static WTF::TextEncoding findTextEncoding(const char* encodingName, int length) | |
| 85 { | |
| 86 Vector<char, 64> buffer(length + 1); | |
| 87 memcpy(buffer.data(), encodingName, length); | |
| 88 buffer[length] = '\0'; | |
| 89 return buffer.data(); | |
| 90 } | |
| 91 | |
| 92 TextResourceDecoder::ContentType TextResourceDecoder::determineContentType(const
String& mimeType) | |
| 93 { | |
| 94 if (equalIgnoringCase(mimeType, "text/css")) | |
| 95 return CSS; | |
| 96 if (equalIgnoringCase(mimeType, "text/html")) | |
| 97 return HTML; | |
| 98 if (DOMImplementation::isXMLMIMEType(mimeType)) | |
| 99 return XML; | |
| 100 return PlainText; | |
| 101 } | |
| 102 | |
| 103 const WTF::TextEncoding& TextResourceDecoder::defaultEncoding(ContentType conten
tType, const WTF::TextEncoding& specifiedDefaultEncoding) | |
| 104 { | |
| 105 // Despite 8.5 "Text/xml with Omitted Charset" of RFC 3023, we assume UTF-8
instead of US-ASCII | |
| 106 // for text/xml. This matches Firefox. | |
| 107 if (contentType == XML) | |
| 108 return UTF8Encoding(); | |
| 109 if (!specifiedDefaultEncoding.isValid()) | |
| 110 return Latin1Encoding(); | |
| 111 return specifiedDefaultEncoding; | |
| 112 } | |
| 113 | |
| 114 TextResourceDecoder::TextResourceDecoder(const String& mimeType, const WTF::Text
Encoding& specifiedDefaultEncoding, bool usesEncodingDetector) | |
| 115 : m_contentType(determineContentType(mimeType)) | |
| 116 , m_encoding(defaultEncoding(m_contentType, specifiedDefaultEncoding)) | |
| 117 , m_source(DefaultEncoding) | |
| 118 , m_hintEncoding(0) | |
| 119 , m_checkedForBOM(false) | |
| 120 , m_checkedForCSSCharset(false) | |
| 121 , m_checkedForXMLCharset(false) | |
| 122 , m_checkedForMetaCharset(false) | |
| 123 , m_useLenientXMLDecoding(false) | |
| 124 , m_sawError(false) | |
| 125 , m_usesEncodingDetector(usesEncodingDetector) | |
| 126 { | |
| 127 } | |
| 128 | |
| 129 TextResourceDecoder::~TextResourceDecoder() | |
| 130 { | |
| 131 } | |
| 132 | |
| 133 void TextResourceDecoder::setEncoding(const WTF::TextEncoding& encoding, Encodin
gSource source) | |
| 134 { | |
| 135 // In case the encoding didn't exist, we keep the old one (helps some sites
specifying invalid encodings). | |
| 136 if (!encoding.isValid()) | |
| 137 return; | |
| 138 | |
| 139 // When encoding comes from meta tag (i.e. it cannot be XML files sent via X
HR), | |
| 140 // treat x-user-defined as windows-1252 (bug 18270) | |
| 141 if (source == EncodingFromMetaTag && !strcasecmp(encoding.name(), "x-user-de
fined")) | |
| 142 m_encoding = "windows-1252"; | |
| 143 else if (source == EncodingFromMetaTag || source == EncodingFromXMLHeader ||
source == EncodingFromCSSCharset) | |
| 144 m_encoding = encoding.closestByteBasedEquivalent(); | |
| 145 else | |
| 146 m_encoding = encoding; | |
| 147 | |
| 148 m_codec.clear(); | |
| 149 m_source = source; | |
| 150 } | |
| 151 | |
| 152 // Returns the position of the encoding string. | |
| 153 static int findXMLEncoding(const char* str, int len, int& encodingLength) | |
| 154 { | |
| 155 int pos = find(str, len, "encoding"); | |
| 156 if (pos == -1) | |
| 157 return -1; | |
| 158 pos += 8; | |
| 159 | |
| 160 // Skip spaces and stray control characters. | |
| 161 while (pos < len && str[pos] <= ' ') | |
| 162 ++pos; | |
| 163 | |
| 164 // Skip equals sign. | |
| 165 if (pos >= len || str[pos] != '=') | |
| 166 return -1; | |
| 167 ++pos; | |
| 168 | |
| 169 // Skip spaces and stray control characters. | |
| 170 while (pos < len && str[pos] <= ' ') | |
| 171 ++pos; | |
| 172 | |
| 173 // Skip quotation mark. | |
| 174 if (pos >= len) | |
| 175 return - 1; | |
| 176 char quoteMark = str[pos]; | |
| 177 if (quoteMark != '"' && quoteMark != '\'') | |
| 178 return -1; | |
| 179 ++pos; | |
| 180 | |
| 181 // Find the trailing quotation mark. | |
| 182 int end = pos; | |
| 183 while (end < len && str[end] != quoteMark) | |
| 184 ++end; | |
| 185 if (end >= len) | |
| 186 return -1; | |
| 187 | |
| 188 encodingLength = end - pos; | |
| 189 return pos; | |
| 190 } | |
| 191 | |
| 192 size_t TextResourceDecoder::checkForBOM(const char* data, size_t len) | |
| 193 { | |
| 194 // Check for UTF-16/32 or UTF-8 BOM mark at the beginning, which is a sure s
ign of a Unicode encoding. | |
| 195 // We let it override even a user-chosen encoding. | |
| 196 ASSERT(!m_checkedForBOM); | |
| 197 | |
| 198 size_t lengthOfBOM = 0; | |
| 199 | |
| 200 size_t bufferLength = m_buffer.size(); | |
| 201 | |
| 202 size_t buf1Len = bufferLength; | |
| 203 size_t buf2Len = len; | |
| 204 const unsigned char* buf1 = reinterpret_cast<const unsigned char*>(m_buffer.
data()); | |
| 205 const unsigned char* buf2 = reinterpret_cast<const unsigned char*>(data); | |
| 206 unsigned char c1 = buf1Len ? (--buf1Len, *buf1++) : buf2Len ? (--buf2Len, *b
uf2++) : 0; | |
| 207 unsigned char c2 = buf1Len ? (--buf1Len, *buf1++) : buf2Len ? (--buf2Len, *b
uf2++) : 0; | |
| 208 unsigned char c3 = buf1Len ? (--buf1Len, *buf1++) : buf2Len ? (--buf2Len, *b
uf2++) : 0; | |
| 209 unsigned char c4 = buf2Len ? (--buf2Len, *buf2++) : 0; | |
| 210 | |
| 211 // Check for the BOM. | |
| 212 if (c1 == 0xFF && c2 == 0xFE) { | |
| 213 if (c3 || c4) { | |
| 214 setEncoding(UTF16LittleEndianEncoding(), AutoDetectedEncoding); | |
| 215 lengthOfBOM = 2; | |
| 216 } else { | |
| 217 setEncoding(UTF32LittleEndianEncoding(), AutoDetectedEncoding); | |
| 218 lengthOfBOM = 4; | |
| 219 } | |
| 220 } else if (c1 == 0xEF && c2 == 0xBB && c3 == 0xBF) { | |
| 221 setEncoding(UTF8Encoding(), AutoDetectedEncoding); | |
| 222 lengthOfBOM = 3; | |
| 223 } else if (c1 == 0xFE && c2 == 0xFF) { | |
| 224 setEncoding(UTF16BigEndianEncoding(), AutoDetectedEncoding); | |
| 225 lengthOfBOM = 2; | |
| 226 } else if (!c1 && !c2 && c3 == 0xFE && c4 == 0xFF) { | |
| 227 setEncoding(UTF32BigEndianEncoding(), AutoDetectedEncoding); | |
| 228 lengthOfBOM = 4; | |
| 229 } | |
| 230 | |
| 231 if (lengthOfBOM || bufferLength + len >= 4) | |
| 232 m_checkedForBOM = true; | |
| 233 | |
| 234 return lengthOfBOM; | |
| 235 } | |
| 236 | |
| 237 bool TextResourceDecoder::checkForCSSCharset(const char* data, size_t len, bool&
movedDataToBuffer) | |
| 238 { | |
| 239 if (m_source != DefaultEncoding && m_source != EncodingFromParentFrame) { | |
| 240 m_checkedForCSSCharset = true; | |
| 241 return true; | |
| 242 } | |
| 243 | |
| 244 size_t oldSize = m_buffer.size(); | |
| 245 m_buffer.grow(oldSize + len); | |
| 246 memcpy(m_buffer.data() + oldSize, data, len); | |
| 247 | |
| 248 movedDataToBuffer = true; | |
| 249 | |
| 250 if (m_buffer.size() <= 13) // strlen('@charset "x";') == 13 | |
| 251 return false; | |
| 252 | |
| 253 const char* dataStart = m_buffer.data(); | |
| 254 const char* dataEnd = dataStart + m_buffer.size(); | |
| 255 | |
| 256 if (bytesEqual(dataStart, '@', 'c', 'h', 'a', 'r', 's', 'e', 't', ' ', '"'))
{ | |
| 257 dataStart += 10; | |
| 258 const char* pos = dataStart; | |
| 259 | |
| 260 while (pos < dataEnd && *pos != '"') | |
| 261 ++pos; | |
| 262 if (pos == dataEnd) | |
| 263 return false; | |
| 264 | |
| 265 int encodingNameLength = pos - dataStart; | |
| 266 | |
| 267 ++pos; | |
| 268 | |
| 269 if (*pos == ';') | |
| 270 setEncoding(findTextEncoding(dataStart, encodingNameLength), Encodin
gFromCSSCharset); | |
| 271 } | |
| 272 | |
| 273 m_checkedForCSSCharset = true; | |
| 274 return true; | |
| 275 } | |
| 276 | |
| 277 bool TextResourceDecoder::checkForXMLCharset(const char* data, size_t len, bool&
movedDataToBuffer) | |
| 278 { | |
| 279 if (m_source != DefaultEncoding && m_source != EncodingFromParentFrame) { | |
| 280 m_checkedForXMLCharset = true; | |
| 281 return true; | |
| 282 } | |
| 283 | |
| 284 // This is not completely efficient, since the function might go | |
| 285 // through the HTML head several times. | |
| 286 | |
| 287 size_t oldSize = m_buffer.size(); | |
| 288 m_buffer.grow(oldSize + len); | |
| 289 memcpy(m_buffer.data() + oldSize, data, len); | |
| 290 | |
| 291 movedDataToBuffer = true; | |
| 292 | |
| 293 const char* ptr = m_buffer.data(); | |
| 294 const char* pEnd = ptr + m_buffer.size(); | |
| 295 | |
| 296 // Is there enough data available to check for XML declaration? | |
| 297 if (m_buffer.size() < 8) | |
| 298 return false; | |
| 299 | |
| 300 // Handle XML declaration, which can have encoding in it. This encoding is h
onored even for HTML documents. | |
| 301 // It is an error for an XML declaration not to be at the start of an XML do
cument, and it is ignored in HTML documents in such case. | |
| 302 if (bytesEqual(ptr, '<', '?', 'x', 'm', 'l')) { | |
| 303 const char* xmlDeclarationEnd = ptr; | |
| 304 while (xmlDeclarationEnd != pEnd && *xmlDeclarationEnd != '>') | |
| 305 ++xmlDeclarationEnd; | |
| 306 if (xmlDeclarationEnd == pEnd) | |
| 307 return false; | |
| 308 // No need for +1, because we have an extra "?" to lose at the end of XM
L declaration. | |
| 309 int len = 0; | |
| 310 int pos = findXMLEncoding(ptr, xmlDeclarationEnd - ptr, len); | |
| 311 if (pos != -1) | |
| 312 setEncoding(findTextEncoding(ptr + pos, len), EncodingFromXMLHeader)
; | |
| 313 // continue looking for a charset - it may be specified in an HTTP-Equiv
meta | |
| 314 } else if (bytesEqual(ptr, '<', 0, '?', 0, 'x', 0)) { | |
| 315 setEncoding(UTF16LittleEndianEncoding(), AutoDetectedEncoding); | |
| 316 } else if (bytesEqual(ptr, 0, '<', 0, '?', 0, 'x')) { | |
| 317 setEncoding(UTF16BigEndianEncoding(), AutoDetectedEncoding); | |
| 318 } else if (bytesEqual(ptr, '<', 0, 0, 0, '?', 0, 0, 0)) { | |
| 319 setEncoding(UTF32LittleEndianEncoding(), AutoDetectedEncoding); | |
| 320 } else if (bytesEqual(ptr, 0, 0, 0, '<', 0, 0, 0, '?')) { | |
| 321 setEncoding(UTF32BigEndianEncoding(), AutoDetectedEncoding); | |
| 322 } | |
| 323 | |
| 324 m_checkedForXMLCharset = true; | |
| 325 return true; | |
| 326 } | |
| 327 | |
| 328 void TextResourceDecoder::checkForMetaCharset(const char* data, size_t length) | |
| 329 { | |
| 330 if (m_source == UserChosenEncoding || m_source == EncodingFromHTTPHeader ||
m_source == AutoDetectedEncoding) { | |
| 331 m_checkedForMetaCharset = true; | |
| 332 return; | |
| 333 } | |
| 334 | |
| 335 if (!m_charsetParser) | |
| 336 m_charsetParser = HTMLMetaCharsetParser::create(); | |
| 337 | |
| 338 if (!m_charsetParser->checkForMetaCharset(data, length)) | |
| 339 return; | |
| 340 | |
| 341 setEncoding(m_charsetParser->encoding(), EncodingFromMetaTag); | |
| 342 m_charsetParser.clear(); | |
| 343 m_checkedForMetaCharset = true; | |
| 344 return; | |
| 345 } | |
| 346 | |
| 347 // We use the encoding detector in two cases: | |
| 348 // 1. Encoding detector is turned ON and no other encoding source is | |
| 349 // available (that is, it's DefaultEncoding). | |
| 350 // 2. Encoding detector is turned ON and the encoding is set to | |
| 351 // the encoding of the parent frame, which is also auto-detected. | |
| 352 // Note that condition #2 is NOT satisfied unless parent-child frame | |
| 353 // relationship is compliant to the same-origin policy. If they're from | |
| 354 // different domains, |m_source| would not be set to EncodingFromParentFrame | |
| 355 // in the first place. | |
| 356 bool TextResourceDecoder::shouldAutoDetect() const | |
| 357 { | |
| 358 // Just checking m_hintEncoding suffices here because it's only set | |
| 359 // in setHintEncoding when the source is AutoDetectedEncoding. | |
| 360 return m_usesEncodingDetector | |
| 361 && (m_source == DefaultEncoding || (m_source == EncodingFromParentFrame
&& m_hintEncoding)); | |
| 362 } | |
| 363 | |
| 364 String TextResourceDecoder::decode(const char* data, size_t len) | |
| 365 { | |
| 366 size_t lengthOfBOM = 0; | |
| 367 if (!m_checkedForBOM) | |
| 368 lengthOfBOM = checkForBOM(data, len); | |
| 369 | |
| 370 bool movedDataToBuffer = false; | |
| 371 | |
| 372 if (m_contentType == CSS && !m_checkedForCSSCharset) { | |
| 373 if (!checkForCSSCharset(data, len, movedDataToBuffer)) | |
| 374 return emptyString(); | |
| 375 } | |
| 376 | |
| 377 if ((m_contentType == HTML || m_contentType == XML) && !m_checkedForXMLChars
et) { | |
| 378 if (!checkForXMLCharset(data, len, movedDataToBuffer)) | |
| 379 return emptyString(); | |
| 380 } | |
| 381 | |
| 382 const char* dataForDecode = data + lengthOfBOM; | |
| 383 size_t lengthForDecode = len - lengthOfBOM; | |
| 384 | |
| 385 if (!m_buffer.isEmpty()) { | |
| 386 if (!movedDataToBuffer) { | |
| 387 size_t oldSize = m_buffer.size(); | |
| 388 m_buffer.grow(oldSize + len); | |
| 389 memcpy(m_buffer.data() + oldSize, data, len); | |
| 390 } | |
| 391 | |
| 392 dataForDecode = m_buffer.data() + lengthOfBOM; | |
| 393 lengthForDecode = m_buffer.size() - lengthOfBOM; | |
| 394 } | |
| 395 | |
| 396 if (m_contentType == HTML && !m_checkedForMetaCharset) | |
| 397 checkForMetaCharset(dataForDecode, lengthForDecode); | |
| 398 | |
| 399 if (shouldAutoDetect()) { | |
| 400 WTF::TextEncoding detectedEncoding; | |
| 401 if (detectTextEncoding(data, len, m_hintEncoding, &detectedEncoding)) | |
| 402 setEncoding(detectedEncoding, EncodingFromContentSniffing); | |
| 403 } | |
| 404 | |
| 405 ASSERT(m_encoding.isValid()); | |
| 406 | |
| 407 if (!m_codec) | |
| 408 m_codec = newTextCodec(m_encoding); | |
| 409 | |
| 410 String result = m_codec->decode(dataForDecode, lengthForDecode, false, m_con
tentType == XML && !m_useLenientXMLDecoding, m_sawError); | |
| 411 | |
| 412 m_buffer.clear(); | |
| 413 return result; | |
| 414 } | |
| 415 | |
| 416 String TextResourceDecoder::flush() | |
| 417 { | |
| 418 // If we can not identify the encoding even after a document is completely | |
| 419 // loaded, we need to detect the encoding if other conditions for | |
| 420 // autodetection is satisfied. | |
| 421 if (m_buffer.size() && shouldAutoDetect() | |
| 422 && ((!m_checkedForXMLCharset && (m_contentType == HTML || m_contentType
== XML)) || (!m_checkedForCSSCharset && (m_contentType == CSS)))) { | |
| 423 WTF::TextEncoding detectedEncoding; | |
| 424 if (detectTextEncoding(m_buffer.data(), m_buffer.size(), m_hintEncoding,
&detectedEncoding)) | |
| 425 setEncoding(detectedEncoding, EncodingFromContentSniffing); | |
| 426 } | |
| 427 | |
| 428 if (!m_codec) | |
| 429 m_codec = newTextCodec(m_encoding); | |
| 430 | |
| 431 String result = m_codec->decode(m_buffer.data(), m_buffer.size(), true, m_co
ntentType == XML && !m_useLenientXMLDecoding, m_sawError); | |
| 432 m_buffer.clear(); | |
| 433 m_codec.clear(); | |
| 434 m_checkedForBOM = false; // Skip BOM again when re-decoding. | |
| 435 return result; | |
| 436 } | |
| 437 | |
| 438 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 74513003:
Moved text decoding to the parser thread (Closed)
Base URL: https://chromium.googlesource.com/chromium/blink.git@parserthread_step25