| OLD | NEW |
| 1 /* | 1 /* |
| 2 * Copyright (C) 2004, 2006, 2007, 2008, 2011 Apple Inc. All rights reserved. | 2 * Copyright (C) 2004, 2006, 2007, 2008, 2011 Apple Inc. All rights reserved. |
| 3 * Copyright (C) 2006 Alexey Proskuryakov <ap@nypop.com> | 3 * Copyright (C) 2006 Alexey Proskuryakov <ap@nypop.com> |
| 4 * | 4 * |
| 5 * Redistribution and use in source and binary forms, with or without | 5 * Redistribution and use in source and binary forms, with or without |
| 6 * modification, are permitted provided that the following conditions | 6 * modification, are permitted provided that the following conditions |
| 7 * are met: | 7 * are met: |
| 8 * 1. Redistributions of source code must retain the above copyright | 8 * 1. Redistributions of source code must retain the above copyright |
| 9 * notice, this list of conditions and the following disclaimer. | 9 * notice, this list of conditions and the following disclaimer. |
| 10 * 2. Redistributions in binary form must reproduce the above copyright | 10 * 2. Redistributions in binary form must reproduce the above copyright |
| (...skipping 23 matching lines...) Expand all Loading... |
| 34 #include "wtf/text/CString.h" | 34 #include "wtf/text/CString.h" |
| 35 #include "wtf/text/CharacterNames.h" | 35 #include "wtf/text/CharacterNames.h" |
| 36 #include "wtf/text/StringBuilder.h" | 36 #include "wtf/text/StringBuilder.h" |
| 37 #include <unicode/ucnv.h> | 37 #include <unicode/ucnv.h> |
| 38 #include <unicode/ucnv_cb.h> | 38 #include <unicode/ucnv_cb.h> |
| 39 | 39 |
| 40 namespace WTF { | 40 namespace WTF { |
| 41 | 41 |
| 42 const size_t ConversionBufferSize = 16384; | 42 const size_t ConversionBufferSize = 16384; |
| 43 | 43 |
| 44 ICUConverterWrapper::~ICUConverterWrapper() | 44 ICUConverterWrapper::~ICUConverterWrapper() { |
| 45 if (converter) |
| 46 ucnv_close(converter); |
| 47 } |
| 48 |
| 49 static UConverter*& cachedConverterICU() { |
| 50 return wtfThreadData().cachedConverterICU().converter; |
| 51 } |
| 52 |
| 53 PassOwnPtr<TextCodec> TextCodecICU::create(const TextEncoding& encoding, const v
oid*) { |
| 54 return adoptPtr(new TextCodecICU(encoding)); |
| 55 } |
| 56 |
| 57 void TextCodecICU::registerEncodingNames(EncodingNameRegistrar registrar) { |
| 58 // We register Hebrew with logical ordering using a separate name. |
| 59 // Otherwise, this would share the same canonical name as the |
| 60 // visual ordering case, and then TextEncoding could not tell them |
| 61 // apart; ICU treats these names as synonyms. |
| 62 registrar("ISO-8859-8-I", "ISO-8859-8-I"); |
| 63 |
| 64 int32_t numEncodings = ucnv_countAvailable(); |
| 65 for (int32_t i = 0; i < numEncodings; ++i) { |
| 66 const char* name = ucnv_getAvailableName(i); |
| 67 UErrorCode error = U_ZERO_ERROR; |
| 68 #if !defined(USING_SYSTEM_ICU) |
| 69 const char* primaryStandard = "HTML"; |
| 70 const char* secondaryStandard = "MIME"; |
| 71 #else |
| 72 const char* primaryStandard = "MIME"; |
| 73 const char* secondaryStandard = "IANA"; |
| 74 #endif |
| 75 const char* standardName = ucnv_getStandardName(name, primaryStandard, &erro
r); |
| 76 if (U_FAILURE(error) || !standardName) { |
| 77 error = U_ZERO_ERROR; |
| 78 // Try IANA to pick up 'windows-12xx' and other names |
| 79 // which are not preferred MIME names but are widely used. |
| 80 standardName = ucnv_getStandardName(name, secondaryStandard, &error); |
| 81 if (U_FAILURE(error) || !standardName) |
| 82 continue; |
| 83 } |
| 84 |
| 85 // A number of these aliases are handled in Chrome's copy of ICU, but |
| 86 // Chromium can be compiled with the system ICU. |
| 87 |
| 88 // 1. Treat GB2312 encoding as GBK (its more modern superset), to match other br
owsers. |
| 89 // 2. On the Web, GB2312 is encoded as EUC-CN or HZ, while ICU provides a native
encoding |
| 90 // for encoding GB_2312-80 and several others. So, we need to override this b
ehavior, too. |
| 91 #if defined(USING_SYSTEM_ICU) |
| 92 if (!strcmp(standardName, "GB2312") || !strcmp(standardName, "GB_2312-80")) |
| 93 standardName = "GBK"; |
| 94 // Similarly, EUC-KR encodings all map to an extended version, but |
| 95 // per HTML5, the canonical name still should be EUC-KR. |
| 96 else if (!strcmp(standardName, "EUC-KR") || !strcmp(standardName, "KSC_5601"
) || !strcmp(standardName, "cp1363")) |
| 97 standardName = "EUC-KR"; |
| 98 // And so on. |
| 99 else if (!strcasecmp(standardName, "iso-8859-9")) // This name is returned
in different case by ICU 3.2 and 3.6. |
| 100 standardName = "windows-1254"; |
| 101 else if (!strcmp(standardName, "TIS-620")) |
| 102 standardName = "windows-874"; |
| 103 #endif |
| 104 |
| 105 registrar(standardName, standardName); |
| 106 |
| 107 uint16_t numAliases = ucnv_countAliases(name, &error); |
| 108 ASSERT(U_SUCCESS(error)); |
| 109 if (U_SUCCESS(error)) |
| 110 for (uint16_t j = 0; j < numAliases; ++j) { |
| 111 error = U_ZERO_ERROR; |
| 112 const char* alias = ucnv_getAlias(name, j, &error); |
| 113 ASSERT(U_SUCCESS(error)); |
| 114 if (U_SUCCESS(error) && alias != standardName) |
| 115 registrar(alias, standardName); |
| 116 } |
| 117 } |
| 118 |
| 119 // These two entries have to be added here because ICU's converter table |
| 120 // cannot have both ISO-8859-8-I and ISO-8859-8. |
| 121 registrar("csISO88598I", "ISO-8859-8-I"); |
| 122 registrar("logical", "ISO-8859-8-I"); |
| 123 |
| 124 #if defined(USING_SYSTEM_ICU) |
| 125 // Additional alias for MacCyrillic not present in ICU. |
| 126 registrar("maccyrillic", "x-mac-cyrillic"); |
| 127 |
| 128 // Additional aliases that historically were present in the encoding |
| 129 // table in WebKit on Macintosh that don't seem to be present in ICU. |
| 130 // Perhaps we can prove these are not used on the web and remove them. |
| 131 // Or perhaps we can get them added to ICU. |
| 132 registrar("x-mac-roman", "macintosh"); |
| 133 registrar("x-mac-ukrainian", "x-mac-cyrillic"); |
| 134 registrar("cn-big5", "Big5"); |
| 135 registrar("x-x-big5", "Big5"); |
| 136 registrar("cn-gb", "GBK"); |
| 137 registrar("csgb231280", "GBK"); |
| 138 registrar("x-euc-cn", "GBK"); |
| 139 registrar("x-gbk", "GBK"); |
| 140 registrar("koi", "KOI8-R"); |
| 141 registrar("visual", "ISO-8859-8"); |
| 142 registrar("winarabic", "windows-1256"); |
| 143 registrar("winbaltic", "windows-1257"); |
| 144 registrar("wincyrillic", "windows-1251"); |
| 145 registrar("iso-8859-11", "windows-874"); |
| 146 registrar("iso8859-11", "windows-874"); |
| 147 registrar("dos-874", "windows-874"); |
| 148 registrar("wingreek", "windows-1253"); |
| 149 registrar("winhebrew", "windows-1255"); |
| 150 registrar("winlatin2", "windows-1250"); |
| 151 registrar("winturkish", "windows-1254"); |
| 152 registrar("winvietnamese", "windows-1258"); |
| 153 registrar("x-cp1250", "windows-1250"); |
| 154 registrar("x-cp1251", "windows-1251"); |
| 155 registrar("x-euc", "EUC-JP"); |
| 156 registrar("x-windows-949", "EUC-KR"); |
| 157 registrar("KSC5601", "EUC-KR"); |
| 158 registrar("x-uhc", "EUC-KR"); |
| 159 registrar("shift-jis", "Shift_JIS"); |
| 160 |
| 161 // Alternative spelling of ISO encoding names. |
| 162 registrar("ISO8859-1", "ISO-8859-1"); |
| 163 registrar("ISO8859-2", "ISO-8859-2"); |
| 164 registrar("ISO8859-3", "ISO-8859-3"); |
| 165 registrar("ISO8859-4", "ISO-8859-4"); |
| 166 registrar("ISO8859-5", "ISO-8859-5"); |
| 167 registrar("ISO8859-6", "ISO-8859-6"); |
| 168 registrar("ISO8859-7", "ISO-8859-7"); |
| 169 registrar("ISO8859-8", "ISO-8859-8"); |
| 170 registrar("ISO8859-8-I", "ISO-8859-8-I"); |
| 171 registrar("ISO8859-9", "ISO-8859-9"); |
| 172 registrar("ISO8859-10", "ISO-8859-10"); |
| 173 registrar("ISO8859-13", "ISO-8859-13"); |
| 174 registrar("ISO8859-14", "ISO-8859-14"); |
| 175 registrar("ISO8859-15", "ISO-8859-15"); |
| 176 // No need to have an entry for ISO8859-16. ISO-8859-16 has just one label |
| 177 // listed in WHATWG Encoding Living Standard (http://encoding.spec.whatwg.org/
). |
| 178 |
| 179 // Additional aliases present in the WHATWG Encoding Standard |
| 180 // and Firefox (as of Oct 2014), but not in the upstream ICU. |
| 181 // Three entries for windows-1252 need not be listed here because |
| 182 // TextCodecLatin1 registers them. |
| 183 registrar("csiso58gb231280", "GBK"); |
| 184 registrar("csiso88596e", "ISO-8859-6"); |
| 185 registrar("csiso88596i", "ISO-8859-6"); |
| 186 registrar("csiso88598e", "ISO-8859-8"); |
| 187 registrar("gb_2312", "GBK"); |
| 188 registrar("iso88592", "ISO-8859-2"); |
| 189 registrar("iso88593", "ISO-8859-3"); |
| 190 registrar("iso88594", "ISO-8859-4"); |
| 191 registrar("iso88595", "ISO-8859-5"); |
| 192 registrar("iso88596", "ISO-8859-6"); |
| 193 registrar("iso88597", "ISO-8859-7"); |
| 194 registrar("iso88598", "ISO-8859-8"); |
| 195 registrar("iso88599", "windows-1254"); |
| 196 registrar("iso885910", "ISO-8859-10"); |
| 197 registrar("iso885911", "windows-874"); |
| 198 registrar("iso885913", "ISO-8859-13"); |
| 199 registrar("iso885914", "ISO-8859-14"); |
| 200 registrar("iso885915", "ISO-8859-15"); |
| 201 registrar("iso_8859-2", "ISO-8859-2"); |
| 202 registrar("iso_8859-3", "ISO-8859-3"); |
| 203 registrar("iso_8859-4", "ISO-8859-4"); |
| 204 registrar("iso_8859-5", "ISO-8859-5"); |
| 205 registrar("iso_8859-6", "ISO-8859-6"); |
| 206 registrar("iso_8859-7", "ISO-8859-7"); |
| 207 registrar("iso_8859-8", "ISO-8859-8"); |
| 208 registrar("iso_8859-9", "windows-1254"); |
| 209 registrar("iso_8859-15", "ISO-8859-15"); |
| 210 registrar("koi8_r", "KOI8-R"); |
| 211 registrar("x-cp1253", "windows-1253"); |
| 212 registrar("x-cp1254", "windows-1254"); |
| 213 registrar("x-cp1255", "windows-1255"); |
| 214 registrar("x-cp1256", "windows-1256"); |
| 215 registrar("x-cp1257", "windows-1257"); |
| 216 registrar("x-cp1258", "windows-1258"); |
| 217 #endif |
| 218 } |
| 219 |
| 220 void TextCodecICU::registerCodecs(TextCodecRegistrar registrar) { |
| 221 // See comment above in registerEncodingNames. |
| 222 registrar("ISO-8859-8-I", create, 0); |
| 223 |
| 224 int32_t numEncodings = ucnv_countAvailable(); |
| 225 for (int32_t i = 0; i < numEncodings; ++i) { |
| 226 const char* name = ucnv_getAvailableName(i); |
| 227 UErrorCode error = U_ZERO_ERROR; |
| 228 const char* standardName = ucnv_getStandardName(name, "MIME", &error); |
| 229 if (!U_SUCCESS(error) || !standardName) { |
| 230 error = U_ZERO_ERROR; |
| 231 standardName = ucnv_getStandardName(name, "IANA", &error); |
| 232 if (!U_SUCCESS(error) || !standardName) |
| 233 continue; |
| 234 } |
| 235 registrar(standardName, create, 0); |
| 236 } |
| 237 } |
| 238 |
| 239 TextCodecICU::TextCodecICU(const TextEncoding& encoding) |
| 240 : m_encoding(encoding), m_converterICU(0) |
| 241 #if defined(USING_SYSTEM_ICU) |
| 242 , |
| 243 m_needsGBKFallbacks(false) |
| 244 #endif |
| 45 { | 245 { |
| 46 if (converter) | 246 } |
| 47 ucnv_close(converter); | 247 |
| 48 } | 248 TextCodecICU::~TextCodecICU() { |
| 49 | 249 releaseICUConverter(); |
| 50 static UConverter*& cachedConverterICU() | 250 } |
| 51 { | 251 |
| 52 return wtfThreadData().cachedConverterICU().converter; | 252 void TextCodecICU::releaseICUConverter() const { |
| 53 } | 253 if (m_converterICU) { |
| 54 | 254 UConverter*& cachedConverter = cachedConverterICU(); |
| 55 PassOwnPtr<TextCodec> TextCodecICU::create(const TextEncoding& encoding, const v
oid*) | 255 if (cachedConverter) |
| 56 { | 256 ucnv_close(cachedConverter); |
| 57 return adoptPtr(new TextCodecICU(encoding)); | 257 cachedConverter = m_converterICU; |
| 58 } | 258 m_converterICU = 0; |
| 59 | 259 } |
| 60 void TextCodecICU::registerEncodingNames(EncodingNameRegistrar registrar) | 260 } |
| 61 { | 261 |
| 62 // We register Hebrew with logical ordering using a separate name. | 262 void TextCodecICU::createICUConverter() const { |
| 63 // Otherwise, this would share the same canonical name as the | 263 ASSERT(!m_converterICU); |
| 64 // visual ordering case, and then TextEncoding could not tell them | 264 |
| 65 // apart; ICU treats these names as synonyms. | 265 #if defined(USING_SYSTEM_ICU) |
| 66 registrar("ISO-8859-8-I", "ISO-8859-8-I"); | 266 const char* name = m_encoding.name(); |
| 67 | 267 m_needsGBKFallbacks = name[0] == 'G' && name[1] == 'B' && name[2] == 'K' && !n
ame[3]; |
| 68 int32_t numEncodings = ucnv_countAvailable(); | 268 #endif |
| 69 for (int32_t i = 0; i < numEncodings; ++i) { | 269 |
| 70 const char* name = ucnv_getAvailableName(i); | 270 UErrorCode err; |
| 71 UErrorCode error = U_ZERO_ERROR; | 271 |
| 272 UConverter*& cachedConverter = cachedConverterICU(); |
| 273 if (cachedConverter) { |
| 274 err = U_ZERO_ERROR; |
| 275 const char* cachedName = ucnv_getName(cachedConverter, &err); |
| 276 if (U_SUCCESS(err) && m_encoding == cachedName) { |
| 277 m_converterICU = cachedConverter; |
| 278 cachedConverter = 0; |
| 279 return; |
| 280 } |
| 281 } |
| 282 |
| 283 err = U_ZERO_ERROR; |
| 284 m_converterICU = ucnv_open(m_encoding.name(), &err); |
| 285 #if !LOG_DISABLED |
| 286 if (err == U_AMBIGUOUS_ALIAS_WARNING) |
| 287 WTF_LOG_ERROR("ICU ambiguous alias warning for encoding: %s", m_encoding.nam
e()); |
| 288 #endif |
| 289 if (m_converterICU) |
| 290 ucnv_setFallback(m_converterICU, TRUE); |
| 291 } |
| 292 |
| 293 int TextCodecICU::decodeToBuffer(UChar* target, UChar* targetLimit, const char*&
source, const char* sourceLimit, int32_t* offsets, bool flush, UErrorCode& err)
{ |
| 294 UChar* targetStart = target; |
| 295 err = U_ZERO_ERROR; |
| 296 ucnv_toUnicode(m_converterICU, &target, targetLimit, &source, sourceLimit, off
sets, flush, &err); |
| 297 return target - targetStart; |
| 298 } |
| 299 |
| 300 class ErrorCallbackSetter { |
| 301 public: |
| 302 ErrorCallbackSetter(UConverter* converter, bool stopOnError) |
| 303 : m_converter(converter), m_shouldStopOnEncodingErrors(stopOnError) { |
| 304 if (m_shouldStopOnEncodingErrors) { |
| 305 UErrorCode err = U_ZERO_ERROR; |
| 306 ucnv_setToUCallBack(m_converter, UCNV_TO_U_CALLBACK_SUBSTITUTE, |
| 307 UCNV_SUB_STOP_ON_ILLEGAL, &m_savedAction, |
| 308 &m_savedContext, &err); |
| 309 ASSERT(err == U_ZERO_ERROR); |
| 310 } |
| 311 } |
| 312 ~ErrorCallbackSetter() { |
| 313 if (m_shouldStopOnEncodingErrors) { |
| 314 UErrorCode err = U_ZERO_ERROR; |
| 315 const void* oldContext; |
| 316 UConverterToUCallback oldAction; |
| 317 ucnv_setToUCallBack(m_converter, m_savedAction, m_savedContext, &oldAction
, &oldContext, &err); |
| 318 ASSERT(oldAction == UCNV_TO_U_CALLBACK_SUBSTITUTE); |
| 319 ASSERT(!strcmp(static_cast<const char*>(oldContext), UCNV_SUB_STOP_ON_ILLE
GAL)); |
| 320 ASSERT(err == U_ZERO_ERROR); |
| 321 } |
| 322 } |
| 323 |
| 324 private: |
| 325 UConverter* m_converter; |
| 326 bool m_shouldStopOnEncodingErrors; |
| 327 const void* m_savedContext; |
| 328 UConverterToUCallback m_savedAction; |
| 329 }; |
| 330 |
| 331 String TextCodecICU::decode(const char* bytes, size_t length, FlushBehavior flus
h, bool stopOnError, bool& sawError) { |
| 332 // Get a converter for the passed-in encoding. |
| 333 if (!m_converterICU) { |
| 334 createICUConverter(); |
| 335 ASSERT(m_converterICU); |
| 336 if (!m_converterICU) { |
| 337 WTF_LOG_ERROR("error creating ICU encoder even though encoding was in tabl
e"); |
| 338 return String(); |
| 339 } |
| 340 } |
| 341 |
| 342 ErrorCallbackSetter callbackSetter(m_converterICU, stopOnError); |
| 343 |
| 344 StringBuilder result; |
| 345 |
| 346 UChar buffer[ConversionBufferSize]; |
| 347 UChar* bufferLimit = buffer + ConversionBufferSize; |
| 348 const char* source = reinterpret_cast<const char*>(bytes); |
| 349 const char* sourceLimit = source + length; |
| 350 int32_t* offsets = nullptr; |
| 351 UErrorCode err = U_ZERO_ERROR; |
| 352 |
| 353 do { |
| 354 int ucharsDecoded = decodeToBuffer(buffer, bufferLimit, source, sourceLimit,
offsets, flush != DoNotFlush, err); |
| 355 result.append(buffer, ucharsDecoded); |
| 356 } while (err == U_BUFFER_OVERFLOW_ERROR); |
| 357 |
| 358 if (U_FAILURE(err)) { |
| 359 // flush the converter so it can be reused, and not be bothered by this erro
r. |
| 360 do { |
| 361 decodeToBuffer(buffer, bufferLimit, source, sourceLimit, offsets, true, er
r); |
| 362 } while (source < sourceLimit); |
| 363 sawError = true; |
| 364 } |
| 365 |
| 72 #if !defined(USING_SYSTEM_ICU) | 366 #if !defined(USING_SYSTEM_ICU) |
| 73 const char* primaryStandard = "HTML"; | 367 // Chrome's copy of ICU does not have the issue described below. |
| 74 const char* secondaryStandard = "MIME"; | 368 return result.toString(); |
| 75 #else | 369 #else |
| 76 const char* primaryStandard = "MIME"; | 370 String resultString = result.toString(); |
| 77 const char* secondaryStandard = "IANA"; | 371 |
| 78 #endif | 372 // <http://bugs.webkit.org/show_bug.cgi?id=17014> |
| 79 const char* standardName = ucnv_getStandardName(name, primaryStandard, &
error); | 373 // Simplified Chinese pages use the code A3A0 to mean "full-width space", but
ICU decodes it as U+E5E5. |
| 80 if (U_FAILURE(error) || !standardName) { | 374 if (!strcmp(m_encoding.name(), "GBK")) { |
| 81 error = U_ZERO_ERROR; | 375 if (!strcasecmp(m_encoding.name(), "gb18030")) |
| 82 // Try IANA to pick up 'windows-12xx' and other names | 376 resultString.replace(0xE5E5, ideographicSpaceCharacter); |
| 83 // which are not preferred MIME names but are widely used. | 377 // Make GBK compliant to the encoding spec and align with GB18030 |
| 84 standardName = ucnv_getStandardName(name, secondaryStandard, &error)
; | 378 resultString.replace(0x01F9, 0xE7C8); |
| 85 if (U_FAILURE(error) || !standardName) | 379 // FIXME: Once https://www.w3.org/Bugs/Public/show_bug.cgi?id=28740#c3 |
| 86 continue; | 380 // is resolved, add U+1E3F => 0xE7C7. |
| 87 } | 381 } |
| 88 | 382 |
| 89 // A number of these aliases are handled in Chrome's copy of ICU, but | 383 return resultString; |
| 90 // Chromium can be compiled with the system ICU. | |
| 91 | |
| 92 // 1. Treat GB2312 encoding as GBK (its more modern superset), to match
other browsers. | |
| 93 // 2. On the Web, GB2312 is encoded as EUC-CN or HZ, while ICU provides
a native encoding | |
| 94 // for encoding GB_2312-80 and several others. So, we need to overrid
e this behavior, too. | |
| 95 #if defined(USING_SYSTEM_ICU) | |
| 96 if (!strcmp(standardName, "GB2312") || !strcmp(standardName, "GB_2312-80
")) | |
| 97 standardName = "GBK"; | |
| 98 // Similarly, EUC-KR encodings all map to an extended version, but | |
| 99 // per HTML5, the canonical name still should be EUC-KR. | |
| 100 else if (!strcmp(standardName, "EUC-KR") || !strcmp(standardName, "KSC_5
601") || !strcmp(standardName, "cp1363")) | |
| 101 standardName = "EUC-KR"; | |
| 102 // And so on. | |
| 103 else if (!strcasecmp(standardName, "iso-8859-9")) // This name is return
ed in different case by ICU 3.2 and 3.6. | |
| 104 standardName = "windows-1254"; | |
| 105 else if (!strcmp(standardName, "TIS-620")) | |
| 106 standardName = "windows-874"; | |
| 107 #endif | |
| 108 | |
| 109 registrar(standardName, standardName); | |
| 110 | |
| 111 uint16_t numAliases = ucnv_countAliases(name, &error); | |
| 112 ASSERT(U_SUCCESS(error)); | |
| 113 if (U_SUCCESS(error)) | |
| 114 for (uint16_t j = 0; j < numAliases; ++j) { | |
| 115 error = U_ZERO_ERROR; | |
| 116 const char* alias = ucnv_getAlias(name, j, &error); | |
| 117 ASSERT(U_SUCCESS(error)); | |
| 118 if (U_SUCCESS(error) && alias != standardName) | |
| 119 registrar(alias, standardName); | |
| 120 } | |
| 121 } | |
| 122 | |
| 123 // These two entries have to be added here because ICU's converter table | |
| 124 // cannot have both ISO-8859-8-I and ISO-8859-8. | |
| 125 registrar("csISO88598I", "ISO-8859-8-I"); | |
| 126 registrar("logical", "ISO-8859-8-I"); | |
| 127 | |
| 128 #if defined(USING_SYSTEM_ICU) | |
| 129 // Additional alias for MacCyrillic not present in ICU. | |
| 130 registrar("maccyrillic", "x-mac-cyrillic"); | |
| 131 | |
| 132 // Additional aliases that historically were present in the encoding | |
| 133 // table in WebKit on Macintosh that don't seem to be present in ICU. | |
| 134 // Perhaps we can prove these are not used on the web and remove them. | |
| 135 // Or perhaps we can get them added to ICU. | |
| 136 registrar("x-mac-roman", "macintosh"); | |
| 137 registrar("x-mac-ukrainian", "x-mac-cyrillic"); | |
| 138 registrar("cn-big5", "Big5"); | |
| 139 registrar("x-x-big5", "Big5"); | |
| 140 registrar("cn-gb", "GBK"); | |
| 141 registrar("csgb231280", "GBK"); | |
| 142 registrar("x-euc-cn", "GBK"); | |
| 143 registrar("x-gbk", "GBK"); | |
| 144 registrar("koi", "KOI8-R"); | |
| 145 registrar("visual", "ISO-8859-8"); | |
| 146 registrar("winarabic", "windows-1256"); | |
| 147 registrar("winbaltic", "windows-1257"); | |
| 148 registrar("wincyrillic", "windows-1251"); | |
| 149 registrar("iso-8859-11", "windows-874"); | |
| 150 registrar("iso8859-11", "windows-874"); | |
| 151 registrar("dos-874", "windows-874"); | |
| 152 registrar("wingreek", "windows-1253"); | |
| 153 registrar("winhebrew", "windows-1255"); | |
| 154 registrar("winlatin2", "windows-1250"); | |
| 155 registrar("winturkish", "windows-1254"); | |
| 156 registrar("winvietnamese", "windows-1258"); | |
| 157 registrar("x-cp1250", "windows-1250"); | |
| 158 registrar("x-cp1251", "windows-1251"); | |
| 159 registrar("x-euc", "EUC-JP"); | |
| 160 registrar("x-windows-949", "EUC-KR"); | |
| 161 registrar("KSC5601", "EUC-KR"); | |
| 162 registrar("x-uhc", "EUC-KR"); | |
| 163 registrar("shift-jis", "Shift_JIS"); | |
| 164 | |
| 165 // Alternative spelling of ISO encoding names. | |
| 166 registrar("ISO8859-1", "ISO-8859-1"); | |
| 167 registrar("ISO8859-2", "ISO-8859-2"); | |
| 168 registrar("ISO8859-3", "ISO-8859-3"); | |
| 169 registrar("ISO8859-4", "ISO-8859-4"); | |
| 170 registrar("ISO8859-5", "ISO-8859-5"); | |
| 171 registrar("ISO8859-6", "ISO-8859-6"); | |
| 172 registrar("ISO8859-7", "ISO-8859-7"); | |
| 173 registrar("ISO8859-8", "ISO-8859-8"); | |
| 174 registrar("ISO8859-8-I", "ISO-8859-8-I"); | |
| 175 registrar("ISO8859-9", "ISO-8859-9"); | |
| 176 registrar("ISO8859-10", "ISO-8859-10"); | |
| 177 registrar("ISO8859-13", "ISO-8859-13"); | |
| 178 registrar("ISO8859-14", "ISO-8859-14"); | |
| 179 registrar("ISO8859-15", "ISO-8859-15"); | |
| 180 // No need to have an entry for ISO8859-16. ISO-8859-16 has just one label | |
| 181 // listed in WHATWG Encoding Living Standard (http://encoding.spec.whatwg.or
g/ ). | |
| 182 | |
| 183 // Additional aliases present in the WHATWG Encoding Standard | |
| 184 // and Firefox (as of Oct 2014), but not in the upstream ICU. | |
| 185 // Three entries for windows-1252 need not be listed here because | |
| 186 // TextCodecLatin1 registers them. | |
| 187 registrar("csiso58gb231280", "GBK"); | |
| 188 registrar("csiso88596e", "ISO-8859-6"); | |
| 189 registrar("csiso88596i", "ISO-8859-6"); | |
| 190 registrar("csiso88598e", "ISO-8859-8"); | |
| 191 registrar("gb_2312", "GBK"); | |
| 192 registrar("iso88592", "ISO-8859-2"); | |
| 193 registrar("iso88593", "ISO-8859-3"); | |
| 194 registrar("iso88594", "ISO-8859-4"); | |
| 195 registrar("iso88595", "ISO-8859-5"); | |
| 196 registrar("iso88596", "ISO-8859-6"); | |
| 197 registrar("iso88597", "ISO-8859-7"); | |
| 198 registrar("iso88598", "ISO-8859-8"); | |
| 199 registrar("iso88599", "windows-1254"); | |
| 200 registrar("iso885910", "ISO-8859-10"); | |
| 201 registrar("iso885911", "windows-874"); | |
| 202 registrar("iso885913", "ISO-8859-13"); | |
| 203 registrar("iso885914", "ISO-8859-14"); | |
| 204 registrar("iso885915", "ISO-8859-15"); | |
| 205 registrar("iso_8859-2", "ISO-8859-2"); | |
| 206 registrar("iso_8859-3", "ISO-8859-3"); | |
| 207 registrar("iso_8859-4", "ISO-8859-4"); | |
| 208 registrar("iso_8859-5", "ISO-8859-5"); | |
| 209 registrar("iso_8859-6", "ISO-8859-6"); | |
| 210 registrar("iso_8859-7", "ISO-8859-7"); | |
| 211 registrar("iso_8859-8", "ISO-8859-8"); | |
| 212 registrar("iso_8859-9", "windows-1254"); | |
| 213 registrar("iso_8859-15", "ISO-8859-15"); | |
| 214 registrar("koi8_r", "KOI8-R"); | |
| 215 registrar("x-cp1253", "windows-1253"); | |
| 216 registrar("x-cp1254", "windows-1254"); | |
| 217 registrar("x-cp1255", "windows-1255"); | |
| 218 registrar("x-cp1256", "windows-1256"); | |
| 219 registrar("x-cp1257", "windows-1257"); | |
| 220 registrar("x-cp1258", "windows-1258"); | |
| 221 #endif | |
| 222 } | |
| 223 | |
| 224 void TextCodecICU::registerCodecs(TextCodecRegistrar registrar) | |
| 225 { | |
| 226 // See comment above in registerEncodingNames. | |
| 227 registrar("ISO-8859-8-I", create, 0); | |
| 228 | |
| 229 int32_t numEncodings = ucnv_countAvailable(); | |
| 230 for (int32_t i = 0; i < numEncodings; ++i) { | |
| 231 const char* name = ucnv_getAvailableName(i); | |
| 232 UErrorCode error = U_ZERO_ERROR; | |
| 233 const char* standardName = ucnv_getStandardName(name, "MIME", &error); | |
| 234 if (!U_SUCCESS(error) || !standardName) { | |
| 235 error = U_ZERO_ERROR; | |
| 236 standardName = ucnv_getStandardName(name, "IANA", &error); | |
| 237 if (!U_SUCCESS(error) || !standardName) | |
| 238 continue; | |
| 239 } | |
| 240 registrar(standardName, create, 0); | |
| 241 } | |
| 242 } | |
| 243 | |
| 244 TextCodecICU::TextCodecICU(const TextEncoding& encoding) | |
| 245 : m_encoding(encoding) | |
| 246 , m_converterICU(0) | |
| 247 #if defined(USING_SYSTEM_ICU) | |
| 248 , m_needsGBKFallbacks(false) | |
| 249 #endif | |
| 250 { | |
| 251 } | |
| 252 | |
| 253 TextCodecICU::~TextCodecICU() | |
| 254 { | |
| 255 releaseICUConverter(); | |
| 256 } | |
| 257 | |
| 258 void TextCodecICU::releaseICUConverter() const | |
| 259 { | |
| 260 if (m_converterICU) { | |
| 261 UConverter*& cachedConverter = cachedConverterICU(); | |
| 262 if (cachedConverter) | |
| 263 ucnv_close(cachedConverter); | |
| 264 cachedConverter = m_converterICU; | |
| 265 m_converterICU = 0; | |
| 266 } | |
| 267 } | |
| 268 | |
| 269 void TextCodecICU::createICUConverter() const | |
| 270 { | |
| 271 ASSERT(!m_converterICU); | |
| 272 | |
| 273 #if defined(USING_SYSTEM_ICU) | |
| 274 const char* name = m_encoding.name(); | |
| 275 m_needsGBKFallbacks = name[0] == 'G' && name[1] == 'B' && name[2] == 'K' &&
!name[3]; | |
| 276 #endif | |
| 277 | |
| 278 UErrorCode err; | |
| 279 | |
| 280 UConverter*& cachedConverter = cachedConverterICU(); | |
| 281 if (cachedConverter) { | |
| 282 err = U_ZERO_ERROR; | |
| 283 const char* cachedName = ucnv_getName(cachedConverter, &err); | |
| 284 if (U_SUCCESS(err) && m_encoding == cachedName) { | |
| 285 m_converterICU = cachedConverter; | |
| 286 cachedConverter = 0; | |
| 287 return; | |
| 288 } | |
| 289 } | |
| 290 | |
| 291 err = U_ZERO_ERROR; | |
| 292 m_converterICU = ucnv_open(m_encoding.name(), &err); | |
| 293 #if !LOG_DISABLED | |
| 294 if (err == U_AMBIGUOUS_ALIAS_WARNING) | |
| 295 WTF_LOG_ERROR("ICU ambiguous alias warning for encoding: %s", m_encoding
.name()); | |
| 296 #endif | |
| 297 if (m_converterICU) | |
| 298 ucnv_setFallback(m_converterICU, TRUE); | |
| 299 } | |
| 300 | |
| 301 int TextCodecICU::decodeToBuffer(UChar* target, UChar* targetLimit, const char*&
source, const char* sourceLimit, int32_t* offsets, bool flush, UErrorCode& err) | |
| 302 { | |
| 303 UChar* targetStart = target; | |
| 304 err = U_ZERO_ERROR; | |
| 305 ucnv_toUnicode(m_converterICU, &target, targetLimit, &source, sourceLimit, o
ffsets, flush, &err); | |
| 306 return target - targetStart; | |
| 307 } | |
| 308 | |
| 309 class ErrorCallbackSetter { | |
| 310 public: | |
| 311 ErrorCallbackSetter(UConverter* converter, bool stopOnError) | |
| 312 : m_converter(converter) | |
| 313 , m_shouldStopOnEncodingErrors(stopOnError) | |
| 314 { | |
| 315 if (m_shouldStopOnEncodingErrors) { | |
| 316 UErrorCode err = U_ZERO_ERROR; | |
| 317 ucnv_setToUCallBack(m_converter, UCNV_TO_U_CALLBACK_SUBSTITUTE, | |
| 318 UCNV_SUB_STOP_ON_ILLEGAL, &m_savedAction, | |
| 319 &m_savedContext, &err); | |
| 320 ASSERT(err == U_ZERO_ERROR); | |
| 321 } | |
| 322 } | |
| 323 ~ErrorCallbackSetter() | |
| 324 { | |
| 325 if (m_shouldStopOnEncodingErrors) { | |
| 326 UErrorCode err = U_ZERO_ERROR; | |
| 327 const void* oldContext; | |
| 328 UConverterToUCallback oldAction; | |
| 329 ucnv_setToUCallBack(m_converter, m_savedAction, m_savedContext, &old
Action, &oldContext, &err); | |
| 330 ASSERT(oldAction == UCNV_TO_U_CALLBACK_SUBSTITUTE); | |
| 331 ASSERT(!strcmp(static_cast<const char*>(oldContext), UCNV_SUB_STOP_O
N_ILLEGAL)); | |
| 332 ASSERT(err == U_ZERO_ERROR); | |
| 333 } | |
| 334 } | |
| 335 | |
| 336 private: | |
| 337 UConverter* m_converter; | |
| 338 bool m_shouldStopOnEncodingErrors; | |
| 339 const void* m_savedContext; | |
| 340 UConverterToUCallback m_savedAction; | |
| 341 }; | |
| 342 | |
| 343 String TextCodecICU::decode(const char* bytes, size_t length, FlushBehavior flus
h, bool stopOnError, bool& sawError) | |
| 344 { | |
| 345 // Get a converter for the passed-in encoding. | |
| 346 if (!m_converterICU) { | |
| 347 createICUConverter(); | |
| 348 ASSERT(m_converterICU); | |
| 349 if (!m_converterICU) { | |
| 350 WTF_LOG_ERROR("error creating ICU encoder even though encoding was i
n table"); | |
| 351 return String(); | |
| 352 } | |
| 353 } | |
| 354 | |
| 355 ErrorCallbackSetter callbackSetter(m_converterICU, stopOnError); | |
| 356 | |
| 357 StringBuilder result; | |
| 358 | |
| 359 UChar buffer[ConversionBufferSize]; | |
| 360 UChar* bufferLimit = buffer + ConversionBufferSize; | |
| 361 const char* source = reinterpret_cast<const char*>(bytes); | |
| 362 const char* sourceLimit = source + length; | |
| 363 int32_t* offsets = nullptr; | |
| 364 UErrorCode err = U_ZERO_ERROR; | |
| 365 | |
| 366 do { | |
| 367 int ucharsDecoded = decodeToBuffer(buffer, bufferLimit, source, sourceLi
mit, offsets, flush != DoNotFlush, err); | |
| 368 result.append(buffer, ucharsDecoded); | |
| 369 } while (err == U_BUFFER_OVERFLOW_ERROR); | |
| 370 | |
| 371 if (U_FAILURE(err)) { | |
| 372 // flush the converter so it can be reused, and not be bothered by this
error. | |
| 373 do { | |
| 374 decodeToBuffer(buffer, bufferLimit, source, sourceLimit, offsets, tr
ue, err); | |
| 375 } while (source < sourceLimit); | |
| 376 sawError = true; | |
| 377 } | |
| 378 | |
| 379 #if !defined(USING_SYSTEM_ICU) | |
| 380 // Chrome's copy of ICU does not have the issue described below. | |
| 381 return result.toString(); | |
| 382 #else | |
| 383 String resultString = result.toString(); | |
| 384 | |
| 385 // <http://bugs.webkit.org/show_bug.cgi?id=17014> | |
| 386 // Simplified Chinese pages use the code A3A0 to mean "full-width space", bu
t ICU decodes it as U+E5E5. | |
| 387 if (!strcmp(m_encoding.name(), "GBK")) { | |
| 388 if (!strcasecmp(m_encoding.name(), "gb18030")) | |
| 389 resultString.replace(0xE5E5, ideographicSpaceCharacter); | |
| 390 // Make GBK compliant to the encoding spec and align with GB18030 | |
| 391 resultString.replace(0x01F9, 0xE7C8); | |
| 392 // FIXME: Once https://www.w3.org/Bugs/Public/show_bug.cgi?id=28740#c3 | |
| 393 // is resolved, add U+1E3F => 0xE7C7. | |
| 394 } | |
| 395 | |
| 396 return resultString; | |
| 397 #endif | 384 #endif |
| 398 } | 385 } |
| 399 | 386 |
| 400 #if defined(USING_SYSTEM_ICU) | 387 #if defined(USING_SYSTEM_ICU) |
| 401 // U+01F9 and U+1E3F have to be mapped to xA8xBF and xA8xBC per the encoding | 388 // U+01F9 and U+1E3F have to be mapped to xA8xBF and xA8xBC per the encoding |
| 402 // spec, but ICU converter does not have them. | 389 // spec, but ICU converter does not have them. |
| 403 static UChar fallbackForGBK(UChar32 character) | 390 static UChar fallbackForGBK(UChar32 character) { |
| 404 { | 391 switch (character) { |
| 405 switch (character) { | |
| 406 case 0x01F9: | 392 case 0x01F9: |
| 407 return 0xE7C8; // mapped to xA8xBF by ICU. | 393 return 0xE7C8; // mapped to xA8xBF by ICU. |
| 408 case 0x1E3F: | 394 case 0x1E3F: |
| 409 return 0xE7C7; // mapped to xA8xBC by ICU. | 395 return 0xE7C7; // mapped to xA8xBC by ICU. |
| 410 } | 396 } |
| 411 return 0; | 397 return 0; |
| 412 } | 398 } |
| 413 #endif | 399 #endif |
| 414 | 400 |
| 415 // Invalid character handler when writing escaped entities for unrepresentable | 401 // Invalid character handler when writing escaped entities for unrepresentable |
| 416 // characters. See the declaration of TextCodec::encode for more. | 402 // characters. See the declaration of TextCodec::encode for more. |
| 417 static void urlEscapedEntityCallback(const void* context, UConverterFromUnicodeA
rgs* fromUArgs, const UChar* codeUnits, int32_t length, | 403 static void urlEscapedEntityCallback(const void* context, UConverterFromUnicodeA
rgs* fromUArgs, const UChar* codeUnits, int32_t length, UChar32 codePoint, UConv
erterCallbackReason reason, UErrorCode* err) { |
| 418 UChar32 codePoint, UConverterCallbackReason reason, UErrorCode* err) | 404 if (reason == UCNV_UNASSIGNED) { |
| 419 { | 405 *err = U_ZERO_ERROR; |
| 420 if (reason == UCNV_UNASSIGNED) { | 406 |
| 421 *err = U_ZERO_ERROR; | 407 UnencodableReplacementArray entity; |
| 422 | 408 int entityLen = TextCodec::getUnencodableReplacement(codePoint, URLEncodedEn
titiesForUnencodables, entity); |
| 423 UnencodableReplacementArray entity; | 409 ucnv_cbFromUWriteBytes(fromUArgs, entity, entityLen, 0, err); |
| 424 int entityLen = TextCodec::getUnencodableReplacement(codePoint, URLEncod
edEntitiesForUnencodables, entity); | 410 } else { |
| 425 ucnv_cbFromUWriteBytes(fromUArgs, entity, entityLen, 0, err); | 411 UCNV_FROM_U_CALLBACK_ESCAPE(context, fromUArgs, codeUnits, length, codePoint
, reason, err); |
| 426 } else { | 412 } |
| 427 UCNV_FROM_U_CALLBACK_ESCAPE(context, fromUArgs, codeUnits, length, codeP
oint, reason, err); | |
| 428 } | |
| 429 } | 413 } |
| 430 | 414 |
| 431 #if defined(USING_SYSTEM_ICU) | 415 #if defined(USING_SYSTEM_ICU) |
| 432 // Substitutes special GBK characters, escaping all other unassigned entities. | 416 // Substitutes special GBK characters, escaping all other unassigned entities. |
| 433 static void gbkCallbackEscape(const void* context, UConverterFromUnicodeArgs* fr
omUArgs, const UChar* codeUnits, int32_t length, | 417 static void gbkCallbackEscape(const void* context, UConverterFromUnicodeArgs* fr
omUArgs, const UChar* codeUnits, int32_t length, UChar32 codePoint, UConverterCa
llbackReason reason, UErrorCode* err) { |
| 434 UChar32 codePoint, UConverterCallbackReason reason, UErrorCode* err) | 418 UChar outChar; |
| 435 { | 419 if (reason == UCNV_UNASSIGNED && (outChar = fallbackForGBK(codePoint))) { |
| 436 UChar outChar; | 420 const UChar* source = &outChar; |
| 437 if (reason == UCNV_UNASSIGNED && (outChar = fallbackForGBK(codePoint))) { | 421 *err = U_ZERO_ERROR; |
| 438 const UChar* source = &outChar; | 422 ucnv_cbFromUWriteUChars(fromUArgs, &source, source + 1, 0, err); |
| 439 *err = U_ZERO_ERROR; | 423 return; |
| 440 ucnv_cbFromUWriteUChars(fromUArgs, &source, source + 1, 0, err); | 424 } |
| 441 return; | 425 UCNV_FROM_U_CALLBACK_ESCAPE(context, fromUArgs, codeUnits, length, codePoint,
reason, err); |
| 442 } | |
| 443 UCNV_FROM_U_CALLBACK_ESCAPE(context, fromUArgs, codeUnits, length, codePoint
, reason, err); | |
| 444 } | 426 } |
| 445 | 427 |
| 446 // Combines both gbkUrlEscapedEntityCallback and GBK character substitution. | 428 // Combines both gbkUrlEscapedEntityCallback and GBK character substitution. |
| 447 static void gbkUrlEscapedEntityCallack(const void* context, UConverterFromUnicod
eArgs* fromUArgs, const UChar* codeUnits, int32_t length, | 429 static void gbkUrlEscapedEntityCallack(const void* context, UConverterFromUnicod
eArgs* fromUArgs, const UChar* codeUnits, int32_t length, UChar32 codePoint, UCo
nverterCallbackReason reason, UErrorCode* err) { |
| 448 UChar32 codePoint, UConverterCallbackReason reason, UErrorCode* err) | 430 if (reason == UCNV_UNASSIGNED) { |
| 449 { | 431 if (UChar outChar = fallbackForGBK(codePoint)) { |
| 450 if (reason == UCNV_UNASSIGNED) { | 432 const UChar* source = &outChar; |
| 451 if (UChar outChar = fallbackForGBK(codePoint)) { | 433 *err = U_ZERO_ERROR; |
| 452 const UChar* source = &outChar; | 434 ucnv_cbFromUWriteUChars(fromUArgs, &source, source + 1, 0, err); |
| 453 *err = U_ZERO_ERROR; | 435 return; |
| 454 ucnv_cbFromUWriteUChars(fromUArgs, &source, source + 1, 0, err); | 436 } |
| 455 return; | 437 urlEscapedEntityCallback(context, fromUArgs, codeUnits, length, codePoint, r
eason, err); |
| 456 } | 438 return; |
| 457 urlEscapedEntityCallback(context, fromUArgs, codeUnits, length, codePoin
t, reason, err); | 439 } |
| 458 return; | 440 UCNV_FROM_U_CALLBACK_ESCAPE(context, fromUArgs, codeUnits, length, codePoint,
reason, err); |
| 459 } | 441 } |
| 460 UCNV_FROM_U_CALLBACK_ESCAPE(context, fromUArgs, codeUnits, length, codePoint
, reason, err); | 442 |
| 461 } | 443 static void gbkCallbackSubstitute(const void* context, UConverterFromUnicodeArgs
* fromUArgs, const UChar* codeUnits, int32_t length, UChar32 codePoint, UConvert
erCallbackReason reason, UErrorCode* err) { |
| 462 | 444 UChar outChar; |
| 463 static void gbkCallbackSubstitute(const void* context, UConverterFromUnicodeArgs
* fromUArgs, const UChar* codeUnits, int32_t length, | 445 if (reason == UCNV_UNASSIGNED && (outChar = fallbackForGBK(codePoint))) { |
| 464 UChar32 codePoint, UConverterCallbackReason reason, UErrorCode* err) | 446 const UChar* source = &outChar; |
| 465 { | 447 *err = U_ZERO_ERROR; |
| 466 UChar outChar; | 448 ucnv_cbFromUWriteUChars(fromUArgs, &source, source + 1, 0, err); |
| 467 if (reason == UCNV_UNASSIGNED && (outChar = fallbackForGBK(codePoint))) { | 449 return; |
| 468 const UChar* source = &outChar; | 450 } |
| 469 *err = U_ZERO_ERROR; | 451 UCNV_FROM_U_CALLBACK_SUBSTITUTE(context, fromUArgs, codeUnits, length, codePoi
nt, reason, err); |
| 470 ucnv_cbFromUWriteUChars(fromUArgs, &source, source + 1, 0, err); | 452 } |
| 471 return; | 453 #endif // USING_SYSTEM_ICU |
| 472 } | |
| 473 UCNV_FROM_U_CALLBACK_SUBSTITUTE(context, fromUArgs, codeUnits, length, codeP
oint, reason, err); | |
| 474 } | |
| 475 #endif // USING_SYSTEM_ICU | |
| 476 | 454 |
| 477 class TextCodecInput { | 455 class TextCodecInput { |
| 478 public: | 456 public: |
| 479 TextCodecInput(const TextEncoding& encoding, const UChar* characters, size_t
length) | 457 TextCodecInput(const TextEncoding& encoding, const UChar* characters, size_t l
ength) |
| 480 : m_begin(characters) | 458 : m_begin(characters), m_end(characters + length) {} |
| 481 , m_end(characters + length) | 459 |
| 482 { } | 460 TextCodecInput(const TextEncoding& encoding, const LChar* characters, size_t l
ength) { |
| 483 | 461 m_buffer.reserveInitialCapacity(length); |
| 484 TextCodecInput(const TextEncoding& encoding, const LChar* characters, size_t
length) | 462 for (size_t i = 0; i < length; ++i) |
| 485 { | 463 m_buffer.append(characters[i]); |
| 486 m_buffer.reserveInitialCapacity(length); | 464 m_begin = m_buffer.data(); |
| 487 for (size_t i = 0; i < length; ++i) | 465 m_end = m_begin + m_buffer.size(); |
| 488 m_buffer.append(characters[i]); | 466 } |
| 489 m_begin = m_buffer.data(); | 467 |
| 490 m_end = m_begin + m_buffer.size(); | 468 const UChar* begin() const { return m_begin; } |
| 491 } | 469 const UChar* end() const { return m_end; } |
| 492 | 470 |
| 493 const UChar* begin() const { return m_begin; } | 471 private: |
| 494 const UChar* end() const { return m_end; } | 472 const UChar* m_begin; |
| 495 | 473 const UChar* m_end; |
| 496 private: | 474 Vector<UChar> m_buffer; |
| 497 const UChar* m_begin; | |
| 498 const UChar* m_end; | |
| 499 Vector<UChar> m_buffer; | |
| 500 }; | 475 }; |
| 501 | 476 |
| 502 CString TextCodecICU::encodeInternal(const TextCodecInput& input, UnencodableHan
dling handling) | 477 CString TextCodecICU::encodeInternal(const TextCodecInput& input, UnencodableHan
dling handling) { |
| 503 { | 478 const UChar* source = input.begin(); |
| 504 const UChar* source = input.begin(); | 479 const UChar* end = input.end(); |
| 505 const UChar* end = input.end(); | 480 |
| 506 | 481 UErrorCode err = U_ZERO_ERROR; |
| 507 UErrorCode err = U_ZERO_ERROR; | 482 |
| 508 | 483 switch (handling) { |
| 509 switch (handling) { | |
| 510 case QuestionMarksForUnencodables: | 484 case QuestionMarksForUnencodables: |
| 511 ucnv_setSubstChars(m_converterICU, "?", 1, &err); | 485 ucnv_setSubstChars(m_converterICU, "?", 1, &err); |
| 512 #if !defined(USING_SYSTEM_ICU) | 486 #if !defined(USING_SYSTEM_ICU) |
| 513 ucnv_setFromUCallBack(m_converterICU, UCNV_FROM_U_CALLBACK_SUBSTITUTE, 0
, 0, 0, &err); | 487 ucnv_setFromUCallBack(m_converterICU, UCNV_FROM_U_CALLBACK_SUBSTITUTE, 0,
0, 0, &err); |
| 514 #else | 488 #else |
| 515 ucnv_setFromUCallBack(m_converterICU, m_needsGBKFallbacks ? gbkCallbackS
ubstitute : UCNV_FROM_U_CALLBACK_SUBSTITUTE, 0, 0, 0, &err); | 489 ucnv_setFromUCallBack(m_converterICU, m_needsGBKFallbacks ? gbkCallbackSub
stitute : UCNV_FROM_U_CALLBACK_SUBSTITUTE, 0, 0, 0, &err); |
| 516 #endif | 490 #endif |
| 517 break; | 491 break; |
| 518 case EntitiesForUnencodables: | 492 case EntitiesForUnencodables: |
| 519 #if !defined(USING_SYSTEM_ICU) | 493 #if !defined(USING_SYSTEM_ICU) |
| 520 ucnv_setFromUCallBack(m_converterICU, UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_
ESCAPE_XML_DEC, 0, 0, &err); | 494 ucnv_setFromUCallBack(m_converterICU, UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_ES
CAPE_XML_DEC, 0, 0, &err); |
| 521 #else | 495 #else |
| 522 ucnv_setFromUCallBack(m_converterICU, m_needsGBKFallbacks ? gbkCallbackE
scape : UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_ESCAPE_XML_DEC, 0, 0, &err); | 496 ucnv_setFromUCallBack(m_converterICU, m_needsGBKFallbacks ? gbkCallbackEsc
ape : UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_ESCAPE_XML_DEC, 0, 0, &err); |
| 523 #endif | 497 #endif |
| 524 break; | 498 break; |
| 525 case URLEncodedEntitiesForUnencodables: | 499 case URLEncodedEntitiesForUnencodables: |
| 526 #if !defined(USING_SYSTEM_ICU) | 500 #if !defined(USING_SYSTEM_ICU) |
| 527 ucnv_setFromUCallBack(m_converterICU, urlEscapedEntityCallback, 0, 0, 0,
&err); | 501 ucnv_setFromUCallBack(m_converterICU, urlEscapedEntityCallback, 0, 0, 0, &
err); |
| 528 #else | 502 #else |
| 529 ucnv_setFromUCallBack(m_converterICU, m_needsGBKFallbacks ? gbkUrlEscape
dEntityCallack : urlEscapedEntityCallback, 0, 0, 0, &err); | 503 ucnv_setFromUCallBack(m_converterICU, m_needsGBKFallbacks ? gbkUrlEscapedE
ntityCallack : urlEscapedEntityCallback, 0, 0, 0, &err); |
| 530 #endif | 504 #endif |
| 531 break; | 505 break; |
| 532 } | 506 } |
| 533 | 507 |
| 534 ASSERT(U_SUCCESS(err)); | 508 ASSERT(U_SUCCESS(err)); |
| 535 if (U_FAILURE(err)) | 509 if (U_FAILURE(err)) |
| 536 return CString(); | 510 return CString(); |
| 537 | 511 |
| 538 Vector<char> result; | 512 Vector<char> result; |
| 539 size_t size = 0; | 513 size_t size = 0; |
| 540 do { | 514 do { |
| 541 char buffer[ConversionBufferSize]; | 515 char buffer[ConversionBufferSize]; |
| 542 char* target = buffer; | 516 char* target = buffer; |
| 543 char* targetLimit = target + ConversionBufferSize; | 517 char* targetLimit = target + ConversionBufferSize; |
| 544 err = U_ZERO_ERROR; | 518 err = U_ZERO_ERROR; |
| 545 ucnv_fromUnicode(m_converterICU, &target, targetLimit, &source, end, 0,
true, &err); | 519 ucnv_fromUnicode(m_converterICU, &target, targetLimit, &source, end, 0, true
, &err); |
| 546 size_t count = target - buffer; | 520 size_t count = target - buffer; |
| 547 result.grow(size + count); | 521 result.grow(size + count); |
| 548 memcpy(result.data() + size, buffer, count); | 522 memcpy(result.data() + size, buffer, count); |
| 549 size += count; | 523 size += count; |
| 550 } while (err == U_BUFFER_OVERFLOW_ERROR); | 524 } while (err == U_BUFFER_OVERFLOW_ERROR); |
| 551 | 525 |
| 552 return CString(result.data(), size); | 526 return CString(result.data(), size); |
| 553 } | 527 } |
| 554 | 528 |
| 555 template<typename CharType> | 529 template <typename CharType> |
| 556 CString TextCodecICU::encodeCommon(const CharType* characters, size_t length, Un
encodableHandling handling) | 530 CString TextCodecICU::encodeCommon(const CharType* characters, size_t length, Un
encodableHandling handling) { |
| 557 { | 531 if (!length) |
| 558 if (!length) | 532 return ""; |
| 559 return ""; | 533 |
| 560 | 534 if (!m_converterICU) |
| 561 if (!m_converterICU) | 535 createICUConverter(); |
| 562 createICUConverter(); | 536 if (!m_converterICU) |
| 563 if (!m_converterICU) | 537 return CString(); |
| 564 return CString(); | 538 |
| 565 | 539 TextCodecInput input(m_encoding, characters, length); |
| 566 TextCodecInput input(m_encoding, characters, length); | 540 return encodeInternal(input, handling); |
| 567 return encodeInternal(input, handling); | 541 } |
| 568 } | 542 |
| 569 | 543 CString TextCodecICU::encode(const UChar* characters, size_t length, Unencodable
Handling handling) { |
| 570 CString TextCodecICU::encode(const UChar* characters, size_t length, Unencodable
Handling handling) | 544 return encodeCommon(characters, length, handling); |
| 571 { | 545 } |
| 572 return encodeCommon(characters, length, handling); | 546 |
| 573 } | 547 CString TextCodecICU::encode(const LChar* characters, size_t length, Unencodable
Handling handling) { |
| 574 | 548 return encodeCommon(characters, length, handling); |
| 575 CString TextCodecICU::encode(const LChar* characters, size_t length, Unencodable
Handling handling) | 549 } |
| 576 { | 550 |
| 577 return encodeCommon(characters, length, handling); | 551 } // namespace WTF |
| 578 } | |
| 579 | |
| 580 } // namespace WTF | |
| OLD | NEW |