OLD | NEW |
1 /* | 1 /* |
2 * Copyright (C) 2007 Apple Inc. All rights reserved. | 2 * Copyright (C) 2007 Apple Inc. All rights reserved. |
3 * Copyright (C) 2010 Patrick Gansterer <paroga@paroga.com> | 3 * Copyright (C) 2010 Patrick Gansterer <paroga@paroga.com> |
4 * | 4 * |
5 * Redistribution and use in source and binary forms, with or without | 5 * Redistribution and use in source and binary forms, with or without |
6 * modification, are permitted provided that the following conditions | 6 * modification, are permitted provided that the following conditions |
7 * are met: | 7 * are met: |
8 * 1. Redistributions of source code must retain the above copyright | 8 * 1. Redistributions of source code must retain the above copyright |
9 * notice, this list of conditions and the following disclaimer. | 9 * notice, this list of conditions and the following disclaimer. |
10 * 2. Redistributions in binary form must reproduce the above copyright | 10 * 2. Redistributions in binary form must reproduce the above copyright |
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
48 } | 48 } |
49 | 49 |
50 inline int inlineUTF8SequenceLength(char b0) | 50 inline int inlineUTF8SequenceLength(char b0) |
51 { | 51 { |
52 return isASCII(b0) ? 1 : inlineUTF8SequenceLengthNonASCII(b0); | 52 return isASCII(b0) ? 1 : inlineUTF8SequenceLengthNonASCII(b0); |
53 } | 53 } |
54 | 54 |
55 // Once the bits are split out into bytes of UTF-8, this is a mask OR-ed | 55 // Once the bits are split out into bytes of UTF-8, this is a mask OR-ed |
56 // into the first byte, depending on how many bytes follow. There are | 56 // into the first byte, depending on how many bytes follow. There are |
57 // as many entries in this table as there are UTF-8 sequence types. | 57 // as many entries in this table as there are UTF-8 sequence types. |
58 // (I.e., one byte sequence, two byte... etc.). Remember that sequencs | 58 // (I.e., one byte sequence, two byte... etc.). Remember that sequences |
59 // for *legal* UTF-8 will be 4 or fewer bytes total. | 59 // for *legal* UTF-8 will be 4 or fewer bytes total. |
60 static const unsigned char firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0x
F8, 0xFC }; | 60 static const unsigned char firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0x
F8, 0xFC }; |
61 | 61 |
62 ConversionResult convertLatin1ToUTF8( | 62 ConversionResult convertLatin1ToUTF8( |
63 const LChar** sourceStart, const LChar* sou
rceEnd, | 63 const LChar** sourceStart, const LChar* sourceEnd, |
64 char** targetStart, char* targetEnd) | 64 char** targetStart, char* targetEnd) |
65 { | 65 { |
66 ConversionResult result = conversionOK; | 66 ConversionResult result = conversionOK; |
67 const LChar* source = *sourceStart; | 67 const LChar* source = *sourceStart; |
68 char* target = *targetStart; | 68 char* target = *targetStart; |
69 while (source < sourceEnd) { | 69 while (source < sourceEnd) { |
70 UChar32 ch; | 70 UChar32 ch; |
71 unsigned short bytesToWrite = 0; | 71 unsigned short bytesToWrite = 0; |
72 const UChar32 byteMask = 0xBF; | 72 const UChar32 byteMask = 0xBF; |
73 const UChar32 byteMark = 0x80; | 73 const UChar32 byteMark = 0x80; |
74 const LChar* oldSource = source; // In case we have to back up because o
f target overflow. | 74 const LChar* oldSource = source; // In case we have to back up because o
f target overflow. |
(...skipping 82 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
157 } | 157 } |
158 | 158 |
159 target += bytesToWrite; | 159 target += bytesToWrite; |
160 if (target > targetEnd) { | 160 if (target > targetEnd) { |
161 source = oldSource; // Back up source pointer! | 161 source = oldSource; // Back up source pointer! |
162 target -= bytesToWrite; | 162 target -= bytesToWrite; |
163 result = targetExhausted; | 163 result = targetExhausted; |
164 break; | 164 break; |
165 } | 165 } |
166 switch (bytesToWrite) { // note: everything falls through. | 166 switch (bytesToWrite) { // note: everything falls through. |
167 case 4: *--target = (char)((ch | byteMark) & byteMask); ch >>= 6; | 167 case 4: |
168 case 3: *--target = (char)((ch | byteMark) & byteMask); ch >>= 6; | 168 *--target = (char)((ch | byteMark) & byteMask); |
169 case 2: *--target = (char)((ch | byteMark) & byteMask); ch >>= 6; | 169 ch >>= 6; |
170 case 1: *--target = (char)(ch | firstByteMark[bytesToWrite]); | 170 case 3: |
| 171 *--target = (char)((ch | byteMark) & byteMask); |
| 172 ch >>= 6; |
| 173 case 2: |
| 174 *--target = (char)((ch | byteMark) & byteMask); |
| 175 ch >>= 6; |
| 176 case 1: |
| 177 *--target = (char)(ch | firstByteMark[bytesToWrite]); |
171 } | 178 } |
172 target += bytesToWrite; | 179 target += bytesToWrite; |
173 } | 180 } |
174 *sourceStart = source; | 181 *sourceStart = source; |
175 *targetStart = target; | 182 *targetStart = target; |
176 return result; | 183 return result; |
177 } | 184 } |
178 | 185 |
179 // This must be called with the length pre-determined by the first byte. | 186 // This must be called with the length pre-determined by the first byte. |
180 // If presented with a length > 4, this returns false. The Unicode | 187 // If presented with a length > 4, this returns false. The Unicode |
181 // definition of UTF-8 goes up to 4-byte sequences. | 188 // definition of UTF-8 goes up to 4-byte sequences. |
182 static bool isLegalUTF8(const unsigned char* source, int length) | 189 static bool isLegalUTF8(const unsigned char* source, int length) |
183 { | 190 { |
184 unsigned char a; | 191 unsigned char a; |
185 const unsigned char* srcptr = source + length; | 192 const unsigned char* srcptr = source + length; |
186 switch (length) { | 193 switch (length) { |
187 default: return false; | 194 default: |
188 // Everything else falls through when "true"... | 195 return false; |
189 case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; | 196 // Everything else falls through when "true"... |
190 case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; | 197 case 4: |
191 case 2: if ((a = (*--srcptr)) > 0xBF) return false; | 198 if ((a = (*--srcptr)) < 0x80 || a > 0xBF) |
| 199 return false; |
| 200 case 3: |
| 201 if ((a = (*--srcptr)) < 0x80 || a > 0xBF) |
| 202 return false; |
| 203 case 2: |
| 204 if ((a = (*--srcptr)) > 0xBF) |
| 205 return false; |
192 | 206 |
| 207 // no fall-through in this inner switch |
193 switch (*source) { | 208 switch (*source) { |
194 // no fall-through in this inner switch | 209 case 0xE0: |
195 case 0xE0: if (a < 0xA0) return false; break; | 210 if (a < 0xA0) |
196 case 0xED: if (a > 0x9F) return false; break; | 211 return false; |
197 case 0xF0: if (a < 0x90) return false; break; | 212 break; |
198 case 0xF4: if (a > 0x8F) return false; break; | 213 case 0xED: |
199 default: if (a < 0x80) return false; | 214 if (a > 0x9F) |
| 215 return false; |
| 216 break; |
| 217 case 0xF0: |
| 218 if (a < 0x90) |
| 219 return false; |
| 220 break; |
| 221 case 0xF4: |
| 222 if (a > 0x8F) |
| 223 return false; |
| 224 break; |
| 225 default: |
| 226 if (a < 0x80) |
| 227 return false; |
200 } | 228 } |
201 | 229 |
202 case 1: if (*source >= 0x80 && *source < 0xC2) return false; | 230 case 1: |
| 231 if (*source >= 0x80 && *source < 0xC2) |
| 232 return false; |
203 } | 233 } |
204 if (*source > 0xF4) | 234 if (*source > 0xF4) |
205 return false; | 235 return false; |
206 return true; | 236 return true; |
207 } | 237 } |
208 | 238 |
209 // Magic values subtracted from a buffer value during UTF8 conversion. | 239 // Magic values subtracted from a buffer value during UTF8 conversion. |
210 // This table contains as many values as there might be trailing bytes | 240 // This table contains as many values as there might be trailing bytes |
211 // in a UTF-8 sequence. | 241 // in a UTF-8 sequence. |
212 static const UChar32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E20
80UL, 0x03C82080UL, static_cast<UChar32>(0xFA082080UL), static_cast<UChar32>(0x8
2082080UL) }; | 242 static const UChar32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E20
80UL, 0x03C82080UL, static_cast<UChar32>(0xFA082080UL), static_cast<UChar32>(0x8
2082080UL) }; |
213 | 243 |
214 static inline UChar32 readUTF8Sequence(const char*& sequence, unsigned length) | 244 static inline UChar32 readUTF8Sequence(const char*& sequence, unsigned length) |
215 { | 245 { |
216 UChar32 character = 0; | 246 UChar32 character = 0; |
217 | 247 |
218 // The cases all fall through. | 248 // The cases all fall through. |
219 switch (length) { | 249 switch (length) { |
220 case 6: character += static_cast<unsigned char>(*sequence++); character
<<= 6; | 250 case 6: |
221 case 5: character += static_cast<unsigned char>(*sequence++); character
<<= 6; | 251 character += static_cast<unsigned char>(*sequence++); |
222 case 4: character += static_cast<unsigned char>(*sequence++); character
<<= 6; | 252 character <<= 6; |
223 case 3: character += static_cast<unsigned char>(*sequence++); character
<<= 6; | 253 case 5: |
224 case 2: character += static_cast<unsigned char>(*sequence++); character
<<= 6; | 254 character += static_cast<unsigned char>(*sequence++); |
225 case 1: character += static_cast<unsigned char>(*sequence++); | 255 character <<= 6; |
| 256 case 4: |
| 257 character += static_cast<unsigned char>(*sequence++); |
| 258 character <<= 6; |
| 259 case 3: |
| 260 character += static_cast<unsigned char>(*sequence++); |
| 261 character <<= 6; |
| 262 case 2: |
| 263 character += static_cast<unsigned char>(*sequence++); |
| 264 character <<= 6; |
| 265 case 1: |
| 266 character += static_cast<unsigned char>(*sequence++); |
226 } | 267 } |
227 | 268 |
228 return character - offsetsFromUTF8[length - 1]; | 269 return character - offsetsFromUTF8[length - 1]; |
229 } | 270 } |
230 | 271 |
231 ConversionResult convertUTF8ToUTF16( | 272 ConversionResult convertUTF8ToUTF16( |
232 const char** sourceStart, const char* sourceEnd, | 273 const char** sourceStart, const char* sourceEnd, |
233 UChar** targetStart, UChar* targetEnd, bool* sourceAllASCII, bool strict) | 274 UChar** targetStart, UChar* targetEnd, bool* sourceAllASCII, bool strict) |
234 { | 275 { |
235 ConversionResult result = conversionOK; | 276 ConversionResult result = conversionOK; |
(...skipping 20 matching lines...) Expand all Loading... |
256 break; | 297 break; |
257 } | 298 } |
258 | 299 |
259 if (U_IS_BMP(character)) { | 300 if (U_IS_BMP(character)) { |
260 // UTF-16 surrogate values are illegal in UTF-32 | 301 // UTF-16 surrogate values are illegal in UTF-32 |
261 if (U_IS_SURROGATE(character)) { | 302 if (U_IS_SURROGATE(character)) { |
262 if (strict) { | 303 if (strict) { |
263 source -= utf8SequenceLength; // return to the illegal value
itself | 304 source -= utf8SequenceLength; // return to the illegal value
itself |
264 result = sourceIllegal; | 305 result = sourceIllegal; |
265 break; | 306 break; |
266 } else { | |
267 *target++ = replacementCharacter; | |
268 orAllData |= replacementCharacter; | |
269 } | 307 } |
| 308 *target++ = replacementCharacter; |
| 309 orAllData |= replacementCharacter; |
270 } else { | 310 } else { |
271 *target++ = static_cast<UChar>(character); // normal case | 311 *target++ = static_cast<UChar>(character); // normal case |
272 orAllData |= character; | 312 orAllData |= character; |
273 } | 313 } |
274 } else if (U_IS_SUPPLEMENTARY(character)) { | 314 } else if (U_IS_SUPPLEMENTARY(character)) { |
275 // target is a character in range 0xFFFF - 0x10FFFF | 315 // target is a character in range 0xFFFF - 0x10FFFF |
276 if (target + 1 >= targetEnd) { | 316 if (target + 1 >= targetEnd) { |
277 source -= utf8SequenceLength; // Back up source pointer! | 317 source -= utf8SequenceLength; // Back up source pointer! |
278 result = targetExhausted; | 318 result = targetExhausted; |
279 break; | 319 break; |
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
319 } | 359 } |
320 | 360 |
321 int utf8SequenceLength = inlineUTF8SequenceLengthNonASCII(*data); | 361 int utf8SequenceLength = inlineUTF8SequenceLengthNonASCII(*data); |
322 dataLength += utf8SequenceLength; | 362 dataLength += utf8SequenceLength; |
323 | 363 |
324 if (!dataEnd) { | 364 if (!dataEnd) { |
325 for (int i = 1; i < utf8SequenceLength; ++i) { | 365 for (int i = 1; i < utf8SequenceLength; ++i) { |
326 if (!data[i]) | 366 if (!data[i]) |
327 return 0; | 367 return 0; |
328 } | 368 } |
329 } else if (dataEnd - data < utf8SequenceLength) | 369 } else if (dataEnd - data < utf8SequenceLength) { |
330 return 0; | 370 return 0; |
| 371 } |
331 | 372 |
332 if (!isLegalUTF8(reinterpret_cast<const unsigned char*>(data), utf8Seque
nceLength)) | 373 if (!isLegalUTF8(reinterpret_cast<const unsigned char*>(data), utf8Seque
nceLength)) |
333 return 0; | 374 return 0; |
334 | 375 |
335 UChar32 character = readUTF8Sequence(data, utf8SequenceLength); | 376 UChar32 character = readUTF8Sequence(data, utf8SequenceLength); |
336 ASSERT(!isASCII(character)); | 377 ASSERT(!isASCII(character)); |
337 | 378 |
338 if (U_IS_BMP(character)) { | 379 if (U_IS_BMP(character)) { |
339 // UTF-16 surrogate values are illegal in UTF-32 | 380 // UTF-16 surrogate values are illegal in UTF-32 |
340 if (U_IS_SURROGATE(character)) | 381 if (U_IS_SURROGATE(character)) |
341 return 0; | 382 return 0; |
342 stringHasher.addCharacter(static_cast<UChar>(character)); // normal
case | 383 stringHasher.addCharacter(static_cast<UChar>(character)); // normal
case |
343 utf16Length++; | 384 utf16Length++; |
344 } else if (U_IS_SUPPLEMENTARY(character)) { | 385 } else if (U_IS_SUPPLEMENTARY(character)) { |
345 stringHasher.addCharacters(static_cast<UChar>(U16_LEAD(character)), | 386 stringHasher.addCharacters(static_cast<UChar>(U16_LEAD(character)),
static_cast<UChar>(U16_TRAIL(character))); |
346 static_cast<UChar>(U16_TRAIL(character)))
; | |
347 utf16Length += 2; | 387 utf16Length += 2; |
348 } else | 388 } else { |
349 return 0; | 389 return 0; |
| 390 } |
350 } | 391 } |
351 | 392 |
352 return stringHasher.hashWithTop8BitsMasked(); | 393 return stringHasher.hashWithTop8BitsMasked(); |
353 } | 394 } |
354 | 395 |
355 template<typename CharType> | 396 template<typename CharType> |
356 ALWAYS_INLINE bool equalWithUTF8Internal(const CharType* a, const CharType* aEnd
, const char* b, const char* bEnd) | 397 ALWAYS_INLINE bool equalWithUTF8Internal(const CharType* a, const CharType* aEnd
, const char* b, const char* bEnd) |
357 { | 398 { |
358 while (b < bEnd) { | 399 while (b < bEnd) { |
359 if (isASCII(*b)) { | 400 if (isASCII(*b)) { |
(...skipping 17 matching lines...) Expand all Loading... |
377 // UTF-16 surrogate values are illegal in UTF-32 | 418 // UTF-16 surrogate values are illegal in UTF-32 |
378 if (U_IS_SURROGATE(character)) | 419 if (U_IS_SURROGATE(character)) |
379 return false; | 420 return false; |
380 if (*a++ != character) | 421 if (*a++ != character) |
381 return false; | 422 return false; |
382 } else if (U_IS_SUPPLEMENTARY(character)) { | 423 } else if (U_IS_SUPPLEMENTARY(character)) { |
383 if (*a++ != U16_LEAD(character)) | 424 if (*a++ != U16_LEAD(character)) |
384 return false; | 425 return false; |
385 if (*a++ != U16_TRAIL(character)) | 426 if (*a++ != U16_TRAIL(character)) |
386 return false; | 427 return false; |
387 } else | 428 } else { |
388 return false; | 429 return false; |
| 430 } |
389 } | 431 } |
390 | 432 |
391 return a == aEnd; | 433 return a == aEnd; |
392 } | 434 } |
393 | 435 |
394 bool equalUTF16WithUTF8(const UChar* a, const UChar* aEnd, const char* b, const
char* bEnd) | 436 bool equalUTF16WithUTF8(const UChar* a, const UChar* aEnd, const char* b, const
char* bEnd) |
395 { | 437 { |
396 return equalWithUTF8Internal(a, aEnd, b, bEnd); | 438 return equalWithUTF8Internal(a, aEnd, b, bEnd); |
397 } | 439 } |
398 | 440 |
399 bool equalLatin1WithUTF8(const LChar* a, const LChar* aEnd, const char* b, const
char* bEnd) | 441 bool equalLatin1WithUTF8(const LChar* a, const LChar* aEnd, const char* b, const
char* bEnd) |
400 { | 442 { |
401 return equalWithUTF8Internal(a, aEnd, b, bEnd); | 443 return equalWithUTF8Internal(a, aEnd, b, bEnd); |
402 } | 444 } |
403 | 445 |
404 } // namespace Unicode | 446 } // namespace Unicode |
405 } // namespace WTF | 447 } // namespace WTF |
OLD | NEW |