| OLD | NEW |
| 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 #include "vm/unicode.h" | 5 #include "vm/unicode.h" |
| 6 | 6 |
| 7 #include "vm/allocation.h" | 7 #include "vm/allocation.h" |
| 8 #include "vm/globals.h" | 8 #include "vm/globals.h" |
| 9 #include "vm/object.h" | 9 #include "vm/object.h" |
| 10 | 10 |
| 11 namespace dart { | 11 namespace dart { |
| 12 | 12 |
| 13 static const int8_t kTrailBytes[256] = { | 13 const int8_t Utf8::kTrailBytes[256] = { |
| 14 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, | 14 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, |
| 15 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, | 15 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, |
| 16 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, | 16 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, |
| 17 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, | 17 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, |
| 18 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, | 18 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, |
| 19 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, | 19 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, |
| 20 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, | 20 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, |
| 21 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, | 21 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, |
| 22 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | 22 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 23 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | 23 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 24 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | 24 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 25 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, | 25 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 26 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, | 26 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, |
| 27 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, | 27 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, |
| 28 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, | 28 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, |
| 29 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0 | 29 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0 |
| 30 }; | 30 }; |
| 31 | 31 |
| 32 | 32 |
| 33 static const uint32_t kMagicBits[7] = { | 33 const uint32_t Utf8::kMagicBits[7] = { |
| 34 0, // padding | 34 0, // padding |
| 35 0x00000000, | 35 0x00000000, |
| 36 0x00003080, | 36 0x00003080, |
| 37 0x000E2080, | 37 0x000E2080, |
| 38 0x03C82080, | 38 0x03C82080, |
| 39 0xFA082080, | 39 0xFA082080, |
| 40 0x82082080 | 40 0x82082080 |
| 41 }; | 41 }; |
| 42 | 42 |
| 43 | 43 |
| 44 // Minimum values of code points used to check shortest form. | 44 // Minimum values of code points used to check shortest form. |
| 45 static const uint32_t kOverlongMinimum[7] = { | 45 const uint32_t Utf8::kOverlongMinimum[7] = { |
| 46 0, // padding | 46 0, // padding |
| 47 0x0, | 47 0x0, |
| 48 0x80, | 48 0x80, |
| 49 0x800, | 49 0x800, |
| 50 0x10000, | 50 0x10000, |
| 51 0xFFFFFFFF, | 51 0xFFFFFFFF, |
| 52 0xFFFFFFFF | 52 0xFFFFFFFF |
| 53 }; | 53 }; |
| 54 | 54 |
| 55 | 55 |
| 56 static bool IsTrailByte(uint8_t code_unit) { | |
| 57 return (code_unit & 0xc0) == 0x80; | |
| 58 } | |
| 59 | |
| 60 | |
| 61 static bool IsLatin1SequenceStart(uint8_t code_unit) { | |
| 62 // Check is codepoint is <= U+00FF | |
| 63 return (code_unit <= Utf8::kMaxOneByteChar); | |
| 64 } | |
| 65 | |
| 66 | |
| 67 static bool IsSupplementarySequenceStart(uint8_t code_unit) { | |
| 68 // Check is codepoint is >= U+10000. | |
| 69 return (code_unit >= 0xF0); | |
| 70 } | |
| 71 | |
| 72 | |
| 73 // Returns true if the code point value is above Plane 17. | |
| 74 static bool IsOutOfRange(uint32_t code_point) { | |
| 75 return (code_point > 0x10FFFF); | |
| 76 } | |
| 77 | |
| 78 | |
| 79 // Returns true if the byte sequence is ill-formed. | |
| 80 static bool IsNonShortestForm(uint32_t code_point, size_t num_bytes) { | |
| 81 return code_point < kOverlongMinimum[num_bytes]; | |
| 82 } | |
| 83 | |
| 84 | |
| 85 // Returns a count of the number of UTF-8 trail bytes. | 56 // Returns a count of the number of UTF-8 trail bytes. |
| 86 intptr_t Utf8::CodePointCount(const uint8_t* utf8_array, | 57 intptr_t Utf8::CodePointCount(const uint8_t* utf8_array, |
| 87 intptr_t array_len, | 58 intptr_t array_len, |
| 88 Type* type) { | 59 Type* type) { |
| 89 intptr_t len = 0; | 60 intptr_t len = 0; |
| 90 Type char_type = kLatin1; | 61 Type char_type = kLatin1; |
| 91 for (intptr_t i = 0; i < array_len; i++) { | 62 for (intptr_t i = 0; i < array_len; i++) { |
| 92 uint8_t code_unit = utf8_array[i]; | 63 uint8_t code_unit = utf8_array[i]; |
| 93 if (!IsTrailByte(code_unit)) { | 64 if (!IsTrailByte(code_unit)) { |
| 94 ++len; | 65 ++len; |
| (...skipping 26 matching lines...) Expand all Loading... |
| 121 uint8_t code_unit = utf8_array[i + j]; | 92 uint8_t code_unit = utf8_array[i + j]; |
| 122 is_malformed |= !IsTrailByte(code_unit); | 93 is_malformed |= !IsTrailByte(code_unit); |
| 123 ch = (ch << 6) + code_unit; | 94 ch = (ch << 6) + code_unit; |
| 124 } else { | 95 } else { |
| 125 return false; | 96 return false; |
| 126 } | 97 } |
| 127 } | 98 } |
| 128 ch -= kMagicBits[num_trail_bytes]; | 99 ch -= kMagicBits[num_trail_bytes]; |
| 129 if (!((is_malformed == false) && | 100 if (!((is_malformed == false) && |
| 130 (j == num_trail_bytes) && | 101 (j == num_trail_bytes) && |
| 131 !IsOutOfRange(ch) && | 102 !Utf::IsOutOfRange(ch) && |
| 132 !IsNonShortestForm(ch, j) && | 103 !IsNonShortestForm(ch, j) && |
| 133 !Utf16::IsSurrogate(ch))) { | 104 !Utf16::IsSurrogate(ch))) { |
| 134 return false; | 105 return false; |
| 135 } | 106 } |
| 136 } | 107 } |
| 137 i += j; | 108 i += j; |
| 138 } | 109 } |
| 139 return true; | 110 return true; |
| 140 } | 111 } |
| 141 | 112 |
| (...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 220 is_malformed |= !IsTrailByte(code_unit); | 191 is_malformed |= !IsTrailByte(code_unit); |
| 221 ch = (ch << 6) + code_unit; | 192 ch = (ch << 6) + code_unit; |
| 222 } else { | 193 } else { |
| 223 *dst = -1; | 194 *dst = -1; |
| 224 return 0; | 195 return 0; |
| 225 } | 196 } |
| 226 } | 197 } |
| 227 ch -= kMagicBits[num_trail_bytes]; | 198 ch -= kMagicBits[num_trail_bytes]; |
| 228 if (!((is_malformed == false) && | 199 if (!((is_malformed == false) && |
| 229 (i == num_trail_bytes) && | 200 (i == num_trail_bytes) && |
| 230 !IsOutOfRange(ch) && | 201 !Utf::IsOutOfRange(ch) && |
| 231 !IsNonShortestForm(ch, i) && | 202 !IsNonShortestForm(ch, i) && |
| 232 !Utf16::IsSurrogate(ch))) { | 203 !Utf16::IsSurrogate(ch))) { |
| 233 *dst = -1; | 204 *dst = -1; |
| 234 return 0; | 205 return 0; |
| 235 } | 206 } |
| 236 } | 207 } |
| 237 *dst = ch; | 208 *dst = ch; |
| 238 return i; | 209 return i; |
| 239 } | 210 } |
| 240 | 211 |
| 241 | 212 |
| 242 bool Utf8::DecodeToLatin1(const uint8_t* utf8_array, | 213 bool Utf8::DecodeToLatin1(const uint8_t* utf8_array, |
| 243 intptr_t array_len, | 214 intptr_t array_len, |
| 244 uint8_t* dst, | 215 uint8_t* dst, |
| 245 intptr_t len) { | 216 intptr_t len) { |
| 246 intptr_t i = 0; | 217 intptr_t i = 0; |
| 247 intptr_t j = 0; | 218 intptr_t j = 0; |
| 248 intptr_t num_bytes; | 219 intptr_t num_bytes; |
| 249 for (; (i < array_len) && (j < len); i += num_bytes, ++j) { | 220 for (; (i < array_len) && (j < len); i += num_bytes, ++j) { |
| 250 int32_t ch; | 221 int32_t ch; |
| 251 ASSERT(IsLatin1SequenceStart(utf8_array[i])); | 222 ASSERT(IsLatin1SequenceStart(utf8_array[i])); |
| 252 num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch); | 223 num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch); |
| 253 if (ch == -1) { | 224 if (ch == -1) { |
| 254 return false; // invalid input | 225 return false; // invalid input |
| 255 } | 226 } |
| 256 ASSERT(ch <= 0xff); | 227 ASSERT(Utf::IsLatin1(ch)); |
| 257 dst[j] = ch; | 228 dst[j] = ch; |
| 258 } | 229 } |
| 259 if ((i < array_len) && (j == len)) { | 230 if ((i < array_len) && (j == len)) { |
| 260 return false; // output overflow | 231 return false; // output overflow |
| 261 } | 232 } |
| 262 return true; // success | 233 return true; // success |
| 263 } | 234 } |
| 264 | 235 |
| 265 | 236 |
| 266 bool Utf8::DecodeToUTF16(const uint8_t* utf8_array, | 237 bool Utf8::DecodeToUTF16(const uint8_t* utf8_array, |
| (...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 306 } | 277 } |
| 307 dst[j] = ch; | 278 dst[j] = ch; |
| 308 } | 279 } |
| 309 if ((i < array_len) && (j == len)) { | 280 if ((i < array_len) && (j == len)) { |
| 310 return false; // output overflow | 281 return false; // output overflow |
| 311 } | 282 } |
| 312 return true; // success | 283 return true; // success |
| 313 } | 284 } |
| 314 | 285 |
| 315 | 286 |
| 287 bool Utf8::DecodeCStringToUTF32(const char* str, int32_t* dst, intptr_t len) { |
| 288 ASSERT(str != NULL); |
| 289 intptr_t array_len = strlen(str); |
| 290 const uint8_t* utf8_array = reinterpret_cast<const uint8_t*>(str); |
| 291 return Utf8::DecodeToUTF32(utf8_array, array_len, dst, len); |
| 292 } |
| 293 |
| 294 |
| 316 void Utf16::Encode(int32_t codepoint, uint16_t* dst) { | 295 void Utf16::Encode(int32_t codepoint, uint16_t* dst) { |
| 317 ASSERT(codepoint > kMaxBmpCodepoint); | 296 ASSERT(codepoint > Utf16::kMaxCodeUnit); |
| 318 ASSERT(dst != NULL); | 297 ASSERT(dst != NULL); |
| 319 dst[0] = (Utf16::kLeadSurrogateOffset + (codepoint >> 10)); | 298 dst[0] = (Utf16::kLeadSurrogateOffset + (codepoint >> 10)); |
| 320 dst[1] = (0xDC00 + (codepoint & 0x3FF)); | 299 dst[1] = (0xDC00 + (codepoint & 0x3FF)); |
| 321 } | 300 } |
| 322 | 301 |
| 323 } // namespace dart | 302 } // namespace dart |
| OLD | NEW |