Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(731)

Side by Side Diff: runtime/vm/unicode.cc

Issue 11414249: Move various top-level Unicode definitions into classes and methods. (Closed) Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart
Patch Set: Created 8 years ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« runtime/vm/unicode.h ('K') | « runtime/vm/unicode.h ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 #include "vm/unicode.h" 5 #include "vm/unicode.h"
6 6
7 #include "vm/allocation.h" 7 #include "vm/allocation.h"
8 #include "vm/globals.h" 8 #include "vm/globals.h"
9 #include "vm/object.h" 9 #include "vm/object.h"
10 10
11 namespace dart { 11 namespace dart {
12 12
13 static const int8_t kTrailBytes[256] = { 13 const int8_t Utf8::kTrailBytes[256] = {
14 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 14 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
15 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 15 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
16 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 16 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
17 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 17 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
18 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 18 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
19 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 19 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
20 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 20 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
21 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 21 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
22 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
23 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
24 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
25 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
26 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 26 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
27 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 27 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
28 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 28 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
29 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0 29 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0
30 }; 30 };
31 31
32 32
33 static const uint32_t kMagicBits[7] = { 33 const uint32_t Utf8::kMagicBits[7] = {
34 0, // padding 34 0, // padding
35 0x00000000, 35 0x00000000,
36 0x00003080, 36 0x00003080,
37 0x000E2080, 37 0x000E2080,
38 0x03C82080, 38 0x03C82080,
39 0xFA082080, 39 0xFA082080,
40 0x82082080 40 0x82082080
41 }; 41 };
42 42
43 43
44 // Minimum values of code points used to check shortest form. 44 // Minimum values of code points used to check shortest form.
45 static const uint32_t kOverlongMinimum[7] = { 45 const uint32_t Utf8::kOverlongMinimum[7] = {
46 0, // padding 46 0, // padding
47 0x0, 47 0x0,
48 0x80, 48 0x80,
49 0x800, 49 0x800,
50 0x10000, 50 0x10000,
51 0xFFFFFFFF, 51 0xFFFFFFFF,
52 0xFFFFFFFF 52 0xFFFFFFFF
53 }; 53 };
54 54
55 55
56 static bool IsTrailByte(uint8_t code_unit) {
57 return (code_unit & 0xc0) == 0x80;
58 }
59
60
61 static bool IsLatin1SequenceStart(uint8_t code_unit) {
62 // Check is codepoint is <= U+00FF
63 return (code_unit <= Utf8::kMaxOneByteChar);
64 }
65
66
67 static bool IsSupplementarySequenceStart(uint8_t code_unit) {
68 // Check is codepoint is >= U+10000.
69 return (code_unit >= 0xF0);
70 }
71
72
73 // Returns true if the code point value is above Plane 17.
74 static bool IsOutOfRange(uint32_t code_point) {
75 return (code_point > 0x10FFFF);
76 }
77
78
79 // Returns true if the byte sequence is ill-formed.
80 static bool IsNonShortestForm(uint32_t code_point, size_t num_bytes) {
81 return code_point < kOverlongMinimum[num_bytes];
82 }
83
84
85 // Returns a count of the number of UTF-8 trail bytes. 56 // Returns a count of the number of UTF-8 trail bytes.
86 intptr_t Utf8::CodePointCount(const uint8_t* utf8_array, 57 intptr_t Utf8::CodePointCount(const uint8_t* utf8_array,
87 intptr_t array_len, 58 intptr_t array_len,
88 Type* type) { 59 Type* type) {
89 intptr_t len = 0; 60 intptr_t len = 0;
90 Type char_type = kLatin1; 61 Type char_type = kLatin1;
91 for (intptr_t i = 0; i < array_len; i++) { 62 for (intptr_t i = 0; i < array_len; i++) {
92 uint8_t code_unit = utf8_array[i]; 63 uint8_t code_unit = utf8_array[i];
93 if (!IsTrailByte(code_unit)) { 64 if (!IsTrailByte(code_unit)) {
94 ++len; 65 ++len;
(...skipping 26 matching lines...) Expand all
121 uint8_t code_unit = utf8_array[i + j]; 92 uint8_t code_unit = utf8_array[i + j];
122 is_malformed |= !IsTrailByte(code_unit); 93 is_malformed |= !IsTrailByte(code_unit);
123 ch = (ch << 6) + code_unit; 94 ch = (ch << 6) + code_unit;
124 } else { 95 } else {
125 return false; 96 return false;
126 } 97 }
127 } 98 }
128 ch -= kMagicBits[num_trail_bytes]; 99 ch -= kMagicBits[num_trail_bytes];
129 if (!((is_malformed == false) && 100 if (!((is_malformed == false) &&
130 (j == num_trail_bytes) && 101 (j == num_trail_bytes) &&
131 !IsOutOfRange(ch) && 102 !Utf::IsOutOfRange(ch) &&
132 !IsNonShortestForm(ch, j) && 103 !IsNonShortestForm(ch, j) &&
133 !Utf16::IsSurrogate(ch))) { 104 !Utf16::IsSurrogate(ch))) {
134 return false; 105 return false;
135 } 106 }
136 } 107 }
137 i += j; 108 i += j;
138 } 109 }
139 return true; 110 return true;
140 } 111 }
141 112
(...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after
220 is_malformed |= !IsTrailByte(code_unit); 191 is_malformed |= !IsTrailByte(code_unit);
221 ch = (ch << 6) + code_unit; 192 ch = (ch << 6) + code_unit;
222 } else { 193 } else {
223 *dst = -1; 194 *dst = -1;
224 return 0; 195 return 0;
225 } 196 }
226 } 197 }
227 ch -= kMagicBits[num_trail_bytes]; 198 ch -= kMagicBits[num_trail_bytes];
228 if (!((is_malformed == false) && 199 if (!((is_malformed == false) &&
229 (i == num_trail_bytes) && 200 (i == num_trail_bytes) &&
230 !IsOutOfRange(ch) && 201 !Utf::IsOutOfRange(ch) &&
231 !IsNonShortestForm(ch, i) && 202 !IsNonShortestForm(ch, i) &&
232 !Utf16::IsSurrogate(ch))) { 203 !Utf16::IsSurrogate(ch))) {
233 *dst = -1; 204 *dst = -1;
234 return 0; 205 return 0;
235 } 206 }
236 } 207 }
237 *dst = ch; 208 *dst = ch;
238 return i; 209 return i;
239 } 210 }
240 211
241 212
242 bool Utf8::DecodeToLatin1(const uint8_t* utf8_array, 213 bool Utf8::DecodeToLatin1(const uint8_t* utf8_array,
243 intptr_t array_len, 214 intptr_t array_len,
244 uint8_t* dst, 215 uint8_t* dst,
245 intptr_t len) { 216 intptr_t len) {
246 intptr_t i = 0; 217 intptr_t i = 0;
247 intptr_t j = 0; 218 intptr_t j = 0;
248 intptr_t num_bytes; 219 intptr_t num_bytes;
249 for (; (i < array_len) && (j < len); i += num_bytes, ++j) { 220 for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
250 int32_t ch; 221 int32_t ch;
251 ASSERT(IsLatin1SequenceStart(utf8_array[i])); 222 ASSERT(IsLatin1SequenceStart(utf8_array[i]));
252 num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch); 223 num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
253 if (ch == -1) { 224 if (ch == -1) {
254 return false; // invalid input 225 return false; // invalid input
255 } 226 }
256 ASSERT(ch <= 0xff); 227 ASSERT(Utf::IsLatin1(ch));
257 dst[j] = ch; 228 dst[j] = ch;
258 } 229 }
259 if ((i < array_len) && (j == len)) { 230 if ((i < array_len) && (j == len)) {
260 return false; // output overflow 231 return false; // output overflow
261 } 232 }
262 return true; // success 233 return true; // success
263 } 234 }
264 235
265 236
266 bool Utf8::DecodeToUTF16(const uint8_t* utf8_array, 237 bool Utf8::DecodeToUTF16(const uint8_t* utf8_array,
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after
306 } 277 }
307 dst[j] = ch; 278 dst[j] = ch;
308 } 279 }
309 if ((i < array_len) && (j == len)) { 280 if ((i < array_len) && (j == len)) {
310 return false; // output overflow 281 return false; // output overflow
311 } 282 }
312 return true; // success 283 return true; // success
313 } 284 }
314 285
315 286
287 bool Utf8::DecodeCStringToUTF32(const char* str, int32_t* dst, intptr_t len) {
288 ASSERT(str != NULL);
289 intptr_t array_len = strlen(str);
290 const uint8_t* utf8_array = reinterpret_cast<const uint8_t*>(str);
291 return Utf8::DecodeToUTF32(utf8_array, array_len, dst, len);
292 }
293
294
316 void Utf16::Encode(int32_t codepoint, uint16_t* dst) { 295 void Utf16::Encode(int32_t codepoint, uint16_t* dst) {
317 ASSERT(codepoint > kMaxBmpCodepoint); 296 ASSERT(codepoint > Utf16::kMaxCodeUnit);
318 ASSERT(dst != NULL); 297 ASSERT(dst != NULL);
319 dst[0] = (Utf16::kLeadSurrogateOffset + (codepoint >> 10)); 298 dst[0] = (Utf16::kLeadSurrogateOffset + (codepoint >> 10));
320 dst[1] = (0xDC00 + (codepoint & 0x3FF)); 299 dst[1] = (0xDC00 + (codepoint & 0x3FF));
321 } 300 }
322 301
323 } // namespace dart 302 } // namespace dart
OLDNEW
« runtime/vm/unicode.h ('K') | « runtime/vm/unicode.h ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698