Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(43)

Side by Side Diff: runtime/vm/unicode.h

Issue 11411092: Revert "Add some support for the code-point code-unit distinction." (Closed) Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart
Patch Set: Created 8 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « runtime/vm/symbols.cc ('k') | runtime/vm/unicode.cc » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 #ifndef VM_UNICODE_H_ 5 #ifndef VM_UNICODE_H_
6 #define VM_UNICODE_H_ 6 #define VM_UNICODE_H_
7 7
8 #include "vm/allocation.h" 8 #include "vm/allocation.h"
9 #include "vm/globals.h" 9 #include "vm/globals.h"
10 10
11 namespace dart { 11 namespace dart {
12 12
13 class String; 13 class String;
14 14
15
16 class Utf8 : AllStatic { 15 class Utf8 : AllStatic {
17 public: 16 public:
18 enum Type { 17 enum Type {
19 kLatin1 = 0, // Latin-1 code point [U+0000, U+00FF]. 18 kLatin1 = 0, // Latin-1 code point [U+0000, U+00FF].
20 kBMP, // Basic Multilingual Plane code point [U+0000, U+FFFF]. 19 kBMP, // Basic Multilingual Plane code point [U+0000, U+FFFF].
21 kSupplementary, // Supplementary code point [U+010000, U+10FFFF]. 20 kSupplementary, // Supplementary code point [U+010000, U+10FFFF].
22 }; 21 };
23 22
24 static const intptr_t kMaxOneByteChar = 0x7F; 23 static const intptr_t kMaxOneByteChar = 0x7F;
25 static const intptr_t kMaxTwoByteChar = 0x7FF; 24 static const intptr_t kMaxTwoByteChar = 0x7FF;
26 static const intptr_t kMaxThreeByteChar = 0xFFFF; 25 static const intptr_t kMaxThreeByteChar = 0xFFFF;
27 static const intptr_t kMaxFourByteChar = 0x10FFFF; 26 static const intptr_t kMaxFourByteChar = 0x10FFFF;
28 27
29 static const int32_t kInvalidCodePoint = -1; 28 static intptr_t CodePointCount(const uint8_t* utf8_array,
30 29 intptr_t array_len,
31 static intptr_t CodeUnitCount(const uint8_t* utf8_array, 30 Type* type);
32 intptr_t array_len,
33 Type* type);
34 31
35 // Returns true if 'utf8_array' is a valid UTF-8 string. 32 // Returns true if 'utf8_array' is a valid UTF-8 string.
36 static bool IsValid(const uint8_t* utf8_array, intptr_t array_len); 33 static bool IsValid(const uint8_t* utf8_array, intptr_t array_len);
37 34
38 static intptr_t Length(int32_t ch); 35 static intptr_t Length(int32_t ch);
39 static intptr_t Length(const String& str); 36 static intptr_t Length(const String& str);
40 37
41 static intptr_t Encode(int32_t ch, char* dst); 38 static intptr_t Encode(int32_t ch, char* dst);
42 static intptr_t Encode(const String& src, char* dst, intptr_t len); 39 static intptr_t Encode(const String& src, char* dst, intptr_t len);
43 40
44 static intptr_t Decode(const uint8_t* utf8_array, 41 static intptr_t Decode(const uint8_t* utf8_array,
45 intptr_t array_len, 42 intptr_t array_len,
46 int32_t* ch); 43 int32_t* ch);
47 44
48 static bool DecodeToLatin1(const uint8_t* utf8_array, 45 static bool DecodeToLatin1(const uint8_t* utf8_array,
49 intptr_t array_len, 46 intptr_t array_len,
50 uint8_t* dst, 47 uint8_t* dst,
51 intptr_t len); 48 intptr_t len);
52 static bool DecodeToUTF16(const uint8_t* utf8_array, 49 static bool DecodeToUTF16(const uint8_t* utf8_array,
53 intptr_t array_len, 50 intptr_t array_len,
54 uint16_t* dst, 51 uint16_t* dst,
55 intptr_t len); 52 intptr_t len);
56 static bool DecodeToUTF32(const uint8_t* utf8_array, 53 static bool DecodeToUTF32(const uint8_t* utf8_array,
57 intptr_t array_len, 54 intptr_t array_len,
58 int32_t* dst, 55 uint32_t* dst,
59 intptr_t len); 56 intptr_t len);
60 static bool DecodeCStringToUTF32(const char* str, 57 static bool DecodeCStringToUTF32(const char* str,
61 int32_t* dst, 58 uint32_t* dst,
62 intptr_t len) { 59 intptr_t len) {
63 ASSERT(str != NULL); 60 ASSERT(str != NULL);
64 intptr_t array_len = strlen(str); 61 intptr_t array_len = strlen(str);
65 const uint8_t* utf8_array = reinterpret_cast<const uint8_t*>(str); 62 const uint8_t* utf8_array = reinterpret_cast<const uint8_t*>(str);
66 return DecodeToUTF32(utf8_array, array_len, dst, len); 63 return DecodeToUTF32(utf8_array, array_len, dst, len);
67 } 64 }
68 }; 65 };
69 66
70 67
71 class Utf16 : AllStatic { 68 class Utf16 : AllStatic {
72 public: 69 public:
73 static const int32_t kMaxBmpCodepoint = 0xFFFF; 70 static const int32_t kMaxBmpCodepoint = 0xFFFF;
74 static const int32_t kMaxCodeUnit = 0xFFFF;
75 static const int32_t kMaxCodePoint = 0x10FFFF;
76 71
77 static const int32_t kSurrogateEncodingBase = 0x10000; 72 static const int32_t kLeadSurrogateOffset = (0xD800 - (0x10000 >> 10));
73
74 static const int32_t kSurrogateOffset = (0x10000 - (0xD800 << 10) - 0xDC00);
78 75
79 // Returns the length of the code point in UTF-16 code units. 76 // Returns the length of the code point in UTF-16 code units.
80 static intptr_t Length(int32_t ch) { 77 static intptr_t Length(int32_t ch) {
81 return (ch <= kMaxBmpCodepoint) ? 1 : 2; 78 return (ch <= kMaxBmpCodepoint) ? 1 : 2;
82 } 79 }
83 80
84 // Returns true if ch is a lead or trail surrogate. 81 // Returns true if ch is a lead or trail surrogate.
85 static bool IsSurrogate(int32_t ch) { 82 static bool IsSurrogate(int32_t ch) {
86 return (ch & 0xFFFFF800) == 0xD800; 83 return (ch & 0xFFFFF800) == 0xD800;
87 } 84 }
88 85
89 // Returns true if ch is a lead surrogate. 86 // Returns true if ch is a lead surrogate.
90 static bool IsLeadSurrogate(int32_t ch) { 87 static bool IsLeadSurrogate(int32_t ch) {
91 return (ch & 0xFFFFFC00) == 0xD800; 88 return (ch & 0xFFFFFC00) == 0xD800;
92 } 89 }
93 90
94 // Returns true if ch is a low surrogate. 91 // Returns true if ch is a low surrogate.
95 static bool IsTrailSurrogate(int32_t ch) { 92 static bool IsTrailSurrogate(int32_t ch) {
96 return (ch & 0xFFFFFC00) == 0xDC00; 93 return (ch & 0xFFFFFC00) == 0xDC00;
97 } 94 }
98 95
99 // Decodes a surrogate pair into a supplementary code point. 96 // Decodes a surrogate pair into a supplementary code point.
100 static int32_t Decode(int32_t lead, int32_t trail) { 97 static int32_t Decode(int32_t lead, int32_t trail) {
101 ASSERT(IsLeadSurrogate(lead)); 98 return 0x10000 + ((lead & 0x3FF) << 10) + (trail & 0x3FF);
102 ASSERT(IsTrailSurrogate(trail));
103 return kSurrogateEncodingBase +
104 ((lead & kSurrogateMask) << 10) + (trail & kSurrogateMask);
105 }
106
107 static int32_t LeadFromCodePoint(int32_t code_point) {
108 ASSERT(code_point >= kSurrogateEncodingBase);
109 return kLeadBase +
110 (((code_point - kSurrogateEncodingBase) >> 10) & kSurrogateMask);
111 }
112
113 static int32_t TrailFromCodePoint(int32_t code_point) {
114 ASSERT(code_point >= kSurrogateEncodingBase);
115 return kTrailBase + (code_point & kSurrogateMask);
116 } 99 }
117 100
118 // Encodes a single code point. 101 // Encodes a single code point.
119 static void Encode(int32_t codepoint, uint16_t* dst); 102 static void Encode(int32_t codepoint, uint16_t* dst);
120
121 // Gets the 21 bit Unicode code point at the given index in a string. If the
122 // returned value is greater than kMaxCodePoint then the next position of the
123 // string encodes a trail surrogate and should be skipped on iteration. May
124 // return individual surrogate values if they are not part of a pair.
125 static int32_t CodePointAt(const String& str, int index);
126
127 private:
128 static const int32_t kLeadBase = 0xD800;
129 static const int32_t kTrailBase = 0xDC00;
130 static const int32_t kSurrogateMask = 0x3FF;
131 }; 103 };
132 104
133 105
134 class CaseMapping : AllStatic { 106 class CaseMapping : AllStatic {
135 public: 107 public:
136 // Maps a code point to uppercase. 108 // Maps a code point to uppercase.
137 static int32_t ToUpper(int32_t code_point) { 109 static int32_t ToUpper(int32_t code_point) {
138 return Convert(code_point, kUppercase); 110 return Convert(code_point, kUppercase);
139 } 111 }
140 112
(...skipping 49 matching lines...) Expand 10 before | Expand all | Expand 10 after
190 // Data for small code points with one mapping 162 // Data for small code points with one mapping
191 static const int16_t stage2_[]; 163 static const int16_t stage2_[];
192 164
193 // Data for large code points or code points with both mappings. 165 // Data for large code points or code points with both mappings.
194 static const int32_t stage2_exception_[][2]; 166 static const int32_t stage2_exception_[][2];
195 }; 167 };
196 168
197 } // namespace dart 169 } // namespace dart
198 170
199 #endif // VM_UNICODE_H_ 171 #endif // VM_UNICODE_H_
OLDNEW
« no previous file with comments | « runtime/vm/symbols.cc ('k') | runtime/vm/unicode.cc » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698