| OLD | NEW |
| 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 #ifndef VM_UNICODE_H_ | 5 #ifndef VM_UNICODE_H_ |
| 6 #define VM_UNICODE_H_ | 6 #define VM_UNICODE_H_ |
| 7 | 7 |
| 8 #include "vm/allocation.h" | 8 #include "vm/allocation.h" |
| 9 #include "vm/globals.h" | 9 #include "vm/globals.h" |
| 10 | 10 |
| 11 namespace dart { | 11 namespace dart { |
| 12 | 12 |
| 13 class String; | 13 class String; |
| 14 | 14 |
| 15 class Utf : AllStatic { |
| 16 public: |
| 17 static const int32_t kMaxCodePoint = 0x10FFFF; |
| 18 |
| 19 static bool IsLatin1(int32_t code_point) { |
| 20 return (code_point >= 0) && (code_point <= 0xFF); |
| 21 } |
| 22 |
| 23 static bool IsBmp(int32_t code_point) { |
| 24 return (code_point >= 0) && (code_point <= 0xFFFF); |
| 25 } |
| 26 |
| 27 static bool IsSupplementary(int32_t code_point) { |
| 28 return (code_point > 0xFFFF) && (code_point <= kMaxCodePoint); |
| 29 } |
| 30 |
| 31 // Returns true if the code point value is above Plane 17. |
| 32 static bool IsOutOfRange(int32_t code_point) { |
| 33 return (code_point < 0) || (code_point > kMaxCodePoint); |
| 34 } |
| 35 }; |
| 36 |
| 37 |
| 15 class Utf8 : AllStatic { | 38 class Utf8 : AllStatic { |
| 16 public: | 39 public: |
| 17 enum Type { | 40 enum Type { |
| 18 kLatin1 = 0, // Latin-1 code point [U+0000, U+00FF]. | 41 kLatin1 = 0, // Latin-1 code point [U+0000, U+00FF]. |
| 19 kBMP, // Basic Multilingual Plane code point [U+0000, U+FFFF]. | 42 kBMP, // Basic Multilingual Plane code point [U+0000, U+FFFF]. |
| 20 kSupplementary, // Supplementary code point [U+010000, U+10FFFF]. | 43 kSupplementary, // Supplementary code point [U+010000, U+10FFFF]. |
| 21 }; | 44 }; |
| 22 | 45 |
| 23 static const intptr_t kMaxOneByteChar = 0x7F; | |
| 24 static const intptr_t kMaxTwoByteChar = 0x7FF; | |
| 25 static const intptr_t kMaxThreeByteChar = 0xFFFF; | |
| 26 static const intptr_t kMaxFourByteChar = 0x10FFFF; | |
| 27 | |
| 28 static intptr_t CodePointCount(const uint8_t* utf8_array, | 46 static intptr_t CodePointCount(const uint8_t* utf8_array, |
| 29 intptr_t array_len, | 47 intptr_t array_len, |
| 30 Type* type); | 48 Type* type); |
| 31 | 49 |
| 32 // Returns true if 'utf8_array' is a valid UTF-8 string. | 50 // Returns true if 'utf8_array' is a valid UTF-8 string. |
| 33 static bool IsValid(const uint8_t* utf8_array, intptr_t array_len); | 51 static bool IsValid(const uint8_t* utf8_array, intptr_t array_len); |
| 34 | 52 |
| 35 static intptr_t Length(int32_t ch); | 53 static intptr_t Length(int32_t ch); |
| 36 static intptr_t Length(const String& str); | 54 static intptr_t Length(const String& str); |
| 37 | 55 |
| (...skipping 11 matching lines...) Expand all Loading... |
| 49 static bool DecodeToUTF16(const uint8_t* utf8_array, | 67 static bool DecodeToUTF16(const uint8_t* utf8_array, |
| 50 intptr_t array_len, | 68 intptr_t array_len, |
| 51 uint16_t* dst, | 69 uint16_t* dst, |
| 52 intptr_t len); | 70 intptr_t len); |
| 53 static bool DecodeToUTF32(const uint8_t* utf8_array, | 71 static bool DecodeToUTF32(const uint8_t* utf8_array, |
| 54 intptr_t array_len, | 72 intptr_t array_len, |
| 55 int32_t* dst, | 73 int32_t* dst, |
| 56 intptr_t len); | 74 intptr_t len); |
| 57 static bool DecodeCStringToUTF32(const char* str, | 75 static bool DecodeCStringToUTF32(const char* str, |
| 58 int32_t* dst, | 76 int32_t* dst, |
| 59 intptr_t len) { | 77 intptr_t len); |
| 60 ASSERT(str != NULL); | 78 |
| 61 intptr_t array_len = strlen(str); | 79 private: |
| 62 const uint8_t* utf8_array = reinterpret_cast<const uint8_t*>(str); | 80 static const int32_t kMaxOneByteChar = 0x7F; |
| 63 return DecodeToUTF32(utf8_array, array_len, dst, len); | 81 static const int32_t kMaxTwoByteChar = 0x7FF; |
| 82 static const int32_t kMaxThreeByteChar = 0xFFFF; |
| 83 static const int32_t kMaxFourByteChar = Utf::kMaxCodePoint; |
| 84 |
| 85 static bool IsTrailByte(uint8_t code_unit) { |
| 86 return (code_unit & 0xc0) == 0x80; |
| 64 } | 87 } |
| 88 |
| 89 static bool IsNonShortestForm(uint32_t code_point, size_t num_code_units) { |
| 90 return code_point < kOverlongMinimum[num_code_units]; |
| 91 } |
| 92 |
| 93 static bool IsLatin1SequenceStart(uint8_t code_unit) { |
| 94 // Check is codepoint is <= U+00FF |
| 95 return (code_unit <= Utf8::kMaxOneByteChar); |
| 96 } |
| 97 |
| 98 static bool IsSupplementarySequenceStart(uint8_t code_unit) { |
| 99 // Check is codepoint is >= U+10000. |
| 100 return (code_unit >= 0xF0); |
| 101 } |
| 102 |
| 103 static const int8_t kTrailBytes[]; |
| 104 static const uint32_t kMagicBits[]; |
| 105 static const uint32_t kOverlongMinimum[]; |
| 65 }; | 106 }; |
| 66 | 107 |
| 67 | 108 |
| 68 class Utf16 : AllStatic { | 109 class Utf16 : AllStatic { |
| 69 public: | 110 public: |
| 70 static const int32_t kMaxBmpCodepoint = 0xFFFF; | |
| 71 | |
| 72 static const int32_t kLeadSurrogateOffset = (0xD800 - (0x10000 >> 10)); | |
| 73 | |
| 74 static const int32_t kSurrogateOffset = (0x10000 - (0xD800 << 10) - 0xDC00); | |
| 75 | |
| 76 // Returns the length of the code point in UTF-16 code units. | 111 // Returns the length of the code point in UTF-16 code units. |
| 77 static intptr_t Length(int32_t ch) { | 112 static intptr_t Length(int32_t ch) { |
| 78 return (ch <= kMaxBmpCodepoint) ? 1 : 2; | 113 return (ch <= Utf16::kMaxCodeUnit) ? 1 : 2; |
| 79 } | 114 } |
| 80 | 115 |
| 81 // Returns true if ch is a lead or trail surrogate. | 116 // Returns true if ch is a lead or trail surrogate. |
| 82 static bool IsSurrogate(int32_t ch) { | 117 static bool IsSurrogate(int32_t ch) { |
| 83 return (ch & 0xFFFFF800) == 0xD800; | 118 return (ch & 0xFFFFF800) == 0xD800; |
| 84 } | 119 } |
| 85 | 120 |
| 86 // Returns true if ch is a lead surrogate. | 121 // Returns true if ch is a lead surrogate. |
| 87 static bool IsLeadSurrogate(int32_t ch) { | 122 static bool IsLeadSurrogate(int32_t ch) { |
| 88 return (ch & 0xFFFFFC00) == 0xD800; | 123 return (ch & 0xFFFFFC00) == 0xD800; |
| (...skipping 19 matching lines...) Expand all Loading... |
| 108 return ch; | 143 return ch; |
| 109 } | 144 } |
| 110 | 145 |
| 111 // Decodes a surrogate pair into a supplementary code point. | 146 // Decodes a surrogate pair into a supplementary code point. |
| 112 static int32_t Decode(int32_t lead, int32_t trail) { | 147 static int32_t Decode(int32_t lead, int32_t trail) { |
| 113 return 0x10000 + ((lead & 0x3FF) << 10) + (trail & 0x3FF); | 148 return 0x10000 + ((lead & 0x3FF) << 10) + (trail & 0x3FF); |
| 114 } | 149 } |
| 115 | 150 |
| 116 // Encodes a single code point. | 151 // Encodes a single code point. |
| 117 static void Encode(int32_t codepoint, uint16_t* dst); | 152 static void Encode(int32_t codepoint, uint16_t* dst); |
| 153 |
| 154 private: |
| 155 static const int32_t kMaxCodeUnit = 0xFFFF; |
| 156 |
| 157 static const int32_t kLeadSurrogateOffset = (0xD800 - (0x10000 >> 10)); |
| 158 |
| 159 static const int32_t kSurrogateOffset = (0x10000 - (0xD800 << 10) - 0xDC00); |
| 118 }; | 160 }; |
| 119 | 161 |
| 120 | 162 |
| 121 class CaseMapping : AllStatic { | 163 class CaseMapping : AllStatic { |
| 122 public: | 164 public: |
| 123 // Maps a code point to uppercase. | 165 // Maps a code point to uppercase. |
| 124 static int32_t ToUpper(int32_t code_point) { | 166 static int32_t ToUpper(int32_t code_point) { |
| 125 return Convert(code_point, kUppercase); | 167 return Convert(code_point, kUppercase); |
| 126 } | 168 } |
| 127 | 169 |
| (...skipping 18 matching lines...) Expand all Loading... |
| 146 | 188 |
| 147 // The size of the stage 1 index. | 189 // The size of the stage 1 index. |
| 148 // TODO(cshapiro): improve indexing so this value is unnecessary. | 190 // TODO(cshapiro): improve indexing so this value is unnecessary. |
| 149 static const int kStage1Size = 261; | 191 static const int kStage1Size = 261; |
| 150 | 192 |
| 151 // The size of a stage 2 block in bytes. | 193 // The size of a stage 2 block in bytes. |
| 152 static const int kBlockSizeLog2 = 8; | 194 static const int kBlockSizeLog2 = 8; |
| 153 static const int kBlockSize = 1 << kBlockSizeLog2; | 195 static const int kBlockSize = 1 << kBlockSizeLog2; |
| 154 | 196 |
| 155 static int32_t Convert(int32_t ch, int32_t mapping) { | 197 static int32_t Convert(int32_t ch, int32_t mapping) { |
| 156 if (ch <= 0xFF) { | 198 if (Utf::IsLatin1(ch)) { |
| 157 int32_t info = stage2_[ch]; | 199 int32_t info = stage2_[ch]; |
| 158 if ((info & kTypeMask) == mapping) { | 200 if ((info & kTypeMask) == mapping) { |
| 159 ch += info >> kTypeShift; | 201 ch += info >> kTypeShift; |
| 160 } | 202 } |
| 161 } else if (ch <= (kStage1Size << kBlockSizeLog2)) { | 203 } else if (ch <= (kStage1Size << kBlockSizeLog2)) { |
| 162 int16_t offset = stage1_[ch >> kBlockSizeLog2] << kBlockSizeLog2; | 204 int16_t offset = stage1_[ch >> kBlockSizeLog2] << kBlockSizeLog2; |
| 163 int32_t info = stage2_[offset + (ch & (kBlockSize - 1))]; | 205 int32_t info = stage2_[offset + (ch & (kBlockSize - 1))]; |
| 164 int32_t type = info & kTypeMask; | 206 int32_t type = info & kTypeMask; |
| 165 if (type == mapping) { | 207 if (type == mapping) { |
| 166 ch += (info >> kTypeShift); | 208 ch += (info >> kTypeShift); |
| (...skipping 10 matching lines...) Expand all Loading... |
| 177 // Data for small code points with one mapping | 219 // Data for small code points with one mapping |
| 178 static const int16_t stage2_[]; | 220 static const int16_t stage2_[]; |
| 179 | 221 |
| 180 // Data for large code points or code points with both mappings. | 222 // Data for large code points or code points with both mappings. |
| 181 static const int32_t stage2_exception_[][2]; | 223 static const int32_t stage2_exception_[][2]; |
| 182 }; | 224 }; |
| 183 | 225 |
| 184 } // namespace dart | 226 } // namespace dart |
| 185 | 227 |
| 186 #endif // VM_UNICODE_H_ | 228 #endif // VM_UNICODE_H_ |
| OLD | NEW |