OLD | NEW |
1 // Copyright (c) 2009 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2009 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "base/utf_string_conversions.h" | 5 #include "base/utf_offset_string_conversions.h" |
6 | 6 |
7 #include <vector> | 7 #include "base/string_piece.h" |
| 8 #include "base/utf_string_conversion_utils.h" |
8 | 9 |
9 #include "base/basictypes.h" | 10 using base::PrepareForUTF16Or32Output; |
10 #include "base/logging.h" | 11 using base::ReadUnicodeCharacter; |
11 #include "base/string_util.h" | 12 using base::WriteUnicodeCharacter; |
12 #include "base/third_party/icu/icu_utf.h" | |
13 | |
14 namespace { | |
15 | |
16 inline bool IsValidCodepoint(uint32 code_point) { | |
17 // Excludes the surrogate code points ([0xD800, 0xDFFF]) and | |
18 // codepoints larger than 0x10FFFF (the highest codepoint allowed). | |
19 // Non-characters and unassigned codepoints are allowed. | |
20 return code_point < 0xD800u || | |
21 (code_point >= 0xE000u && code_point <= 0x10FFFFu); | |
22 } | |
23 | |
24 // ReadUnicodeCharacter -------------------------------------------------------- | |
25 | |
26 // Reads a UTF-8 stream, placing the next code point into the given output | |
27 // |*code_point|. |src| represents the entire string to read, and |*char_index| | |
28 // is the character offset within the string to start reading at. |*char_index| | |
29 // will be updated to index the last character read, such that incrementing it | |
30 // (as in a for loop) will take the reader to the next character. | |
31 // | |
32 // Returns true on success. On false, |*code_point| will be invalid. | |
33 bool ReadUnicodeCharacter(const char* src, int32 src_len, | |
34 int32* char_index, uint32* code_point_out) { | |
35 // U8_NEXT expects to be able to use -1 to signal an error, so we must | |
36 // use a signed type for code_point. But this function returns false | |
37 // on error anyway, so code_point_out is unsigned. | |
38 int32 code_point; | |
39 CBU8_NEXT(src, *char_index, src_len, code_point); | |
40 *code_point_out = static_cast<uint32>(code_point); | |
41 | |
42 // The ICU macro above moves to the next char, we want to point to the last | |
43 // char consumed. | |
44 (*char_index)--; | |
45 | |
46 // Validate the decoded value. | |
47 return IsValidCodepoint(code_point); | |
48 } | |
49 | |
50 // Reads a UTF-16 character. The usage is the same as the 8-bit version above. | |
51 bool ReadUnicodeCharacter(const char16* src, int32 src_len, | |
52 int32* char_index, uint32* code_point) { | |
53 if (CBU16_IS_SURROGATE(src[*char_index])) { | |
54 if (!CBU16_IS_SURROGATE_LEAD(src[*char_index]) || | |
55 *char_index + 1 >= src_len || | |
56 !CBU16_IS_TRAIL(src[*char_index + 1])) { | |
57 // Invalid surrogate pair. | |
58 return false; | |
59 } | |
60 | |
61 // Valid surrogate pair. | |
62 *code_point = CBU16_GET_SUPPLEMENTARY(src[*char_index], | |
63 src[*char_index + 1]); | |
64 (*char_index)++; | |
65 } else { | |
66 // Not a surrogate, just one 16-bit word. | |
67 *code_point = src[*char_index]; | |
68 } | |
69 | |
70 return IsValidCodepoint(*code_point); | |
71 } | |
72 | |
73 #if defined(WCHAR_T_IS_UTF32) | |
74 // Reads UTF-32 character. The usage is the same as the 8-bit version above. | |
75 bool ReadUnicodeCharacter(const wchar_t* src, int32 src_len, | |
76 int32* char_index, uint32* code_point) { | |
77 // Conversion is easy since the source is 32-bit. | |
78 *code_point = src[*char_index]; | |
79 | |
80 // Validate the value. | |
81 return IsValidCodepoint(*code_point); | |
82 } | |
83 #endif // defined(WCHAR_T_IS_UTF32) | |
84 | |
85 // WriteUnicodeCharacter ------------------------------------------------------- | |
86 | |
87 // Appends a UTF-8 character to the given 8-bit string. Returns the number of | |
88 // bytes written. | |
89 size_t WriteUnicodeCharacter(uint32 code_point, std::string* output) { | |
90 if (code_point <= 0x7f) { | |
91 // Fast path the common case of one byte. | |
92 output->push_back(code_point); | |
93 return 1; | |
94 } | |
95 | |
96 // CBU8_APPEND_UNSAFE can append up to 4 bytes. | |
97 size_t char_offset = output->length(); | |
98 size_t original_char_offset = char_offset; | |
99 output->resize(char_offset + CBU8_MAX_LENGTH); | |
100 | |
101 CBU8_APPEND_UNSAFE(&(*output)[0], char_offset, code_point); | |
102 | |
103 // CBU8_APPEND_UNSAFE will advance our pointer past the inserted character, so | |
104 // it will represent the new length of the string. | |
105 output->resize(char_offset); | |
106 return char_offset - original_char_offset; | |
107 } | |
108 | |
109 // Appends the given code point as a UTF-16 character to the given 16-bit | |
110 // string. Returns the number of 16-bit values written. | |
111 size_t WriteUnicodeCharacter(uint32 code_point, string16* output) { | |
112 if (CBU16_LENGTH(code_point) == 1) { | |
113 // Thie code point is in the Basic Multilingual Plane (BMP). | |
114 output->push_back(static_cast<char16>(code_point)); | |
115 return 1; | |
116 } | |
117 // Non-BMP characters use a double-character encoding. | |
118 size_t char_offset = output->length(); | |
119 output->resize(char_offset + CBU16_MAX_LENGTH); | |
120 CBU16_APPEND_UNSAFE(&(*output)[0], char_offset, code_point); | |
121 return CBU16_MAX_LENGTH; | |
122 } | |
123 | |
124 #if defined(WCHAR_T_IS_UTF32) | |
125 // Appends the given UTF-32 character to the given 32-bit string. Returns the | |
126 // number of 32-bit values written. | |
127 inline size_t WriteUnicodeCharacter(uint32 code_point, std::wstring* output) { | |
128 // This is the easy case, just append the character. | |
129 output->push_back(code_point); | |
130 return 1; | |
131 } | |
132 #endif // defined(WCHAR_T_IS_UTF32) | |
133 | 13 |
134 // Generalized Unicode converter ----------------------------------------------- | 14 // Generalized Unicode converter ----------------------------------------------- |
135 | 15 |
136 // Converts the given source Unicode character type to the given destination | 16 // Converts the given source Unicode character type to the given destination |
137 // Unicode character type as a STL string. The given input buffer and size | 17 // Unicode character type as a STL string. The given input buffer and size |
138 // determine the source, and the given output STL string will be replaced by | 18 // determine the source, and the given output STL string will be replaced by |
139 // the result. | 19 // the result. |
140 template<typename SRC_CHAR, typename DEST_STRING> | 20 template<typename SRC_CHAR> |
141 bool ConvertUnicode(const SRC_CHAR* src, | 21 bool ConvertUnicode(const SRC_CHAR* src, |
142 size_t src_len, | 22 size_t src_len, |
143 DEST_STRING* output, | 23 std::wstring* output, |
144 size_t* offset_for_adjustment) { | 24 size_t* offset_for_adjustment) { |
145 size_t output_offset = | 25 size_t output_offset = |
146 (offset_for_adjustment && *offset_for_adjustment < src_len) ? | 26 (offset_for_adjustment && *offset_for_adjustment < src_len) ? |
147 *offset_for_adjustment : DEST_STRING::npos; | 27 *offset_for_adjustment : std::wstring::npos; |
148 | 28 |
149 // ICU requires 32-bit numbers. | 29 // ICU requires 32-bit numbers. |
150 bool success = true; | 30 bool success = true; |
151 int32 src_len32 = static_cast<int32>(src_len); | 31 int32 src_len32 = static_cast<int32>(src_len); |
152 for (int32 i = 0; i < src_len32; i++) { | 32 for (int32 i = 0; i < src_len32; i++) { |
153 uint32 code_point; | 33 uint32 code_point; |
154 size_t original_i = i; | 34 size_t original_i = i; |
155 size_t chars_written = 0; | 35 size_t chars_written = 0; |
156 if (ReadUnicodeCharacter(src, src_len32, &i, &code_point)) { | 36 if (ReadUnicodeCharacter(src, src_len32, &i, &code_point)) { |
157 chars_written = WriteUnicodeCharacter(code_point, output); | 37 chars_written = WriteUnicodeCharacter(code_point, output); |
158 } else { | 38 } else { |
159 // TODO(jungshik): consider adding 'Replacement character' (U+FFFD) | 39 // TODO(jungshik): consider adding 'Replacement character' (U+FFFD) |
160 // in place of an invalid codepoint. | 40 // in place of an invalid codepoint. |
161 success = false; | 41 success = false; |
162 } | 42 } |
163 if ((output_offset != DEST_STRING::npos) && | 43 if ((output_offset != std::wstring::npos) && |
164 (*offset_for_adjustment > original_i)) { | 44 (*offset_for_adjustment > original_i)) { |
165 // NOTE: ReadUnicodeCharacter() adjusts |i| to point _at_ the last | 45 // NOTE: ReadUnicodeCharacter() adjusts |i| to point _at_ the last |
166 // character read, not after it (so that incrementing it in the loop | 46 // character read, not after it (so that incrementing it in the loop |
167 // increment will place it at the right location), so we need to account | 47 // increment will place it at the right location), so we need to account |
168 // for that in determining the amount that was read. | 48 // for that in determining the amount that was read. |
169 if (*offset_for_adjustment <= static_cast<size_t>(i)) | 49 if (*offset_for_adjustment <= static_cast<size_t>(i)) |
170 output_offset = DEST_STRING::npos; | 50 output_offset = std::wstring::npos; |
171 else | 51 else |
172 output_offset += chars_written - (i - original_i + 1); | 52 output_offset += chars_written - (i - original_i + 1); |
173 } | 53 } |
174 } | 54 } |
175 | 55 |
176 if (offset_for_adjustment) | 56 if (offset_for_adjustment) |
177 *offset_for_adjustment = output_offset; | 57 *offset_for_adjustment = output_offset; |
178 return success; | 58 return success; |
179 } | 59 } |
180 | 60 |
181 // Guesses the length of the output in UTF-8 in bytes, clears that output | |
182 // string, and reserves that amount of space. We assume that the input | |
183 // character types are unsigned, which will be true for UTF-16 and -32 on our | |
184 // systems. | |
185 template<typename CHAR> | |
186 void PrepareForUTF8Output(const CHAR* src, | |
187 size_t src_len, | |
188 std::string* output) { | |
189 output->clear(); | |
190 if (src_len == 0) | |
191 return; | |
192 if (src[0] < 0x80) { | |
193 // Assume that the entire input will be ASCII. | |
194 output->reserve(src_len); | |
195 } else { | |
196 // Assume that the entire input is non-ASCII and will have 3 bytes per char. | |
197 output->reserve(src_len * 3); | |
198 } | |
199 } | |
200 | |
201 // Prepares an output buffer (containing either UTF-16 or -32 data) given some | |
202 // UTF-8 input that will be converted to it. See PrepareForUTF8Output(). | |
203 template<typename STRING> | |
204 void PrepareForUTF16Or32Output(const char* src, | |
205 size_t src_len, | |
206 STRING* output) { | |
207 output->clear(); | |
208 if (src_len == 0) | |
209 return; | |
210 if (static_cast<unsigned char>(src[0]) < 0x80) { | |
211 // Assume the input is all ASCII, which means 1:1 correspondence. | |
212 output->reserve(src_len); | |
213 } else { | |
214 // Otherwise assume that the UTF-8 sequences will have 2 bytes for each | |
215 // character. | |
216 output->reserve(src_len / 2); | |
217 } | |
218 } | |
219 | |
220 } // namespace | |
221 | |
222 // UTF-8 <-> Wide -------------------------------------------------------------- | 61 // UTF-8 <-> Wide -------------------------------------------------------------- |
223 | 62 |
224 bool WideToUTF8(const wchar_t* src, size_t src_len, std::string* output) { | |
225 PrepareForUTF8Output(src, src_len, output); | |
226 return ConvertUnicode<wchar_t, std::string>(src, src_len, output, NULL); | |
227 } | |
228 | |
229 std::string WideToUTF8(const std::wstring& wide) { | |
230 std::string ret; | |
231 // Ignore the success flag of this call, it will do the best it can for | |
232 // invalid input, which is what we want here. | |
233 WideToUTF8(wide.data(), wide.length(), &ret); | |
234 return ret; | |
235 } | |
236 | |
237 bool UTF8ToWideAndAdjustOffset(const char* src, | 63 bool UTF8ToWideAndAdjustOffset(const char* src, |
238 size_t src_len, | 64 size_t src_len, |
239 std::wstring* output, | 65 std::wstring* output, |
240 size_t* offset_for_adjustment) { | 66 size_t* offset_for_adjustment) { |
241 PrepareForUTF16Or32Output(src, src_len, output); | 67 PrepareForUTF16Or32Output(src, src_len, output); |
242 return ConvertUnicode<char, std::wstring>(src, src_len, output, | 68 return ConvertUnicode(src, src_len, output, offset_for_adjustment); |
243 offset_for_adjustment); | |
244 } | 69 } |
245 | 70 |
246 std::wstring UTF8ToWideAndAdjustOffset(const base::StringPiece& utf8, | 71 std::wstring UTF8ToWideAndAdjustOffset(const base::StringPiece& utf8, |
247 size_t* offset_for_adjustment) { | 72 size_t* offset_for_adjustment) { |
248 std::wstring ret; | 73 std::wstring ret; |
249 UTF8ToWideAndAdjustOffset(utf8.data(), utf8.length(), &ret, | 74 UTF8ToWideAndAdjustOffset(utf8.data(), utf8.length(), &ret, |
250 offset_for_adjustment); | 75 offset_for_adjustment); |
251 return ret; | 76 return ret; |
252 } | 77 } |
253 | 78 |
254 // UTF-16 <-> Wide ------------------------------------------------------------- | 79 // UTF-16 <-> Wide ------------------------------------------------------------- |
255 | 80 |
256 #if defined(WCHAR_T_IS_UTF16) | 81 #if defined(WCHAR_T_IS_UTF16) |
257 | 82 |
258 // When wide == UTF-16, then conversions are a NOP. | 83 // When wide == UTF-16, then conversions are a NOP. |
259 bool WideToUTF16(const wchar_t* src, size_t src_len, string16* output) { | |
260 output->assign(src, src_len); | |
261 return true; | |
262 } | |
263 | |
264 string16 WideToUTF16(const std::wstring& wide) { | |
265 return wide; | |
266 } | |
267 | |
268 bool UTF16ToWideAndAdjustOffset(const char16* src, | 84 bool UTF16ToWideAndAdjustOffset(const char16* src, |
269 size_t src_len, | 85 size_t src_len, |
270 std::wstring* output, | 86 std::wstring* output, |
271 size_t* offset_for_adjustment) { | 87 size_t* offset_for_adjustment) { |
272 output->assign(src, src_len); | 88 output->assign(src, src_len); |
273 if (offset_for_adjustment && (*offset_for_adjustment >= src_len)) | 89 if (offset_for_adjustment && (*offset_for_adjustment >= src_len)) |
274 *offset_for_adjustment = std::wstring::npos; | 90 *offset_for_adjustment = std::wstring::npos; |
275 return true; | 91 return true; |
276 } | 92 } |
277 | 93 |
278 std::wstring UTF16ToWideAndAdjustOffset(const string16& utf16, | 94 std::wstring UTF16ToWideAndAdjustOffset(const string16& utf16, |
279 size_t* offset_for_adjustment) { | 95 size_t* offset_for_adjustment) { |
280 if (offset_for_adjustment && (*offset_for_adjustment >= utf16.length())) | 96 if (offset_for_adjustment && (*offset_for_adjustment >= utf16.length())) |
281 *offset_for_adjustment = std::wstring::npos; | 97 *offset_for_adjustment = std::wstring::npos; |
282 return utf16; | 98 return utf16; |
283 } | 99 } |
284 | 100 |
285 #elif defined(WCHAR_T_IS_UTF32) | 101 #elif defined(WCHAR_T_IS_UTF32) |
286 | 102 |
287 bool WideToUTF16(const wchar_t* src, size_t src_len, string16* output) { | |
288 output->clear(); | |
289 // Assume that normally we won't have any non-BMP characters so the counts | |
290 // will be the same. | |
291 output->reserve(src_len); | |
292 return ConvertUnicode<wchar_t, string16>(src, src_len, output, NULL); | |
293 } | |
294 | |
295 string16 WideToUTF16(const std::wstring& wide) { | |
296 string16 ret; | |
297 WideToUTF16(wide.data(), wide.length(), &ret); | |
298 return ret; | |
299 } | |
300 | |
301 bool UTF16ToWideAndAdjustOffset(const char16* src, | 103 bool UTF16ToWideAndAdjustOffset(const char16* src, |
302 size_t src_len, | 104 size_t src_len, |
303 std::wstring* output, | 105 std::wstring* output, |
304 size_t* offset_for_adjustment) { | 106 size_t* offset_for_adjustment) { |
305 output->clear(); | 107 output->clear(); |
306 // Assume that normally we won't have any non-BMP characters so the counts | 108 // Assume that normally we won't have any non-BMP characters so the counts |
307 // will be the same. | 109 // will be the same. |
308 output->reserve(src_len); | 110 output->reserve(src_len); |
309 return ConvertUnicode<char16, std::wstring>(src, src_len, output, | 111 return ConvertUnicode(src, src_len, output, offset_for_adjustment); |
310 offset_for_adjustment); | |
311 } | 112 } |
312 | 113 |
313 std::wstring UTF16ToWideAndAdjustOffset(const string16& utf16, | 114 std::wstring UTF16ToWideAndAdjustOffset(const string16& utf16, |
314 size_t* offset_for_adjustment) { | 115 size_t* offset_for_adjustment) { |
315 std::wstring ret; | 116 std::wstring ret; |
316 UTF16ToWideAndAdjustOffset(utf16.data(), utf16.length(), &ret, | 117 UTF16ToWideAndAdjustOffset(utf16.data(), utf16.length(), &ret, |
317 offset_for_adjustment); | 118 offset_for_adjustment); |
318 return ret; | 119 return ret; |
319 } | 120 } |
320 | 121 |
321 #endif // defined(WCHAR_T_IS_UTF32) | 122 #endif // defined(WCHAR_T_IS_UTF32) |
322 | |
323 // UTF16 <-> UTF8 -------------------------------------------------------------- | |
324 | |
325 #if defined(WCHAR_T_IS_UTF32) | |
326 | |
327 bool UTF8ToUTF16(const char* src, size_t src_len, string16* output) { | |
328 PrepareForUTF16Or32Output(src, src_len, output); | |
329 return ConvertUnicode<char, string16>(src, src_len, output, NULL); | |
330 } | |
331 | |
332 string16 UTF8ToUTF16(const std::string& utf8) { | |
333 string16 ret; | |
334 // Ignore the success flag of this call, it will do the best it can for | |
335 // invalid input, which is what we want here. | |
336 UTF8ToUTF16(utf8.data(), utf8.length(), &ret); | |
337 return ret; | |
338 } | |
339 | |
340 bool UTF16ToUTF8(const char16* src, size_t src_len, std::string* output) { | |
341 PrepareForUTF8Output(src, src_len, output); | |
342 return ConvertUnicode<char16, std::string>(src, src_len, output, NULL); | |
343 } | |
344 | |
345 std::string UTF16ToUTF8(const string16& utf16) { | |
346 std::string ret; | |
347 // Ignore the success flag of this call, it will do the best it can for | |
348 // invalid input, which is what we want here. | |
349 UTF16ToUTF8(utf16.data(), utf16.length(), &ret); | |
350 return ret; | |
351 } | |
352 | |
353 #elif defined(WCHAR_T_IS_UTF16) | |
354 // Easy case since we can use the "wide" versions we already wrote above. | |
355 | |
356 bool UTF8ToUTF16(const char* src, size_t src_len, string16* output) { | |
357 return UTF8ToWide(src, src_len, output); | |
358 } | |
359 | |
360 string16 UTF8ToUTF16(const std::string& utf8) { | |
361 return UTF8ToWide(utf8); | |
362 } | |
363 | |
364 bool UTF16ToUTF8(const char16* src, size_t src_len, std::string* output) { | |
365 return WideToUTF8(src, src_len, output); | |
366 } | |
367 | |
368 std::string UTF16ToUTF8(const string16& utf16) { | |
369 return WideToUTF8(utf16); | |
370 } | |
371 | |
372 #endif | |
OLD | NEW |