Pass through non-character codepoints in UTF-8,16,32 and Wide conversion func...

base/string_util.h

 // Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
 // This file defines utility functions for working with strings.
 
 #ifndef BASE_STRING_UTIL_H_
 #define BASE_STRING_UTIL_H_
 
 #include <stdarg.h>   // va_list
@@ skipping 168 matching lines @@
 std::wstring ASCIIToWide(const StringPiece& ascii);
 std::string UTF16ToASCII(const string16& utf16);
 string16 ASCIIToUTF16(const StringPiece& ascii);
 
 // These convert between UTF-8, -16, and -32 strings. They are potentially slow,
 // so avoid unnecessary conversions. The low-level versions return a boolean
 // indicating whether the conversion was 100% valid. In this case, it will still
 // do the best it can and put the result in the output buffer. The versions that
 // return strings ignore this error and just return the best conversion
 // possible.
+//
+// Note that only the structural validity is checked and non-character
+// codepoints and unassigned are regarded as valid.
+// TODO(jungshik): Consider replacing an invalid input sequence with
+// the Unicode replacement character or adding |replacement_char| parameter.
+// Currently, it's skipped in the ouput, which could be problematic in
+// some situations.
 bool WideToUTF8(const wchar_t* src, size_t src_len, std::string* output);
 std::string WideToUTF8(const std::wstring& wide);
 bool UTF8ToWide(const char* src, size_t src_len, std::wstring* output);
 std::wstring UTF8ToWide(const StringPiece& utf8);
 
 bool WideToUTF16(const wchar_t* src, size_t src_len, string16* output);
 string16 WideToUTF16(const std::wstring& wide);
 bool UTF16ToWide(const char16* src, size_t src_len, std::wstring* output);
 std::wstring UTF16ToWide(const string16& utf16);
 
@@ skipping 44 matching lines @@
                     std::wstring* wide);
 
 // Converts the given wide string to the corresponding Latin1. This will fail
 // (return false) if any characters are more than 255.
 bool WideToLatin1(const std::wstring& wide, std::string* latin1);
 
 // Returns true if the specified string matches the criteria. How can a wide
 // string be 8-bit or UTF8? It contains only characters that are < 256 (in the
 // first case) or characters that use only 8-bits and whose 8-bit
 // representation looks like a UTF-8 string (the second case).
+//
+// Note that IsStringUTF8 checks not only if the input is structrually
+// valid but also if it doesn't contain any non-character codepoint
+// (e.g. U+FFFE). It's done on purpose because all the existing callers want
+// to have the maximum 'discriminating' power from other encodings. If
+// there's a use case for just checking the structural validity, we have to
+// add a new function for that.
 bool IsString8Bit(const std::wstring& str);
 bool IsStringUTF8(const std::string& str);
 bool IsStringWideUTF8(const std::wstring& str);
 bool IsStringASCII(const std::wstring& str);
 bool IsStringASCII(const StringPiece& str);
 bool IsStringASCII(const string16& str);
 
 // ASCII-specific tolower.  The standard library's tolower is locale sensitive,
 // so we don't want to use it here.
 template <class Char> inline Char ToLowerASCII(Char c) {
@@ skipping 358 matching lines @@
 #elif defined(WCHAR_T_IS_UTF32)
   typedef uint32 Unsigned;
 #endif
 };
 template<>
 struct ToUnsigned<short> {
   typedef unsigned short Unsigned;
 };
 
 #endif  // BASE_STRING_UTIL_H_