If we can't read a unicode character, write the standard "unknown" (0xFFFD) c...

base/utf_string_conversions_unittest.cc

 // Copyright (c) 2009 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/basictypes.h"
 #include "base/string_util.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 namespace base {
 
@@ skipping 76 matching lines @@
   struct UTF8ToWideCase {
     const char* utf8;
     const wchar_t* wide;
     bool success;
   } convert_cases[] = {
     // Regular UTF-8 input.
     {"\xe4\xbd\xa0\xe5\xa5\xbd", L"\x4f60\x597d", true},
     // Non-character is passed through.
     {"\xef\xbf\xbfHello", L"\xffffHello", true},
     // Truncated UTF-8 sequence.
-    {"\xe4\xa0\xe5\xa5\xbd", L"\x597d", false},
+    {"\xe4\xa0\xe5\xa5\xbd", L"\xfffd\x597d", false},
     // Truncated off the end.
-    {"\xe5\xa5\xbd\xe4\xa0", L"\x597d", false},
+    {"\xe5\xa5\xbd\xe4\xa0", L"\x597d\xfffd", false},
     // Non-shortest-form UTF-8.
-    {"\xf0\x84\xbd\xa0\xe5\xa5\xbd", L"\x597d", false},
+    {"\xf0\x84\xbd\xa0\xe5\xa5\xbd", L"\xfffd\x597d", false},
     // This UTF-8 character decodes to a UTF-16 surrogate, which is illegal.
-    {"\xed\xb0\x80", L"", false},
+    {"\xed\xb0\x80", L"\xfffd", false},
     // Non-BMP characters. The second is a non-character regarded as valid.
     // The result will either be in UTF-16 or UTF-32.
 #if defined(WCHAR_T_IS_UTF16)
     {"A\xF0\x90\x8C\x80z", L"A\xd800\xdf00z", true},
     {"A\xF4\x8F\xBF\xBEz", L"A\xdbff\xdffez", true},
 #elif defined(WCHAR_T_IS_UTF32)
     {"A\xF0\x90\x8C\x80z", L"A\x10300z", true},
     {"A\xF4\x8F\xBF\xBEz", L"A\x10fffez", true},
 #endif
   };
@@ skipping 31 matching lines @@
     bool success;
   } convert_cases[] = {
     // Regular UTF-16 input.
     {L"\x4f60\x597d", "\xe4\xbd\xa0\xe5\xa5\xbd", true},
     // Test a non-BMP character.
     {L"\xd800\xdf00", "\xF0\x90\x8C\x80", true},
     // Non-characters are passed through.
     {L"\xffffHello", "\xEF\xBF\xBFHello", true},
     {L"\xdbff\xdffeHello", "\xF4\x8F\xBF\xBEHello", true},
     // The first character is a truncated UTF-16 character.
-    {L"\xd800\x597d", "\xe5\xa5\xbd", false},
+    {L"\xd800\x597d", "\xef\xbf\xbd\xe5\xa5\xbd", false},
     // Truncated at the end.
-    {L"\x597d\xd800", "\xe5\xa5\xbd", false},
+    {L"\x597d\xd800", "\xe5\xa5\xbd\xef\xbf\xbd", false},
   };
 
   for (int i = 0; i < arraysize(convert_cases); i++) {
     std::string converted;
     EXPECT_EQ(convert_cases[i].success,
               WideToUTF8(convert_cases[i].utf16,
                          wcslen(convert_cases[i].utf16),
                          &converted));
     std::string expected(convert_cases[i].utf8);
     EXPECT_EQ(expected, converted);
   }
 }
 
 #elif defined(WCHAR_T_IS_UTF32)
 // This test is only valid when wchar_t == UTF-32.
 TEST(UTFStringConversionsTest, ConvertUTF32ToUTF8) {
   struct WideToUTF8Case {
     const wchar_t* utf32;
     const char* utf8;
     bool success;
   } convert_cases[] = {
     // Regular 16-bit input.
     {L"\x4f60\x597d", "\xe4\xbd\xa0\xe5\xa5\xbd", true},
     // Test a non-BMP character.
     {L"A\x10300z", "A\xF0\x90\x8C\x80z", true},
     // Non-characters are passed through.
     {L"\xffffHello", "\xEF\xBF\xBFHello", true},
     {L"\x10fffeHello", "\xF4\x8F\xBF\xBEHello", true},
     // Invalid Unicode code points.
-    {L"\xfffffffHello", "Hello", false},
+    {L"\xfffffffHello", "\xEF\xBF\xBDHello", false},
     // The first character is a truncated UTF-16 character.
-    {L"\xd800\x597d", "\xe5\xa5\xbd", false},
-    {L"\xdc01Hello", "Hello", false},
+    {L"\xd800\x597d", "\xef\xbf\xbd\xe5\xa5\xbd", false},
+    {L"\xdc01Hello", "\xef\xbf\xbdHello", false},
   };
 
   for (size_t i = 0; i < ARRAYSIZE_UNSAFE(convert_cases); i++) {
     std::string converted;
     EXPECT_EQ(convert_cases[i].success,
               WideToUTF8(convert_cases[i].utf32,
                          wcslen(convert_cases[i].utf32),
                          &converted));
     std::string expected(convert_cases[i].utf8);
     EXPECT_EQ(expected, converted);
@@ skipping 19 matching lines @@
   EXPECT_EQ(arraysize(wmulti) - 1, wmultistring.length());
   std::string expected;
   memcpy(WriteInto(&expected, arraysize(multi)), multi, sizeof(multi));
   EXPECT_EQ(arraysize(multi) - 1, expected.length());
   const std::string& converted = WideToUTF8(wmultistring);
   EXPECT_EQ(arraysize(multi) - 1, converted.length());
   EXPECT_EQ(expected, converted);
 }
 
 }  // namaspace base