Use v8's utf8 <-> wide conversion for Linux sys_string_conversion.

base/sys_string_conversions_linux.cc

 // Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/sys_string_conversions.h"
 
 #include <wchar.h>
 
+#include "base/basictypes.h"
 #include "base/string_piece.h"
 #include "base/string_util.h"
 
 namespace base {
 
+// UTF8 <-> UCS-4 conversion from v8.
+namespace {
+
+const uint32_t kMaxEncodedSize   = 4;
+const uint32_t kMaxOneByteChar   = 0x7f;
+const uint32_t kMaxTwoByteChar   = 0x7ff;
+const uint32_t kMaxThreeByteChar = 0xffff;
+const uint32_t kMaxFourByteChar  = 0x1fffff;
+const uint32_t kBadChar          = 0xFFFD;
+
+size_t WriteUnicodeCharAsUTF8(char* str, uint32_t c) {

[darin (slow to review), 2009/07/14 16:51:41]

this stuff seems like it would be better broken out as a base/utf8_util.{h,cc}
file.

also, why not just use iconv here, if you need to avoid ICU--do you??  why write
all of this code?

+  static const int kMask = ~(1 << 6);
+  if (c <= kMaxOneByteChar) {
+    str[0] = c;
+    return 1;
+  } else if (c <= kMaxTwoByteChar) {
+    str[0] = 0xC0 | (c >> 6);
+    str[1] = 0x80 | (c & kMask);
+    return 2;
+  } else if (c <= kMaxThreeByteChar) {
+    str[0] = 0xE0 | (c >> 12);
+    str[1] = 0x80 | ((c >> 6) & kMask);
+    str[2] = 0x80 | (c & kMask);
+    return 3;
+  } else {
+    str[0] = 0xF0 | (c >> 18);
+    str[1] = 0x80 | ((c >> 12) & kMask);
+    str[2] = 0x80 | ((c >> 6) & kMask);
+    str[3] = 0x80 | (c & kMask);
+    return 4;
+  }
+}
+
+size_t UnicodeCharUTF8Length(uint32_t c) {
+  if (c <= kMaxOneByteChar) {
+    return 1;
+  } else if (c <= kMaxTwoByteChar) {
+    return 2;
+  } else if (c <= kMaxThreeByteChar) {
+    return 3;
+  } else {
+    return 4;
+  }
+}
+
+uint32_t UnicodeCharFromUTF8(const uint8_t* str,
+                             size_t length,
+                             size_t* cursor) {
+  if (length <= 0) return kBadChar;
+
+  uint8_t first = str[0];
+  // Characters between 0000 and 0007F are encoded as a single character
+  if (first <= kMaxOneByteChar) {
+    *cursor += 1;
+    return first;
+  }
+
+  // We only get here for non-ascii characters.
+  if (length == 1) {
+    *cursor += 1;
+    return kBadChar;
+  }
+
+  uint8_t second = str[1] ^ 0x80;
+  if (second & 0xC0) {
+    *cursor += 1;
+    return kBadChar;
+  }
+  if (first < 0xE0) {
+    if (first < 0xC0) {
+      *cursor += 1;
+      return kBadChar;
+    }
+    uint32_t l = ((first << 6) | second) & kMaxTwoByteChar;
+    if (l <= kMaxOneByteChar) {
+      *cursor += 1;
+      return kBadChar;
+    }
+    *cursor += 2;
+    return l;
+  }
+  if (length == 2) {
+    *cursor += 1;
+    return kBadChar;
+  }
+  uint8_t third = str[2] ^ 0x80;
+  if (third & 0xC0) {
+    *cursor += 1;
+    return kBadChar;
+  }
+  if (first < 0xF0) {
+    uint32_t l = ((((first << 6) | second) << 6) | third) & kMaxThreeByteChar;
+    if (l <= kMaxTwoByteChar) {
+      *cursor += 1;
+      return kBadChar;
+    }
+    *cursor += 3;
+    return l;
+  }
+  if (length == 3) {
+    *cursor += 1;
+    return kBadChar;
+  }
+  uint8_t fourth = str[3] ^ 0x80;
+  if (fourth & 0xC0) {
+    *cursor += 1;
+    return kBadChar;
+  }
+  if (first < 0xF8) {
+    uint32_t l = (((((first << 6 | second) << 6) | third) << 6) | fourth) &
+                 kMaxFourByteChar;
+    if (l <= kMaxThreeByteChar) {
+      *cursor += 1;
+      return kBadChar;
+    }
+    *cursor += 4;
+    return l;
+  }
+  *cursor += 1;
+  return kBadChar;
+}
+
+}  // namespace
+
 std::string SysWideToUTF8(const std::wstring& wide) {
-  // In theory this should be using the system-provided conversion rather
-  // than our ICU, but this will do for now.
-  return WideToUTF8(wide);
-}
-std::wstring SysUTF8ToWide(const StringPiece& utf8) {
-  // In theory this should be using the system-provided conversion rather
-  // than our ICU, but this will do for now.
-  std::wstring out;
-  UTF8ToWide(utf8.data(), utf8.size(), &out);
+  size_t length = 0;
+  for (size_t i = 0; i < wide.size(); ++i)
+    length += UnicodeCharUTF8Length(wide[i]);
+
+  std::string out(length, 0);
+  size_t out_pos = 0;
+  for (size_t i = 0; i < wide.size(); ++i)
+    out_pos += WriteUnicodeCharAsUTF8(&out[out_pos], wide[i]);
+
   return out;
 }
 
+std::wstring SysUTF8ToWide(const StringPiece& utf8) {
+  size_t wide_length = 0;
+  for (size_t pos = 0; pos < utf8.size(); ++wide_length) {
+    if (UnicodeCharFromUTF8(
+        reinterpret_cast<const uint8_t*>(utf8.data() + pos),
+        utf8.size() - pos, &pos) == kBadChar) {
+      return std::wstring();  // Failure, invalid conversion.
+    }
+  }
+
+  std::wstring out(wide_length, 0);
+  for (size_t pos = 0, wide_pos = 0; pos < utf8.size(); ++wide_pos) {
+    out[wide_pos] = UnicodeCharFromUTF8(
+        reinterpret_cast<const uint8_t*>(utf8.data() + pos),
+        utf8.size() - pos, &pos);
+  }
+
+  return out;
+}
+
 std::string SysWideToNativeMB(const std::wstring& wide) {
   mbstate_t ps;
 
   // Calculate the number of multi-byte characters.  We walk through the string
   // without writing the output, counting the number of multi-byte characters.
   size_t num_out_chars = 0;
   memset(&ps, 0, sizeof(ps));
   for (size_t i = 0; i < wide.size(); ++i) {
     const wchar_t src = wide[i];
     // Use a temp buffer since calling wcrtomb with an output of NULL does not
@@ skipping 98 matching lines @@
       default:
         i += res;
         break;
     }
   }
 
   return out;
 }
 
 }  // namespace base