String:WriteUtf8: Add REPLACE_INVALID_UTF8 option

src/unicode-inl.h

Index: src/unicode-inl.h
diff --git a/src/unicode-inl.h b/src/unicode-inl.h
index f861f9f2d47449945d62a6fbc8044abbcd0b2a2b..1efe782958e534b03dff8b18f0d00f6922450739 100644
--- a/src/unicode-inl.h
+++ b/src/unicode-inl.h
@@ -107,8 +107,17 @@ unsigned Utf8::EncodeOneByte(char* str, uint8_t c) {
   return 2;
 }
 
+// Encode encodes the unicode code point c into the given str buffer. Unless
+// allow_invalid is set to true, surrogate code points will be replaced with
+// kReplacementCharacter. The caller is required to combine surrogate pairs
+// into code points before calling Encode.
+unsigned Utf8::Encode(char* str, uchar c, bool allow_invalid) {

[dcarney, 2014/01/04 15:56:45]

the name of the third argument is not the same as the declaration.

[haimuiba, 2014/01/06 05:40:18]

Thx, will fix.

+  if (!allow_invalid &&
+      Utf16::IsLeadSurrogate(c) ||
+      Utf16::IsTrailSurrogate(c)) {
+    c = kReplacementCharacter;
+  }
 
-unsigned Utf8::Encode(char* str, uchar c, int previous) {
   static const int kMask = ~(1 << 6);
   if (c <= kMaxOneByteChar) {
     str[0] = c;
@@ -118,13 +127,6 @@ unsigned Utf8::Encode(char* str, uchar c, int previous) {
     str[1] = 0x80 | (c & kMask);
     return 2;
   } else if (c <= kMaxThreeByteChar) {

[dcarney, 2014/01/04 15:56:45]

can't chop this section out - need backwards compatibility for the other call
sites of Encode

[haimuiba, 2014/01/06 05:40:18]

I'd prefer changing the other call sites if needed. I'll look into this today.
But yeah, if that can't be done for some reason, I'll put this code back.

-    if (Utf16::IsTrailSurrogate(c) &&
-        Utf16::IsLeadSurrogate(previous)) {
-      const int kUnmatchedSize = kSizeOfUnmatchedSurrogate;
-      return Encode(str - kUnmatchedSize,
-                    Utf16::CombineSurrogatePair(previous, c),
-                    Utf16::kNoPreviousCharacter) - kUnmatchedSize;
-    }
     str[0] = 0xE0 | (c >> 12);
     str[1] = 0x80 | ((c >> 6) & kMask);
     str[2] = 0x80 | (c & kMask);