String:WriteUtf8: Add REPLACE_INVALID_UTF8 option

src/unicode-inl.h

 // Copyright 2007-2010 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 89 matching lines @@
   static const int kMask = ~(1 << 6);
   if (c <= kMaxOneByteChar) {
     str[0] = c;
     return 1;
   }
   str[0] = 0xC0 | (c >> 6);
   str[1] = 0x80 | (c & kMask);
   return 2;
 }
 
-
-unsigned Utf8::Encode(char* str, uchar c, int previous) {
+// Encode encodes the UTF-16 code units c and previous into the given str
+// buffer. Unless allow_invalid is set to true, surrogate code points will be
+// replaced with kReplacementCharacter.
+unsigned Utf8::Encode(char* str,
+                      uchar c,
+                      int previous,
+                      bool allow_invalid = true) {

[dcarney, 2014/01/13 09:19:56]

default should be in declaration

[haimuiba, 2014/01/15 10:52:34]

Done.

   static const int kMask = ~(1 << 6);
   if (c <= kMaxOneByteChar) {
     str[0] = c;
     return 1;
   } else if (c <= kMaxTwoByteChar) {
     str[0] = 0xC0 | (c >> 6);
     str[1] = 0x80 | (c & kMask);
     return 2;
   } else if (c <= kMaxThreeByteChar) {
-    if (Utf16::IsTrailSurrogate(c) &&
-        Utf16::IsLeadSurrogate(previous)) {
+    if (Utf16::IsSurrogatePair(previous, c)) {
       const int kUnmatchedSize = kSizeOfUnmatchedSurrogate;
       return Encode(str - kUnmatchedSize,
                     Utf16::CombineSurrogatePair(previous, c),
-                    Utf16::kNoPreviousCharacter) - kUnmatchedSize;
+                    Utf16::kNoPreviousCharacter,
+                    allow_invalid) - kUnmatchedSize;
+    } else if (!allow_invalid &&
+               (Utf16::IsLeadSurrogate(c) ||
+               Utf16::IsTrailSurrogate(c))) {
+      c = kBadChar;
     }
     str[0] = 0xE0 | (c >> 12);
     str[1] = 0x80 | ((c >> 6) & kMask);
     str[2] = 0x80 | (c & kMask);
     return 3;
   } else {
     str[0] = 0xF0 | (c >> 18);
     str[1] = 0x80 | ((c >> 12) & kMask);
     str[2] = 0x80 | ((c >> 6) & kMask);
     str[3] = 0x80 | (c & kMask);
@@ skipping 72 matching lines @@
   // Copy the rest the slow way.
   WriteUtf16Slow(unbuffered_start_,
                  data + buffer_length,
                  length - buffer_length);
   return length;
 }
 
 }  // namespace unibrow
 
 #endif  // V8_UNICODE_INL_H_