String:WriteUtf8: Add REPLACE_INVALID_UTF8 option

src/unicode-inl.h

 // Copyright 2007-2010 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 89 matching lines @@
   static const int kMask = ~(1 << 6);
   if (c <= kMaxOneByteChar) {
     str[0] = c;
     return 1;
   }
   str[0] = 0xC0 | (c >> 6);
   str[1] = 0x80 | (c & kMask);
   return 2;
 }
 
+// Encode encodes the unicode code point c into the given str buffer. Unless
+// allow_invalid is set to true, surrogate code points will be replaced with
+// kReplacementCharacter. The caller is required to combine surrogate pairs
+// into code points before calling Encode.
+unsigned Utf8::Encode(char* str, uchar c, bool allow_invalid) {
+  if (!allow_invalid &&

[dcarney, 2014/01/07 10:12:16]

move this block down into the kMaxThreeByteChar clause.  it's in the way of the
common case

+      (Utf16::IsLeadSurrogate(c) ||
+      Utf16::IsTrailSurrogate(c))) {
+    c = kReplacementCharacter;
+  }
 
-unsigned Utf8::Encode(char* str, uchar c, int previous) {
   static const int kMask = ~(1 << 6);
   if (c <= kMaxOneByteChar) {
     str[0] = c;
     return 1;
   } else if (c <= kMaxTwoByteChar) {
     str[0] = 0xC0 | (c >> 6);
     str[1] = 0x80 | (c & kMask);
     return 2;
   } else if (c <= kMaxThreeByteChar) {
-    if (Utf16::IsTrailSurrogate(c) &&
-        Utf16::IsLeadSurrogate(previous)) {
-      const int kUnmatchedSize = kSizeOfUnmatchedSurrogate;
-      return Encode(str - kUnmatchedSize,
-                    Utf16::CombineSurrogatePair(previous, c),
-                    Utf16::kNoPreviousCharacter) - kUnmatchedSize;
-    }
     str[0] = 0xE0 | (c >> 12);
     str[1] = 0x80 | ((c >> 6) & kMask);
     str[2] = 0x80 | (c & kMask);
     return 3;
   } else {
     str[0] = 0xF0 | (c >> 18);
     str[1] = 0x80 | ((c >> 12) & kMask);
     str[2] = 0x80 | ((c >> 6) & kMask);
     str[3] = 0x80 | (c & kMask);
     return 4;
   }
 }
 
 
 uchar Utf8::ValueOf(const byte* bytes, unsigned length, unsigned* cursor) {
   if (length <= 0) return kBadChar;
   byte first = bytes[0];
   // Characters between 0000 and 0007F are encoded as a single character
   if (first <= kMaxOneByteChar) {
     *cursor += 1;
     return first;
   }
   return CalculateValue(bytes, length, cursor);
 }
 
+// @TODO give this the same semantics as Encode?

[dcarney, 2014/01/07 11:05:50]

i don't see an easy way to do this.  You'd have to then run over the byte stream
with a second pass to adjust for matching surrogates, but the current
implementation here already doing that correctly and efficiently for the case
where the replacement character is inserted and the case that it isn't.

I think you'll ultimately need to keep the rewriting above in Utf8::Encode as it
is instead of moving it to WritePair in api.cc.  Instead you can just write
kReplacementCharacter if the flag is set.  This makes the Length and Encode
functions easier to understand, since they complement one another and are in the
same place.  Also, you won't have to change any other call sites of Encode,
which is of great benefit.

 unsigned Utf8::Length(uchar c, int previous) {
   if (c <= kMaxOneByteChar) {
     return 1;
   } else if (c <= kMaxTwoByteChar) {
     return 2;
   } else if (c <= kMaxThreeByteChar) {
     if (Utf16::IsTrailSurrogate(c) &&
         Utf16::IsLeadSurrogate(previous)) {
       return kSizeOfUnmatchedSurrogate - kBytesSavedByCombiningSurrogates;
     }
@@ skipping 46 matching lines @@
   // Copy the rest the slow way.
   WriteUtf16Slow(unbuffered_start_,
                  data + buffer_length,
                  length - buffer_length);
   return length;
 }
 
 }  // namespace unibrow
 
 #endif  // V8_UNICODE_INL_H_