String:WriteUtf8: Add REPLACE_INVALID_UTF8 option

src/api.cc

Index: src/api.cc
diff --git a/src/api.cc b/src/api.cc
index 9a68f639efec56559c0aa7ffe76a58ada5776fc4..9f67ea9fcbce5fd549053b57c648d00d89ee98ab 100644
--- a/src/api.cc
+++ b/src/api.cc
@@ -4504,28 +4504,35 @@ int String::Utf8Length() const {
 class Utf8WriterVisitor {
  public:
   Utf8WriterVisitor(
-      char* buffer, int capacity, bool skip_capacity_check)
+      char* buffer,
+      int capacity,
+      bool skip_capacity_check,
+      bool allow_invalid_utf8)
     : early_termination_(false),
       last_character_(unibrow::Utf16::kNoPreviousCharacter),
       buffer_(buffer),
       start_(buffer),
       capacity_(capacity),
       skip_capacity_check_(capacity == -1 || skip_capacity_check),
+      allow_invalid_utf8_(allow_invalid_utf8),
       utf16_chars_read_(0) {
   }
 
   static int WriteEndCharacter(uint16_t character,
                                int last_character,
                                int remaining,
-                               char* const buffer) {
+                               char* const buffer,
+                               bool allow_invalid_utf8) {
     using namespace unibrow;
     ASSERT(remaining > 0);
     // We can't use a local buffer here because Encode needs to modify
     // previous characters in the stream.  We know, however, that
     // exactly one character will be advanced.
-    if (Utf16::IsTrailSurrogate(character) &&
-        Utf16::IsLeadSurrogate(last_character)) {
-      int written = Utf8::Encode(buffer, character, last_character);
+    if (Utf16::IsSurrogatePair(last_character, character)) {
+      int written = Utf8::Encode(buffer,
+                                 character,
+                                 last_character,
+                                 allow_invalid_utf8);
       ASSERT(written == 1);
       return written;
     }
@@ -4534,7 +4541,8 @@ class Utf8WriterVisitor {
     // Can't encode using last_character as gcc has array bounds issues.
     int written = Utf8::Encode(temp_buffer,
                                character,
-                               Utf16::kNoPreviousCharacter);
+                               Utf16::kNoPreviousCharacter,
+                               allow_invalid_utf8);
     // Won't fit.
     if (written > remaining) return 0;
     // Copy over the character from temp_buffer.
@@ -4581,7 +4589,10 @@ class Utf8WriterVisitor {
       } else {
         for (; i < fast_length; i++) {
           uint16_t character = *chars++;
-          buffer += Utf8::Encode(buffer, character, last_character);
+          buffer += Utf8::Encode(buffer,
+                                 character,
+                                 last_character,
+                                 allow_invalid_utf8_);
           last_character = character;
           ASSERT(capacity_ == -1 || (buffer - start_) <= capacity_);
         }
@@ -4604,7 +4615,8 @@ class Utf8WriterVisitor {
       int written = WriteEndCharacter(character,
                                       last_character,
                                       remaining_capacity,
-                                      buffer);
+                                      buffer,
+                                      allow_invalid_utf8_);
       if (written == 0) {
         early_termination_ = true;
         break;
@@ -4652,6 +4664,7 @@ class Utf8WriterVisitor {
   char* const start_;
   int capacity_;
   bool const skip_capacity_check_;
+  bool const allow_invalid_utf8_;
   int utf16_chars_read_;
   DISALLOW_IMPLICIT_CONSTRUCTORS(Utf8WriterVisitor);
 };
@@ -4690,9 +4703,16 @@ int String::WriteUtf8(char* buffer,
   }
   const int string_length = str->length();
   bool write_null = !(options & NO_NULL_TERMINATION);
+  bool allow_invalid_utf8 = !(options & DISALLOW_INVALID_UTF8);
   // First check if we can just write the string without checking capacity.
+  // @TODO Replace magic number 3 with something more descriptive. E.g.

[dcarney, 2014/01/10 16:54:33]

the syntax we use is:

// TODO(username) Comment.

but in this case, it just needs to be done before landing...

[haimuiba, 2014/01/13 07:48:21]

Done.

+  // Utf8::kMaxTwoByteSize (as in the maximum size an unsighed 2 byte code unit
+  // value will take up when encoded to UTF-8)? When I first read this code I
+  // thought there might be a overflow bug here since UTF-8 may take up to 4
+  // bytes per code unit. Then I realized that a surrogate pair has a
+  // str.length of 2, making the code correct.
   if (capacity == -1 || capacity / 3 >= string_length) {
-    Utf8WriterVisitor writer(buffer, capacity, true);
+    Utf8WriterVisitor writer(buffer, capacity, true, allow_invalid_utf8);
     const int kMaxRecursion = 100;
     bool success = RecursivelySerializeToUtf8(*str, &writer, kMaxRecursion);
     if (success) return writer.CompleteWrite(write_null, nchars_ref);
@@ -4720,7 +4740,7 @@ int String::WriteUtf8(char* buffer,
   }
   // Recursive slow path can potentially be unreasonable slow. Flatten.
   str = FlattenGetString(str);
-  Utf8WriterVisitor writer(buffer, capacity, false);
+  Utf8WriterVisitor writer(buffer, capacity, false, allow_invalid_utf8);
   i::String::VisitFlat(&writer, *str);
   return writer.CompleteWrite(write_null, nchars_ref);

[dcarney, 2014/01/10 16:49:55]

think you need to ensure that all return points from this function check for the
case that you  don't allow invalid utf8, and the last character written is one
half of a surrogate pair, which would be incorrectly written as a replacement
character in the current implementation

[haimuiba, 2014/01/13 07:48:21]

Not sure I understand. If allow_invalid_utf8=false, then any lone surrogate is
an invalid last character, and should be replaced.

 }