String:WriteUtf8: Add REPLACE_INVALID_UTF8 option

src/api.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 4496 matching lines @@
       char* buffer, int capacity, bool skip_capacity_check)
     : early_termination_(false),
       last_character_(unibrow::Utf16::kNoPreviousCharacter),
       buffer_(buffer),
       start_(buffer),
       capacity_(capacity),
       skip_capacity_check_(capacity == -1 || skip_capacity_check),
       utf16_chars_read_(0) {
   }
 
-  static int WriteEndCharacter(uint16_t character,
-                               int last_character,
+  // WritePair writes the current UTF-16 code unit to the given buffer. The
+  // function will go back inside the buffer to combine surrogate pairs.
+  // @TODO use uint16_t for previous?

[dcarney, 2014/01/04 15:56:45]

previous is an int because of some special values used in the encoder, like
kNoPreviousCharacter

[haimuiba, 2014/01/06 05:40:18]

Makes sense. Thx.

+  static int WritePair(uint16_t current, int previous, char* buffer) {
+    using namespace unibrow;
+    int code_point = current;
+    int written = 0;
+    if (Utf16::IsSurrogatePair(previous, current)) {
+      code_point = Utf16::CombineSurrogatePair(previous, current);
+      buffer -= Utf8::kSizeOfUnmatchedSurrogate;
+      written -= Utf8::kSizeOfUnmatchedSurrogate;
+    }
+    return written + Utf8::Encode(buffer, code_point, false);

[dcarney, 2014/01/04 15:56:45]

having the length calculation here is too late.  String::WriteUtf8 must compute
the identical length, which it can't do if you're potentially changing it here.

[haimuiba, 2014/01/06 05:40:18]

Ok, I'll take a closer look.

+  }
+
+  // @TODO use uint16_t for previous?
+  static int WriteEndCharacter(uint16_t current,
+                               int previous,
                                int remaining,
                                char* const buffer) {
     using namespace unibrow;
     ASSERT(remaining > 0);
-    // We can't use a local buffer here because Encode needs to modify
-    // previous characters in the stream.  We know, however, that
-    // exactly one character will be advanced.
-    if (Utf16::IsTrailSurrogate(character) &&
-        Utf16::IsLeadSurrogate(last_character)) {
-      int written = Utf8::Encode(buffer, character, last_character);
+    // We can't use a local buffer here because WritePair needs to modify
+    // previous characters in the stream.  We know, however, that exactly one
+    // character will be advanced.
+    if (Utf16::IsSurrogatePair(previous, current)) {
+      int written = WritePair(current, previous, buffer);
       ASSERT(written == 1);
       return written;
     }
     // Use a scratch buffer to check the required characters.
     char temp_buffer[Utf8::kMaxEncodedSize];
     // Can't encode using last_character as gcc has array bounds issues.
-    int written = Utf8::Encode(temp_buffer,
-                               character,
-                               Utf16::kNoPreviousCharacter);
+    int written = WritePair(current, Utf16::kNoPreviousCharacter, temp_buffer);
     // Won't fit.
     if (written > remaining) return 0;
     // Copy over the character from temp_buffer.
     for (int j = 0; j < written; j++) {
       buffer[j] = temp_buffer[j];
     }
     return written;
   }
 
   template<typename Char>
@@ skipping 26 matching lines @@
       // Write the characters to the stream.
       if (sizeof(Char) == 1) {
         for (; i < fast_length; i++) {
           buffer +=
               Utf8::EncodeOneByte(buffer, static_cast<uint8_t>(*chars++));
           ASSERT(capacity_ == -1 || (buffer - start_) <= capacity_);
         }
       } else {
         for (; i < fast_length; i++) {
           uint16_t character = *chars++;
-          buffer += Utf8::Encode(buffer, character, last_character);
+          buffer += WritePair(character, last_character, buffer);
           last_character = character;
           ASSERT(capacity_ == -1 || (buffer - start_) <= capacity_);
         }
       }
       // Array is fully written. Exit.
       if (fast_length == length) {
         // Write state back out to object.
         last_character_ = last_character;
         buffer_ = buffer;
         utf16_chars_read_ += length;
@@ skipping 2920 matching lines @@
   Isolate* isolate = reinterpret_cast<Isolate*>(info.GetIsolate());
   Address callback_address =
       reinterpret_cast<Address>(reinterpret_cast<intptr_t>(callback));
   VMState<EXTERNAL> state(isolate);
   ExternalCallbackScope call_scope(isolate, callback_address);
   callback(info);
 }
 
 
 } }  // namespace v8::internal