String:WriteUtf8: Add REPLACE_INVALID_UTF8 option

src/api.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 4486 matching lines @@
 int String::Utf8Length() const {
   i::Handle<i::String> str = Utils::OpenHandle(this);
   i::Isolate* isolate = str->GetIsolate();
   return v8::Utf8Length(*str, isolate);
 }
 
 
 class Utf8WriterVisitor {
  public:
   Utf8WriterVisitor(
-      char* buffer, int capacity, bool skip_capacity_check)
+      char* buffer,
+      int capacity,
+      bool skip_capacity_check,
+      bool allow_invalid_utf8)
     : early_termination_(false),
       last_character_(unibrow::Utf16::kNoPreviousCharacter),
       buffer_(buffer),
       start_(buffer),
       capacity_(capacity),
       skip_capacity_check_(capacity == -1 || skip_capacity_check),
+      allow_invalid_utf8_(allow_invalid_utf8),
       utf16_chars_read_(0) {
   }
 
-  static int WriteEndCharacter(uint16_t character,
-                               int last_character,
+  // WritePair writes the current UTF-16 code unit to the given buffer. The
+  // function will go back inside the buffer to combine surrogate pairs.
+  static int WritePair(uint16_t current,

[dcarney, 2014/01/07 10:12:16]

WritePair is a bad name here, since it may or not be a pair.

+                       int previous,
+                       char* buffer,
+                       bool allow_invalid_utf8) {
+    using namespace unibrow;
+    int code_point = current;
+    int written = 0;
+    if (Utf16::IsSurrogatePair(previous, current)) {
+      code_point = Utf16::CombineSurrogatePair(previous, current);
+      buffer -= Utf8::kSizeOfUnmatchedSurrogate;
+      written -= Utf8::kSizeOfUnmatchedSurrogate;
+    }
+    return written + Utf8::Encode(buffer, code_point, allow_invalid_utf8);
+  }
+
+  static int WriteEndCharacter(uint16_t current,
+                               int previous,
                                int remaining,
-                               char* const buffer) {
+                               char* const buffer,
+                               bool allow_invalid_utf8) {
     using namespace unibrow;
     ASSERT(remaining > 0);
-    // We can't use a local buffer here because Encode needs to modify
-    // previous characters in the stream.  We know, however, that
-    // exactly one character will be advanced.
-    if (Utf16::IsTrailSurrogate(character) &&
-        Utf16::IsLeadSurrogate(last_character)) {
-      int written = Utf8::Encode(buffer, character, last_character);
+    // We can't use a local buffer here because WritePair needs to modify
+    // previous characters in the stream.  We know, however, that exactly one
+    // character will be advanced.
+    if (Utf16::IsSurrogatePair(previous, current)) {
+      int written = WritePair(current, previous, buffer, allow_invalid_utf8);
       ASSERT(written == 1);
       return written;
     }
     // Use a scratch buffer to check the required characters.
     char temp_buffer[Utf8::kMaxEncodedSize];
     // Can't encode using last_character as gcc has array bounds issues.
-    int written = Utf8::Encode(temp_buffer,
-                               character,
-                               Utf16::kNoPreviousCharacter);
+    int written = WritePair(current,

[dcarney, 2014/01/07 10:12:16]

this is not a surrogate pair, could use Encode directly

+                            Utf16::kNoPreviousCharacter,
+                            temp_buffer,
+                            allow_invalid_utf8);
     // Won't fit.
     if (written > remaining) return 0;
     // Copy over the character from temp_buffer.
     for (int j = 0; j < written; j++) {
       buffer[j] = temp_buffer[j];
     }
     return written;
   }
 
   template<typename Char>
@@ skipping 26 matching lines @@
       // Write the characters to the stream.
       if (sizeof(Char) == 1) {
         for (; i < fast_length; i++) {
           buffer +=
               Utf8::EncodeOneByte(buffer, static_cast<uint8_t>(*chars++));
           ASSERT(capacity_ == -1 || (buffer - start_) <= capacity_);
         }
       } else {
         for (; i < fast_length; i++) {
           uint16_t character = *chars++;
-          buffer += Utf8::Encode(buffer, character, last_character);
+          buffer += WritePair(character,
+                              last_character,
+                              buffer,
+                              allow_invalid_utf8_);
           last_character = character;
           ASSERT(capacity_ == -1 || (buffer - start_) <= capacity_);
         }
       }
       // Array is fully written. Exit.
       if (fast_length == length) {
         // Write state back out to object.
         last_character_ = last_character;
         buffer_ = buffer;
         utf16_chars_read_ += length;
         return;
       }
     }
     ASSERT(!skip_capacity_check_);
     // Slow loop. Must check capacity on each iteration.
     int remaining_capacity = capacity_ - static_cast<int>(buffer - start_);
     ASSERT(remaining_capacity >= 0);
     for (; i < length && remaining_capacity > 0; i++) {
       uint16_t character = *chars++;
       int written = WriteEndCharacter(character,
                                       last_character,
                                       remaining_capacity,
-                                      buffer);
+                                      buffer,
+                                      allow_invalid_utf8_);
       if (written == 0) {
         early_termination_ = true;
         break;
       }
       buffer += written;
       remaining_capacity -= written;
       last_character = character;
     }
     // Write state back out to object.
     last_character_ = last_character;
@@ skipping 27 matching lines @@
     return static_cast<int>(buffer_ - start_);
   }
 
  private:
   bool early_termination_;
   int last_character_;
   char* buffer_;
   char* const start_;
   int capacity_;
   bool const skip_capacity_check_;
+  bool const allow_invalid_utf8_;
   int utf16_chars_read_;
   DISALLOW_IMPLICIT_CONSTRUCTORS(Utf8WriterVisitor);
 };
 
 
 static bool RecursivelySerializeToUtf8(i::String* current,
                                       Utf8WriterVisitor* writer,
                                       int recursion_budget) {
   while (!writer->IsDone()) {
     i::ConsString* cons_string = i::String::VisitFlat(writer, current);
@@ skipping 18 matching lines @@
                       int options) const {
   i::Isolate* isolate = Utils::OpenHandle(this)->GetIsolate();
   LOG_API(isolate, "String::WriteUtf8");
   ENTER_V8(isolate);
   i::Handle<i::String> str = Utils::OpenHandle(this);
   if (options & HINT_MANY_WRITES_EXPECTED) {
     FlattenString(str);  // Flatten the string for efficiency.
   }
   const int string_length = str->length();
   bool write_null = !(options & NO_NULL_TERMINATION);
+  bool allow_invalid_utf8 = !(options & DISALLOW_INVALID_UTF8);
   // First check if we can just write the string without checking capacity.
+  // @TODO Replace magic number 3 with something more descriptive. E.g.
+  // Utf8::kMaxTwoByteSize (as in the maximum size an unsighed 2 byte code unit
+  // value will take up when encoded to UTF-8)? When I first read this code I
+  // thought there might be a overflow bug here since UTF-8 may take up to 4
+  // bytes per code unit. Then I realized that a surrogate pair has a
+  // str.length of 2, making the code correct.
   if (capacity == -1 || capacity / 3 >= string_length) {
-    Utf8WriterVisitor writer(buffer, capacity, true);
+    Utf8WriterVisitor writer(buffer, capacity, true, allow_invalid_utf8);
     const int kMaxRecursion = 100;
     bool success = RecursivelySerializeToUtf8(*str, &writer, kMaxRecursion);
     if (success) return writer.CompleteWrite(write_null, nchars_ref);
   } else if (capacity >= string_length) {
     // First check that the buffer is large enough.
     int utf8_bytes = v8::Utf8Length(*str, str->GetIsolate());
     if (utf8_bytes <= capacity) {
       // ASCII fast path.
       if (utf8_bytes == string_length) {
         WriteOneByte(reinterpret_cast<uint8_t*>(buffer), 0, capacity, options);
         if (nchars_ref != NULL) *nchars_ref = string_length;
         if (write_null && (utf8_bytes+1 <= capacity)) {
           return string_length + 1;
         }
         return string_length;
       }
       if (write_null && (utf8_bytes+1 > capacity)) {
         options |= NO_NULL_TERMINATION;
       }
       // Recurse once without a capacity limit.
       // This will get into the first branch above.
       // TODO(dcarney) Check max left rec. in Utf8Length and fall through.
       return WriteUtf8(buffer, -1, nchars_ref, options);
     }
   }
   // Recursive slow path can potentially be unreasonable slow. Flatten.
   str = FlattenGetString(str);
-  Utf8WriterVisitor writer(buffer, capacity, false);
+  Utf8WriterVisitor writer(buffer, capacity, false, allow_invalid_utf8);
   i::String::VisitFlat(&writer, *str);
   return writer.CompleteWrite(write_null, nchars_ref);
 }
 
 
 template<typename CharType>
 static inline int WriteHelper(const String* string,
                               CharType* buffer,
                               int start,
                               int length,
@@ skipping 2781 matching lines @@
   Isolate* isolate = reinterpret_cast<Isolate*>(info.GetIsolate());
   Address callback_address =
       reinterpret_cast<Address>(reinterpret_cast<intptr_t>(callback));
   VMState<EXTERNAL> state(isolate);
   ExternalCallbackScope call_scope(isolate, callback_address);
   callback(info);
 }
 
 
 } }  // namespace v8::internal