String:WriteUtf8: Add REPLACE_INVALID_UTF8 option

src/api.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 4486 matching lines @@
 int String::Utf8Length() const {
   i::Handle<i::String> str = Utils::OpenHandle(this);
   i::Isolate* isolate = str->GetIsolate();
   return v8::Utf8Length(*str, isolate);
 }
 
 
 class Utf8WriterVisitor {
  public:
   Utf8WriterVisitor(
-      char* buffer, int capacity, bool skip_capacity_check)
+      char* buffer,
+      int capacity,
+      bool skip_capacity_check,
+      bool allow_invalid_utf8)
     : early_termination_(false),
       last_character_(unibrow::Utf16::kNoPreviousCharacter),
       buffer_(buffer),
       start_(buffer),
       capacity_(capacity),
       skip_capacity_check_(capacity == -1 || skip_capacity_check),
+      allow_invalid_utf8_(allow_invalid_utf8),
       utf16_chars_read_(0) {
   }
 
   static int WriteEndCharacter(uint16_t character,
                                int last_character,
                                int remaining,
-                               char* const buffer) {
+                               char* const buffer,
+                               bool allow_invalid_utf8) {
     using namespace unibrow;
     ASSERT(remaining > 0);
     // We can't use a local buffer here because Encode needs to modify
     // previous characters in the stream.  We know, however, that
     // exactly one character will be advanced.
-    if (Utf16::IsTrailSurrogate(character) &&
-        Utf16::IsLeadSurrogate(last_character)) {
-      int written = Utf8::Encode(buffer, character, last_character);
+    if (Utf16::IsSurrogatePair(last_character, character)) {
+      int written = Utf8::Encode(buffer,
+                                 character,
+                                 last_character,
+                                 allow_invalid_utf8);
       ASSERT(written == 1);
       return written;
     }
     // Use a scratch buffer to check the required characters.
     char temp_buffer[Utf8::kMaxEncodedSize];
     // Can't encode using last_character as gcc has array bounds issues.
     int written = Utf8::Encode(temp_buffer,
                                character,
-                               Utf16::kNoPreviousCharacter);
+                               Utf16::kNoPreviousCharacter,
+                               allow_invalid_utf8);
     // Won't fit.
     if (written > remaining) return 0;
     // Copy over the character from temp_buffer.
     for (int j = 0; j < written; j++) {
       buffer[j] = temp_buffer[j];
     }
     return written;
   }
 
   template<typename Char>
@@ skipping 26 matching lines @@
       // Write the characters to the stream.
       if (sizeof(Char) == 1) {
         for (; i < fast_length; i++) {
           buffer +=
               Utf8::EncodeOneByte(buffer, static_cast<uint8_t>(*chars++));
           ASSERT(capacity_ == -1 || (buffer - start_) <= capacity_);
         }
       } else {
         for (; i < fast_length; i++) {
           uint16_t character = *chars++;
-          buffer += Utf8::Encode(buffer, character, last_character);
+          buffer += Utf8::Encode(buffer,
+                                 character,
+                                 last_character,
+                                 allow_invalid_utf8_);
           last_character = character;
           ASSERT(capacity_ == -1 || (buffer - start_) <= capacity_);
         }
       }
       // Array is fully written. Exit.
       if (fast_length == length) {
         // Write state back out to object.
         last_character_ = last_character;
         buffer_ = buffer;
         utf16_chars_read_ += length;
         return;
       }
     }
     ASSERT(!skip_capacity_check_);
     // Slow loop. Must check capacity on each iteration.
     int remaining_capacity = capacity_ - static_cast<int>(buffer - start_);
     ASSERT(remaining_capacity >= 0);
     for (; i < length && remaining_capacity > 0; i++) {
       uint16_t character = *chars++;
       int written = WriteEndCharacter(character,
                                       last_character,
                                       remaining_capacity,
-                                      buffer);
+                                      buffer,
+                                      allow_invalid_utf8_);
       if (written == 0) {
         early_termination_ = true;
         break;
       }
       buffer += written;
       remaining_capacity -= written;
       last_character = character;
     }
     // Write state back out to object.
     last_character_ = last_character;
@@ skipping 27 matching lines @@
     return static_cast<int>(buffer_ - start_);
   }
 
  private:
   bool early_termination_;
   int last_character_;
   char* buffer_;
   char* const start_;
   int capacity_;
   bool const skip_capacity_check_;
+  bool const allow_invalid_utf8_;
   int utf16_chars_read_;
   DISALLOW_IMPLICIT_CONSTRUCTORS(Utf8WriterVisitor);
 };
 
 
 static bool RecursivelySerializeToUtf8(i::String* current,
                                       Utf8WriterVisitor* writer,
                                       int recursion_budget) {
   while (!writer->IsDone()) {
     i::ConsString* cons_string = i::String::VisitFlat(writer, current);
@@ skipping 18 matching lines @@
                       int options) const {
   i::Isolate* isolate = Utils::OpenHandle(this)->GetIsolate();
   LOG_API(isolate, "String::WriteUtf8");
   ENTER_V8(isolate);
   i::Handle<i::String> str = Utils::OpenHandle(this);
   if (options & HINT_MANY_WRITES_EXPECTED) {
     FlattenString(str);  // Flatten the string for efficiency.
   }
   const int string_length = str->length();
   bool write_null = !(options & NO_NULL_TERMINATION);
+  bool allow_invalid_utf8 = !(options & DISALLOW_INVALID_UTF8);
   // First check if we can just write the string without checking capacity.
+  // @TODO Replace magic number 3 with something more descriptive. E.g.

[dcarney, 2014/01/10 16:54:33]

the syntax we use is:

// TODO(username) Comment.

but in this case, it just needs to be done before landing...

[haimuiba, 2014/01/13 07:48:21]

Done.

+  // Utf8::kMaxTwoByteSize (as in the maximum size an unsighed 2 byte code unit
+  // value will take up when encoded to UTF-8)? When I first read this code I
+  // thought there might be a overflow bug here since UTF-8 may take up to 4
+  // bytes per code unit. Then I realized that a surrogate pair has a
+  // str.length of 2, making the code correct.
   if (capacity == -1 || capacity / 3 >= string_length) {
-    Utf8WriterVisitor writer(buffer, capacity, true);
+    Utf8WriterVisitor writer(buffer, capacity, true, allow_invalid_utf8);
     const int kMaxRecursion = 100;
     bool success = RecursivelySerializeToUtf8(*str, &writer, kMaxRecursion);
     if (success) return writer.CompleteWrite(write_null, nchars_ref);
   } else if (capacity >= string_length) {
     // First check that the buffer is large enough.
     int utf8_bytes = v8::Utf8Length(*str, str->GetIsolate());
     if (utf8_bytes <= capacity) {
       // ASCII fast path.
       if (utf8_bytes == string_length) {
         WriteOneByte(reinterpret_cast<uint8_t*>(buffer), 0, capacity, options);
         if (nchars_ref != NULL) *nchars_ref = string_length;
         if (write_null && (utf8_bytes+1 <= capacity)) {
           return string_length + 1;
         }
         return string_length;
       }
       if (write_null && (utf8_bytes+1 > capacity)) {
         options |= NO_NULL_TERMINATION;
       }
       // Recurse once without a capacity limit.
       // This will get into the first branch above.
       // TODO(dcarney) Check max left rec. in Utf8Length and fall through.
       return WriteUtf8(buffer, -1, nchars_ref, options);
     }
   }
   // Recursive slow path can potentially be unreasonable slow. Flatten.
   str = FlattenGetString(str);
-  Utf8WriterVisitor writer(buffer, capacity, false);
+  Utf8WriterVisitor writer(buffer, capacity, false, allow_invalid_utf8);
   i::String::VisitFlat(&writer, *str);
   return writer.CompleteWrite(write_null, nchars_ref);

[dcarney, 2014/01/10 16:49:55]

think you need to ensure that all return points from this function check for the
case that you  don't allow invalid utf8, and the last character written is one
half of a surrogate pair, which would be incorrectly written as a replacement
character in the current implementation

[haimuiba, 2014/01/13 07:48:21]

Not sure I understand. If allow_invalid_utf8=false, then any lone surrogate is
an invalid last character, and should be replaced.

 }
 
 
 template<typename CharType>
 static inline int WriteHelper(const String* string,
                               CharType* buffer,
                               int start,
                               int length,
                               int options) {
   i::Isolate* isolate = Utils::OpenHandle(string)->GetIsolate();
@@ skipping 2779 matching lines @@
   Isolate* isolate = reinterpret_cast<Isolate*>(info.GetIsolate());
   Address callback_address =
       reinterpret_cast<Address>(reinterpret_cast<intptr_t>(callback));
   VMState<EXTERNAL> state(isolate);
   ExternalCallbackScope call_scope(isolate, callback_address);
   callback(info);
 }
 
 
 } }  // namespace v8::internal