Add support for surrogates when serializing and deserializing for native ports

runtime/vm/dart_api_message.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/dart_api_message.h"
 #include "vm/object.h"
 #include "vm/snapshot_ids.h"
 #include "vm/symbols.h"
 #include "vm/unicode.h"
 
@@ skipping 371 matching lines @@
       ::free(latin1);
       return object;
     }
     case kTwoByteStringCid: {
       intptr_t len = ReadSmiValue();
       intptr_t hash = ReadSmiValue();
       USE(hash);
       uint16_t *utf16 =
           reinterpret_cast<uint16_t*>(::malloc(len * sizeof(uint16_t)));
       intptr_t utf8_len = 0;
+      // Read all the UTF-16 code units.
       for (intptr_t i = 0; i < len; i++) {
         utf16[i] = Read<uint16_t>();
-        // TODO(sgjesse): Check for surrogate pairs.
-        utf8_len += Utf8::Length(utf16[i]);
+      }
+      // Calculate the UTF-8 length.
+      for (intptr_t i = 0; i < len; i++) {

[erikcorry, 2012/11/23 14:25:47]

There is a CodePointIterator in object.h that you may be able to use instead.

[Søren Gjesse, 2012/11/26 08:53:00]

That is not possible as this is a raw de-serialized utf16 array outside the VM
and not a VM string. But thanks for mentioning it. I ended up implementing a
Utf16::CodePointIterator matching String::CodePointIterator (both does the same
as the code below).

+        if (i < len - 1 &&
+            Utf16::IsLeadSurrogate(utf16[i]) &&
+            Utf16::IsTrailSurrogate(utf16[i + 1])) {
+          uint32_t ch = Utf16::Decode(utf16[i], utf16[i + 1]);
+          utf8_len += Utf8::Length(ch);
+          i++;
+        } else {
+          utf8_len += Utf8::Length(utf16[i]);
+        }
       }
       Dart_CObject* object = AllocateDartCObjectString(utf8_len);
       AddBackRef(object_id, object, kIsDeserialized);
       char* p = object->value.as_string;
       for (intptr_t i = 0; i < len; i++) {
-        // TODO(sgjesse): Check for surrogate pairs.
-        p += Utf8::Encode(utf16[i], p);
+        if (i < len - 1 &&
+            Utf16::IsLeadSurrogate(utf16[i]) &&
+            Utf16::IsTrailSurrogate(utf16[i + 1])) {
+          uint32_t ch = Utf16::Decode(utf16[i], utf16[i + 1]);
+          p += Utf8::Encode(ch, p);
+          i++;
+        } else {
+          p += Utf8::Encode(utf16[i], p);
+        }
       }
       *p = '\0';
       ASSERT(p == (object->value.as_string + utf8_len));
       ::free(utf16);
       return object;
     }
     case kUint8ArrayCid: {
       intptr_t len = ReadSmiValue();
       Dart_CObject* object = AllocateDartCObjectUint8Array(len);
       AddBackRef(object_id, object, kIsDeserialized);
@@ skipping 368 matching lines @@
       // Write out the class and tags information.
       WriteIndexedObject(kDoubleCid);
       WriteIntptrValue(0);
       // Write double value.
       Write<double>(object->value.as_double);
       break;
     case Dart_CObject::kString: {
       const uint8_t* utf8_str =
           reinterpret_cast<const uint8_t*>(object->value.as_string);
       intptr_t utf8_len = strlen(object->value.as_string);
-      if (!Utf8::IsValid(utf8_str, utf8_len)) {
+      if (!Utf8::IsValidAllowSurrogates(utf8_str, utf8_len)) {
         return false;
       }
 
       Utf8::Type type;
       intptr_t len = Utf8::CodePointCount(utf8_str, utf8_len, &type);
 
       // Write out the serialization header value for this object.
       WriteInlinedHeader(object);
       // Write out the class and tags information.
       WriteIndexedObject(type == Utf8::kLatin1 ? kOneByteStringCid
                                                : kTwoByteStringCid);
       WriteIntptrValue(0);
       // Write string length, hash and content
       WriteSmi(len);
       WriteSmi(0);  // TODO(sgjesse): Hash - not written.
       if (type == Utf8::kLatin1) {
         uint8_t* latin1_str =
             reinterpret_cast<uint8_t*>(::malloc(len * sizeof(uint8_t)));
-        Utf8::DecodeToLatin1(utf8_str, utf8_len, latin1_str, len);
+        bool success = Utf8::DecodeToLatin1(utf8_str,
+                                            utf8_len,
+                                            latin1_str,
+                                            len);
+        ASSERT(success);
         for (intptr_t i = 0; i < len; i++) {
           Write<uint8_t>(latin1_str[i]);
         }
         ::free(latin1_str);
       } else {
         // TODO(sgjesse): Make sure surrogate pairs are handled.
         uint16_t* utf16_str =
             reinterpret_cast<uint16_t*>(::malloc(len * sizeof(uint16_t)));
-        Utf8::DecodeToUTF16(utf8_str, utf8_len, utf16_str, len);
+        bool success = Utf8::DecodeToUTF16AllowSurrogates(utf8_str,
+                                                          utf8_len,
+                                                          utf16_str,
+                                                          len);
+        ASSERT(success);
         for (intptr_t i = 0; i < len; i++) {
           Write<uint16_t>(utf16_str[i]);
         }
         ::free(utf16_str);
       }
       break;
     }
     case Dart_CObject::kUint8Array: {
       // Write out the serialization header value for this object.
       WriteInlinedHeader(object);
@@ skipping 52 matching lines @@
     if (!success) {
       UnmarkAllCObjects(object);
       return false;
     }
   }
   UnmarkAllCObjects(object);
   return true;
 }
 
 }  // namespace dart