Add support for surrogates when serializing and deserializing for native ports

runtime/vm/unicode.cc

-// Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
+// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/unicode.h"
 
 #include "vm/allocation.h"
 #include "vm/globals.h"
 #include "vm/object.h"
 
 namespace dart {
@@ skipping 40 matching lines @@
   0xFFFFFFFF
 };
 
 
 static bool IsTrailByte(uint8_t code_unit) {
   return (code_unit & 0xc0) == 0x80;
 }
 
 
 static bool IsLatin1SequenceStart(uint8_t code_unit) {
-  // Check is codepoint is <= U+00FF
+  // Check is codepoint is <= U+00FF.
   return (code_unit <= Utf8::kMaxOneByteChar);
 }
 
 
 static bool IsSupplementarySequenceStart(uint8_t code_unit) {
   // Check is codepoint is >= U+10000.
   return (code_unit >= 0xF0);
 }
 
 
@@ skipping 28 matching lines @@
         char_type = kBMP;
       }
     }
   }
   *type = char_type;
   return len;
 }
 
 
 // Returns true if str is a valid NUL-terminated UTF-8 string.
-bool Utf8::IsValid(const uint8_t* utf8_array, intptr_t array_len) {
+static bool IsValidUtf8(
+    const uint8_t* utf8_array, intptr_t array_len, bool allow_surrogates) {
   intptr_t i = 0;
   while (i < array_len) {
     uint32_t ch = utf8_array[i] & 0xFF;
     intptr_t j = 1;
     if (ch >= 0x80) {
       int8_t num_trail_bytes = kTrailBytes[ch];
       bool is_malformed = false;
       for (; j < num_trail_bytes; ++j) {
         if ((i + j) < array_len) {
           uint8_t code_unit = utf8_array[i + j];
           is_malformed |= !IsTrailByte(code_unit);
           ch = (ch << 6) + code_unit;
         } else {
           return false;
         }
       }
       ch -= kMagicBits[num_trail_bytes];
       if (!((is_malformed == false) &&
             (j == num_trail_bytes) &&
             !IsOutOfRange(ch) &&
             !IsNonShortestForm(ch, j) &&
-            !Utf16::IsSurrogate(ch))) {
+            (!Utf16::IsSurrogate(ch) || allow_surrogates))) {
         return false;
       }
     }
     i += j;
   }
   return true;
 }
 
 
+bool Utf8::IsValid(const uint8_t* utf8_array, intptr_t array_len) {
+  return IsValidUtf8(utf8_array, array_len, false);
+}
+
+
+bool Utf8::IsValidAllowSurrogates(
+    const uint8_t* utf8_array, intptr_t array_len) {
+  return IsValidUtf8(utf8_array, array_len, true);
+}
+
+
 intptr_t Utf8::Length(int32_t ch) {
   if (ch <= kMaxOneByteChar) {
     return 1;
   } else if (ch <= kMaxTwoByteChar) {
     return 2;
   } else if (ch <= kMaxThreeByteChar) {
     return 3;
   }
   ASSERT(ch <= kMaxFourByteChar);
   return 4;
@@ skipping 46 matching lines @@
     if (pos + num_bytes > len) {
       break;
     }
     Utf8::Encode(ch, &dst[pos]);
     pos += num_bytes;
   }
   return pos;
 }
 
 
-intptr_t Utf8::Decode(const uint8_t* utf8_array,
-                      intptr_t array_len,
-                      int32_t* dst) {
+static intptr_t DecodeUTF8(const uint8_t* utf8_array,
+                           intptr_t array_len,
+                           int32_t* dst,
+                           bool allow_surrogates) {
   uint32_t ch = utf8_array[0] & 0xFF;
   intptr_t i = 1;
   if (ch >= 0x80) {
     intptr_t num_trail_bytes = kTrailBytes[ch];
     bool is_malformed = false;
     for (; i < num_trail_bytes; ++i) {
       if (i < array_len) {
         uint8_t code_unit = utf8_array[i];
         is_malformed |= !IsTrailByte(code_unit);
         ch = (ch << 6) + code_unit;
       } else {
         *dst = -1;
         return 0;
       }
     }
     ch -= kMagicBits[num_trail_bytes];
     if (!((is_malformed == false) &&
           (i == num_trail_bytes) &&
           !IsOutOfRange(ch) &&
           !IsNonShortestForm(ch, i) &&
-          !Utf16::IsSurrogate(ch))) {
+          (!Utf16::IsSurrogate(ch) || allow_surrogates))) {
       *dst = -1;
       return 0;
     }
   }
   *dst = ch;
   return i;
 }
 
 
+intptr_t Utf8::Decode(const uint8_t* utf8_array,
+                      intptr_t array_len,
+                      int32_t* dst) {
+  return DecodeUTF8(utf8_array, array_len, dst, false);
+}
+
+
+intptr_t Utf8::DecodeAllowSurrogates(const uint8_t* utf8_array,
+                      intptr_t array_len,
+                      int32_t* dst) {
+  return DecodeUTF8(utf8_array, array_len, dst, true);
+}
+
+
 bool Utf8::DecodeToLatin1(const uint8_t* utf8_array,
                           intptr_t array_len,
                           uint8_t* dst,
                           intptr_t len) {
   intptr_t i = 0;
   intptr_t j = 0;
   intptr_t num_bytes;
   for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
     int32_t ch;
     ASSERT(IsLatin1SequenceStart(utf8_array[i]));
     num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
     if (ch == -1) {
-      return false;  // invalid input
+      return false;  // Invalid input.
     }
     ASSERT(ch <= 0xff);
     dst[j] = ch;
   }
   if ((i < array_len) && (j == len)) {
-    return false;  // output overflow
+    return false;  // Output overflow.
   }
-  return true;  // success
+  return true;  // Success.
 }
 
 
-bool Utf8::DecodeToUTF16(const uint8_t* utf8_array,
-                         intptr_t array_len,
-                         uint16_t* dst,
-                         intptr_t len) {
+bool DecodeUTF8ToUTF16(const uint8_t* utf8_array,
+                       intptr_t array_len,
+                       uint16_t* dst,
+                       intptr_t len,
+                       bool allow_surrogates) {
   intptr_t i = 0;
   intptr_t j = 0;
   intptr_t num_bytes;
   for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
     int32_t ch;
     bool is_supplementary = IsSupplementarySequenceStart(utf8_array[i]);
-    num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
+    num_bytes = DecodeUTF8(
+        &utf8_array[i], (array_len - i), &ch, allow_surrogates);
     if (ch == -1) {
-      return false;  // invalid input
+      return false;  // Invalid input.
     }
     if (is_supplementary) {
       Utf16::Encode(ch, &dst[j]);
       j = j + 1;
     } else {
       dst[j] = ch;
     }
   }
   if ((i < array_len) && (j == len)) {
-    return false;  // output overflow
+    return false;  // Output overflow.
   }
-  return true;  // success
+  return true;  // Success.
 }
 
 
+bool Utf8::DecodeToUTF16(const uint8_t* utf8_array,
+                         intptr_t array_len,
+                         uint16_t* dst,
+                         intptr_t len) {
+  return DecodeUTF8ToUTF16(utf8_array, array_len, dst, len, false);
+}
+
+
+bool Utf8::DecodeToUTF16AllowSurrogates(const uint8_t* utf8_array,
+                                        intptr_t array_len,
+                                        uint16_t* dst,
+                                        intptr_t len) {
+  return DecodeUTF8ToUTF16(utf8_array, array_len, dst, len, true);
+}
+
+
 bool Utf8::DecodeToUTF32(const uint8_t* utf8_array,
                          intptr_t array_len,
                          int32_t* dst,
                          intptr_t len) {
   intptr_t i = 0;
   intptr_t j = 0;
   intptr_t num_bytes;
   for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
     int32_t ch;
     num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
     if (ch == -1) {
-      return false;  // invalid input
+      return false;  // Invalid input.
     }
     dst[j] = ch;
   }
   if ((i < array_len) && (j == len)) {
-    return false;  // output overflow
+    return false;  // Output overflow.
   }
-  return true;  // success
+  return true;  // Success.
 }
 
 
 void Utf16::Encode(int32_t codepoint, uint16_t* dst) {
   ASSERT(codepoint > kMaxBmpCodepoint);
   ASSERT(dst != NULL);
   dst[0] = (Utf16::kLeadSurrogateOffset + (codepoint >> 10));
   dst[1] = (0xDC00 + (codepoint & 0x3FF));
 }
 
 }  // namespace dart