Add support for surrogates when serializing and deserializing for native ports

runtime/vm/unicode.cc

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/unicode.h"
 
 #include "vm/allocation.h"
 #include "vm/globals.h"
 #include "vm/object.h"
 
@@ skipping 13 matching lines @@
   0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
   0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
   2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
   2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
   3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
   4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0
 };
 
 
 static const uint32_t kMagicBits[7] = {
-  0,  // padding
+  0,  // Padding.
   0x00000000,
   0x00003080,
   0x000E2080,
   0x03C82080,
   0xFA082080,
   0x82082080
 };
 
 
 // Minimum values of code points used to check shortest form.
 static const uint32_t kOverlongMinimum[7] = {
-  0,  // padding
+  0,  // Padding.
   0x0,
   0x80,
   0x800,
   0x10000,
   0xFFFFFFFF,
   0xFFFFFFFF
 };
 
 
 static bool IsTrailByte(uint8_t code_unit) {
   return (code_unit & 0xc0) == 0x80;
 }
 
 
 static bool IsLatin1SequenceStart(uint8_t code_unit) {
-  // Check is codepoint is <= U+00FF
+  // Check if codepoint is <= U+00FF.
   return (code_unit <= Utf8::kMaxOneByteChar);
 }
 
 
 static bool IsSupplementarySequenceStart(uint8_t code_unit) {
-  // Check is codepoint is >= U+10000.
+  // Check if codepoint is >= U+10000.
   return (code_unit >= 0xF0);
 }
 
 
 // Returns true if the code point value is above Plane 17.
 static bool IsOutOfRange(uint32_t code_point) {
   return (code_point > 0x10FFFF);
 }
 
 
 // Returns true if the byte sequence is ill-formed.
 static bool IsNonShortestForm(uint32_t code_point, size_t num_bytes) {
   return code_point < kOverlongMinimum[num_bytes];
 }
 
 
 // Returns a count of the number of UTF-8 trail bytes.
 intptr_t Utf8::CodePointCount(const uint8_t* utf8_array,
                               intptr_t array_len,
                               Type* type) {
   intptr_t len = 0;
   Type char_type = kLatin1;
   for (intptr_t i = 0; i < array_len; i++) {
     uint8_t code_unit = utf8_array[i];
     if (!IsTrailByte(code_unit)) {
       ++len;
     }
-    if (!IsLatin1SequenceStart(code_unit)) {  // > U+00FF
-      if (IsSupplementarySequenceStart(code_unit)) {  // >= U+10000
+    if (!IsLatin1SequenceStart(code_unit)) {  // > U+00FF.

[cshapiro, 2012/11/30 02:49:08]

no period

[Søren Gjesse, 2012/11/30 12:23:07]

Done.

+      if (IsSupplementarySequenceStart(code_unit)) {  // >= U+10000.

[cshapiro, 2012/11/30 02:49:08]

ditto

[Søren Gjesse, 2012/11/30 12:23:07]

Done.

         char_type = kSupplementary;
         ++len;
       } else if (char_type == kLatin1) {
         char_type = kBMP;
       }
     }
   }
   *type = char_type;
   return len;
 }
@@ skipping 50 matching lines @@
   String::CodePointIterator it(str);
   while (it.Next()) {
     int32_t ch = it.Current();
     length += Utf8::Length(ch);
   }
   return length;
 }
 
 
 intptr_t Utf8::Encode(int32_t ch, char* dst) {
+  ASSERT(!Utf16::IsSurrogate(ch));
   static const int kMask = ~(1 << 6);
   if (ch <= kMaxOneByteChar) {
     dst[0] = ch;
     return 1;
   }
   if (ch <= kMaxTwoByteChar) {
     dst[0] = 0xC0 | (ch >> 6);
     dst[1] = 0x80 | (ch & kMask);
     return 2;
   }
@@ skipping 66 matching lines @@
                           uint8_t* dst,
                           intptr_t len) {
   intptr_t i = 0;
   intptr_t j = 0;
   intptr_t num_bytes;
   for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
     int32_t ch;
     ASSERT(IsLatin1SequenceStart(utf8_array[i]));
     num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
     if (ch == -1) {
-      return false;  // invalid input
+      return false;  // Invalid input.
     }
     ASSERT(ch <= 0xff);
     dst[j] = ch;
   }
   if ((i < array_len) && (j == len)) {
-    return false;  // output overflow
+    return false;  // Output overflow.
   }
-  return true;  // success
+  return true;  // Success.
 }
 
 
 bool Utf8::DecodeToUTF16(const uint8_t* utf8_array,
                          intptr_t array_len,
                          uint16_t* dst,
                          intptr_t len) {
   intptr_t i = 0;
   intptr_t j = 0;
   intptr_t num_bytes;
   for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
     int32_t ch;
     bool is_supplementary = IsSupplementarySequenceStart(utf8_array[i]);
     num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
     if (ch == -1) {
-      return false;  // invalid input
+      return false;  // Invalid input.
     }
     if (is_supplementary) {
       Utf16::Encode(ch, &dst[j]);
       j = j + 1;
     } else {
       dst[j] = ch;
     }
   }
   if ((i < array_len) && (j == len)) {
-    return false;  // output overflow
+    return false;  // Output overflow.
   }
-  return true;  // success
+  return true;  // Success.
 }
 
 
 bool Utf8::DecodeToUTF32(const uint8_t* utf8_array,
                          intptr_t array_len,
                          int32_t* dst,
                          intptr_t len) {
   intptr_t i = 0;
   intptr_t j = 0;
   intptr_t num_bytes;
   for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
     int32_t ch;
     num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
     if (ch == -1) {
-      return false;  // invalid input
+      return false;  // Invalid input.
     }
     dst[j] = ch;
   }
   if ((i < array_len) && (j == len)) {
-    return false;  // output overflow
+    return false;  // Output overflow.
   }
-  return true;  // success
+  return true;  // Success.
 }
 
 
 void Utf16::Encode(int32_t codepoint, uint16_t* dst) {
   ASSERT(codepoint > kMaxBmpCodepoint);
   ASSERT(dst != NULL);
   dst[0] = (Utf16::kLeadSurrogateOffset + (codepoint >> 10));
   dst[1] = (0xDC00 + (codepoint & 0x3FF));
 }
 
+
+bool Utf16::CodePointIterator::Next() {
+  ASSERT(index_ >= -1);
+  ASSERT(index_ < array_len_);
+  int d = Length(ch_);
+  if (index_ == (array_len_ - d)) {
+    return false;
+  }
+  index_ += d;
+  ch_ = utf16_array_[index_];
+  if (IsLeadSurrogate(ch_) && (index_ != (array_len_ - 1))) {
+    int32_t ch2 = utf16_array_[index_ + 1];
+    if (IsTrailSurrogate(ch2)) {
+      ch_ = Decode(ch_, ch2);
+    }
+  }
+  return true;
+}
+
 }  // namespace dart