Add support for surrogates when serializing and deserializing for native ports

runtime/vm/unicode.cc

Index: runtime/vm/unicode.cc
diff --git a/runtime/vm/unicode.cc b/runtime/vm/unicode.cc
index 3129a06787edb5cbc96dee61444c63c96a862cc1..f7230bf4335afa5f69bdd3a124bd6a0f3f30c286 100644
--- a/runtime/vm/unicode.cc
+++ b/runtime/vm/unicode.cc
@@ -31,7 +31,7 @@ static const int8_t kTrailBytes[256] = {
 
 
 static const uint32_t kMagicBits[7] = {
-  0,  // padding
+  0,  // Padding.
   0x00000000,
   0x00003080,
   0x000E2080,
@@ -43,7 +43,7 @@ static const uint32_t kMagicBits[7] = {
 
 // Minimum values of code points used to check shortest form.
 static const uint32_t kOverlongMinimum[7] = {
-  0,  // padding
+  0,  // Padding.
   0x0,
   0x80,
   0x800,
@@ -59,13 +59,13 @@ static bool IsTrailByte(uint8_t code_unit) {
 
 
 static bool IsLatin1SequenceStart(uint8_t code_unit) {
-  // Check is codepoint is <= U+00FF
+  // Check if codepoint is <= U+00FF.
   return (code_unit <= Utf8::kMaxOneByteChar);
 }
 
 
 static bool IsSupplementarySequenceStart(uint8_t code_unit) {
-  // Check is codepoint is >= U+10000.
+  // Check if codepoint is >= U+10000.
   return (code_unit >= 0xF0);
 }
 
@@ -93,8 +93,8 @@ intptr_t Utf8::CodePointCount(const uint8_t* utf8_array,
     if (!IsTrailByte(code_unit)) {
       ++len;
     }
-    if (!IsLatin1SequenceStart(code_unit)) {  // > U+00FF
-      if (IsSupplementarySequenceStart(code_unit)) {  // >= U+10000
+    if (!IsLatin1SequenceStart(code_unit)) {  // > U+00FF.

[cshapiro, 2012/11/30 02:49:08]

no period

[Søren Gjesse, 2012/11/30 12:23:07]

Done.

+      if (IsSupplementarySequenceStart(code_unit)) {  // >= U+10000.

[cshapiro, 2012/11/30 02:49:08]

ditto

[Søren Gjesse, 2012/11/30 12:23:07]

Done.

         char_type = kSupplementary;
         ++len;
       } else if (char_type == kLatin1) {
@@ -165,6 +165,7 @@ intptr_t Utf8::Length(const String& str) {
 
 
 intptr_t Utf8::Encode(int32_t ch, char* dst) {
+  ASSERT(!Utf16::IsSurrogate(ch));
   static const int kMask = ~(1 << 6);
   if (ch <= kMaxOneByteChar) {
     dst[0] = ch;
@@ -251,15 +252,15 @@ bool Utf8::DecodeToLatin1(const uint8_t* utf8_array,
     ASSERT(IsLatin1SequenceStart(utf8_array[i]));
     num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
     if (ch == -1) {
-      return false;  // invalid input
+      return false;  // Invalid input.
     }
     ASSERT(ch <= 0xff);
     dst[j] = ch;
   }
   if ((i < array_len) && (j == len)) {
-    return false;  // output overflow
+    return false;  // Output overflow.
   }
-  return true;  // success
+  return true;  // Success.
 }
 
 
@@ -275,7 +276,7 @@ bool Utf8::DecodeToUTF16(const uint8_t* utf8_array,
     bool is_supplementary = IsSupplementarySequenceStart(utf8_array[i]);
     num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
     if (ch == -1) {
-      return false;  // invalid input
+      return false;  // Invalid input.
     }
     if (is_supplementary) {
       Utf16::Encode(ch, &dst[j]);
@@ -285,9 +286,9 @@ bool Utf8::DecodeToUTF16(const uint8_t* utf8_array,
     }
   }
   if ((i < array_len) && (j == len)) {
-    return false;  // output overflow
+    return false;  // Output overflow.
   }
-  return true;  // success
+  return true;  // Success.
 }
 
 
@@ -302,14 +303,14 @@ bool Utf8::DecodeToUTF32(const uint8_t* utf8_array,
     int32_t ch;
     num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
     if (ch == -1) {
-      return false;  // invalid input
+      return false;  // Invalid input.
     }
     dst[j] = ch;
   }
   if ((i < array_len) && (j == len)) {
-    return false;  // output overflow
+    return false;  // Output overflow.
   }
-  return true;  // success
+  return true;  // Success.
 }
 
 
@@ -320,4 +321,23 @@ void Utf16::Encode(int32_t codepoint, uint16_t* dst) {
   dst[1] = (0xDC00 + (codepoint & 0x3FF));
 }
 
+
+bool Utf16::CodePointIterator::Next() {
+  ASSERT(index_ >= -1);
+  ASSERT(index_ < array_len_);
+  int d = Length(ch_);
+  if (index_ == (array_len_ - d)) {
+    return false;
+  }
+  index_ += d;
+  ch_ = utf16_array_[index_];
+  if (IsLeadSurrogate(ch_) && (index_ != (array_len_ - 1))) {
+    int32_t ch2 = utf16_array_[index_ + 1];
+    if (IsTrailSurrogate(ch2)) {
+      ch_ = Decode(ch_, ch2);
+    }
+  }
+  return true;
+}
+
 }  // namespace dart