Add support for surrogates when serializing and deserializing for native ports

runtime/vm/unicode.cc

Index: runtime/vm/unicode.cc
diff --git a/runtime/vm/unicode.cc b/runtime/vm/unicode.cc
index 3129a06787edb5cbc96dee61444c63c96a862cc1..12f17aa855053a3b14c27975cee3a31375e65ddd 100644
--- a/runtime/vm/unicode.cc
+++ b/runtime/vm/unicode.cc
@@ -1,4 +1,4 @@
-// Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
+// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
@@ -59,7 +59,7 @@ static bool IsTrailByte(uint8_t code_unit) {
 
 
 static bool IsLatin1SequenceStart(uint8_t code_unit) {
-  // Check is codepoint is <= U+00FF
+  // Check is codepoint is <= U+00FF.

[siva, 2012/11/28 18:22:46]

Check if codepoint is ...

[Søren Gjesse, 2012/11/29 09:06:14]

Done.

   return (code_unit <= Utf8::kMaxOneByteChar);
 }
 
@@ -129,8 +129,7 @@ bool Utf8::IsValid(const uint8_t* utf8_array, intptr_t array_len) {
       if (!((is_malformed == false) &&
             (j == num_trail_bytes) &&
             !IsOutOfRange(ch) &&
-            !IsNonShortestForm(ch, j) &&
-            !Utf16::IsSurrogate(ch))) {
+            !IsNonShortestForm(ch, j))) {
         return false;
       }
     }
@@ -228,8 +227,7 @@ intptr_t Utf8::Decode(const uint8_t* utf8_array,
     if (!((is_malformed == false) &&
           (i == num_trail_bytes) &&
           !IsOutOfRange(ch) &&
-          !IsNonShortestForm(ch, i) &&
-          !Utf16::IsSurrogate(ch))) {
+          !IsNonShortestForm(ch, i))) {
       *dst = -1;
       return 0;
     }
@@ -251,15 +249,15 @@ bool Utf8::DecodeToLatin1(const uint8_t* utf8_array,
     ASSERT(IsLatin1SequenceStart(utf8_array[i]));
     num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
     if (ch == -1) {
-      return false;  // invalid input
+      return false;  // Invalid input.
     }
     ASSERT(ch <= 0xff);
     dst[j] = ch;
   }
   if ((i < array_len) && (j == len)) {
-    return false;  // output overflow
+    return false;  // Output overflow.
   }
-  return true;  // success
+  return true;  // Success.
 }
 
 
@@ -275,7 +273,7 @@ bool Utf8::DecodeToUTF16(const uint8_t* utf8_array,
     bool is_supplementary = IsSupplementarySequenceStart(utf8_array[i]);
     num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
     if (ch == -1) {
-      return false;  // invalid input
+      return false;  // Invalid input.
     }
     if (is_supplementary) {
       Utf16::Encode(ch, &dst[j]);
@@ -285,9 +283,9 @@ bool Utf8::DecodeToUTF16(const uint8_t* utf8_array,
     }
   }
   if ((i < array_len) && (j == len)) {
-    return false;  // output overflow
+    return false;  // Output overflow.
   }
-  return true;  // success
+  return true;  // Success.
 }
 
 
@@ -302,14 +300,14 @@ bool Utf8::DecodeToUTF32(const uint8_t* utf8_array,
     int32_t ch;
     num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
     if (ch == -1) {
-      return false;  // invalid input
+      return false;  // Invalid input.
     }
     dst[j] = ch;
   }
   if ((i < array_len) && (j == len)) {
-    return false;  // output overflow
+    return false;  // Output overflow.
   }
-  return true;  // success
+  return true;  // Success.
 }
 
 
@@ -320,4 +318,23 @@ void Utf16::Encode(int32_t codepoint, uint16_t* dst) {
   dst[1] = (0xDC00 + (codepoint & 0x3FF));
 }
 
+
+bool Utf16::CodePointIterator::Next() {
+  ASSERT(index_ >= -1);
+  ASSERT(index_ < array_len_);
+  int d = Length(ch_);
+  if (index_ == (array_len_ - d)) {
+    return false;
+  }
+  index_ += d;
+  ch_ = utf16_array_[index_];
+  if (IsLeadSurrogate(ch_) && (index_ != (array_len_ - 1))) {
+    int32_t ch2 = utf16_array_[index_ + 1];
+    if (IsTrailSurrogate(ch2)) {
+      ch_ = Decode(ch_, ch2);
+    }
+  }
+  return true;
+}
+
 }  // namespace dart