Add support for surrogates when serializing and deserializing for native ports

runtime/vm/unicode.cc

-// Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
+// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/unicode.h"
 
 #include "vm/allocation.h"
 #include "vm/globals.h"
 #include "vm/object.h"
 
 namespace dart {
@@ skipping 40 matching lines @@
   0xFFFFFFFF
 };
 
 
 static bool IsTrailByte(uint8_t code_unit) {
   return (code_unit & 0xc0) == 0x80;
 }
 
 
 static bool IsLatin1SequenceStart(uint8_t code_unit) {
-  // Check is codepoint is <= U+00FF
+  // Check is codepoint is <= U+00FF.

[siva, 2012/11/28 18:22:46]

Check if codepoint is ...

[Søren Gjesse, 2012/11/29 09:06:14]

Done.

   return (code_unit <= Utf8::kMaxOneByteChar);
 }
 
 
 static bool IsSupplementarySequenceStart(uint8_t code_unit) {
   // Check is codepoint is >= U+10000.
   return (code_unit >= 0xF0);
 }
 
 
@@ skipping 49 matching lines @@
           is_malformed |= !IsTrailByte(code_unit);
           ch = (ch << 6) + code_unit;
         } else {
           return false;
         }
       }
       ch -= kMagicBits[num_trail_bytes];
       if (!((is_malformed == false) &&
             (j == num_trail_bytes) &&
             !IsOutOfRange(ch) &&
-            !IsNonShortestForm(ch, j) &&
-            !Utf16::IsSurrogate(ch))) {
+            !IsNonShortestForm(ch, j))) {
         return false;
       }
     }
     i += j;
   }
   return true;
 }
 
 
 intptr_t Utf8::Length(int32_t ch) {
@@ skipping 77 matching lines @@
         ch = (ch << 6) + code_unit;
       } else {
         *dst = -1;
         return 0;
       }
     }
     ch -= kMagicBits[num_trail_bytes];
     if (!((is_malformed == false) &&
           (i == num_trail_bytes) &&
           !IsOutOfRange(ch) &&
-          !IsNonShortestForm(ch, i) &&
-          !Utf16::IsSurrogate(ch))) {
+          !IsNonShortestForm(ch, i))) {
       *dst = -1;
       return 0;
     }
   }
   *dst = ch;
   return i;
 }
 
 
 bool Utf8::DecodeToLatin1(const uint8_t* utf8_array,
                           intptr_t array_len,
                           uint8_t* dst,
                           intptr_t len) {
   intptr_t i = 0;
   intptr_t j = 0;
   intptr_t num_bytes;
   for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
     int32_t ch;
     ASSERT(IsLatin1SequenceStart(utf8_array[i]));
     num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
     if (ch == -1) {
-      return false;  // invalid input
+      return false;  // Invalid input.
     }
     ASSERT(ch <= 0xff);
     dst[j] = ch;
   }
   if ((i < array_len) && (j == len)) {
-    return false;  // output overflow
+    return false;  // Output overflow.
   }
-  return true;  // success
+  return true;  // Success.
 }
 
 
 bool Utf8::DecodeToUTF16(const uint8_t* utf8_array,
                          intptr_t array_len,
                          uint16_t* dst,
                          intptr_t len) {
   intptr_t i = 0;
   intptr_t j = 0;
   intptr_t num_bytes;
   for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
     int32_t ch;
     bool is_supplementary = IsSupplementarySequenceStart(utf8_array[i]);
     num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
     if (ch == -1) {
-      return false;  // invalid input
+      return false;  // Invalid input.
     }
     if (is_supplementary) {
       Utf16::Encode(ch, &dst[j]);
       j = j + 1;
     } else {
       dst[j] = ch;
     }
   }
   if ((i < array_len) && (j == len)) {
-    return false;  // output overflow
+    return false;  // Output overflow.
   }
-  return true;  // success
+  return true;  // Success.
 }
 
 
 bool Utf8::DecodeToUTF32(const uint8_t* utf8_array,
                          intptr_t array_len,
                          int32_t* dst,
                          intptr_t len) {
   intptr_t i = 0;
   intptr_t j = 0;
   intptr_t num_bytes;
   for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
     int32_t ch;
     num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
     if (ch == -1) {
-      return false;  // invalid input
+      return false;  // Invalid input.
     }
     dst[j] = ch;
   }
   if ((i < array_len) && (j == len)) {
-    return false;  // output overflow
+    return false;  // Output overflow.
   }
-  return true;  // success
+  return true;  // Success.
 }
 
 
 void Utf16::Encode(int32_t codepoint, uint16_t* dst) {
   ASSERT(codepoint > kMaxBmpCodepoint);
   ASSERT(dst != NULL);
   dst[0] = (Utf16::kLeadSurrogateOffset + (codepoint >> 10));
   dst[1] = (0xDC00 + (codepoint & 0x3FF));
 }
 
+
+bool Utf16::CodePointIterator::Next() {
+  ASSERT(index_ >= -1);
+  ASSERT(index_ < array_len_);
+  int d = Length(ch_);
+  if (index_ == (array_len_ - d)) {
+    return false;
+  }
+  index_ += d;
+  ch_ = utf16_array_[index_];
+  if (IsLeadSurrogate(ch_) && (index_ != (array_len_ - 1))) {
+    int32_t ch2 = utf16_array_[index_ + 1];
+    if (IsTrailSurrogate(ch2)) {
+      ch_ = Decode(ch_, ch2);
+    }
+  }
+  return true;
+}
+
 }  // namespace dart