Add support for surrogates when serializing and deserializing for native ports

runtime/vm/unicode_test.cc

-// Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
+// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/globals.h"
 #include "vm/unicode.h"
 #include "vm/unit_test.h"
 
 namespace dart {
 
 TEST_CASE(Utf8Decode) {
@@ skipping 821 matching lines @@
 
   // 5.1 - Single UTF-16 surrogates
 
   // 5.1.1 - U+D800 = ed a0 80 = "\xED\xA0\x80"
   {
     const char* src = "\xED\xA0\x80";
     int32_t expected[] = { 0xD800 };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.1.2 - U+DB7F = ed ad bf = "\xED\xAD\xBF"
   {
     const char* src = "\xED\xAD\xBF";
     int32_t expected[] = { 0xDB7F };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.1.3 - U+DB80 = ed ae 80 = "\xED\xAE\x80"
   {
     const char* src = "\xED\xAE\x80";
     int32_t expected[] = { 0xDB80 };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.1.4 - U+DBFF = ed af bf = "\xED\xAF\xBF"
   {
     const char* src = "\xED\xAF\xBF";
     int32_t expected[] = { 0xDBFF };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.1.5 - U+DC00 = ed b0 80 = "\xED\xB0\x80"
   {
     const char* src = "\xED\xB0\x80";
     int32_t expected[] = { 0xDC00 };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.1.6 - U+DF80 = ed be 80 = "\xED\xBE\x80"
   {
     const char* src = "\xED\xBE\x80";
     int32_t expected[] = { 0xDF80 };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.1.7 - U+DFFF = ed bf bf = "\xED\xBF\xBF"
   {
     const char* src = "\xED\xBF\xBF";
     int32_t expected[] = { 0xDFFF };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.2 Paired UTF-16 surrogates
 
   // 5.2.1 - U+D800 U+DC00 = ed a0 80 ed b0 80 = "\xED\xA0\x80\xED\xB0\x80"
   {
     const char* src = "\xED\xA0\x80\xED\xB0\x80";
     int32_t expected[] = { 0xD800, 0xDC00 };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.2.2 - U+D800 U+DFFF = ed a0 80 ed bf bf = "\xED\xA0\x80\xED\xBF\xBF"
   {
     const char* src = "\xED\xA0\x80\xED\xBF\xBF";
     int32_t expected[] = { 0xD800, 0xDFFF };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.2.3 - U+DB7F U+DC00 = ed a0 80 ed bf bf = "\xED\xAD\xBF\xED\xB0\x80"
   {
     const char* src = "\xED\xAD\xBF\xED\xB0\x80";
     int32_t expected[] = { 0xDB7F, 0xDC00 };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.2.4 - U+DB7F U+DFFF = ed ad bf ed bf bf = "\xED\xAD\xBF\xED\xBF\xBF"
   {
     const char* src = "\xED\xAD\xBF\xED\xBF\xBF";
     int32_t expected[] = { 0xDB7F, 0xDFFF };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.2.5 - U+DB80 U+DC00 = ed ae 80 ed b0 80 = "\xED\xAE\x80\xED\xB0\x80"
   {
     const char* src = "\xED\xAE\x80\xED\xB0\x80";
     int32_t expected[] = { 0xDB80, 0xDC00 };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.2.6 - U+DB80 U+DFFF = ed ae 80 ed bf bf = "\xED\xAE\x80\xED\xBF\xBF"
   {
     const char* src = "\xED\xAE\x80\xED\xBF\xBF";
     int32_t expected[] = { 0xDB80, 0xDFFF };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.2.7 - U+DBFF U+DC00 = ed af bf ed b0 80 = "\xED\xAF\xBF\xED\xB0\x80"
   {
     const char* src = "\xED\xAF\xBF\xED\xB0\x80";
     int32_t expected[] = { 0xDBFF, 0xDC00 };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.2.8 - U+DBFF U+DFFF = ed af bf ed bf bf = "\xED\xAF\xBF\xED\xBF\xBF"
   {
     const char* src = "\xED\xAF\xBF\xED\xBF\xBF";
     int32_t expected[] = { 0xDBFF, 0xDFFF };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.3 - Other illegal code positions
 
   // 5.3.1 - U+FFFE = ef bf be = "\xEF\xBF\xBE"
   {
     const char* src = "\xEF\xBF\xBE";
     int32_t expected[] = { 0xFFFE };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
     EXPECT(is_valid);
     EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.3.2 - U+FFFF = ef bf bf = "\xEF\xBF\xBF"
   {
     const char* src = "\xEF\xBF\xBF";
     int32_t expected[] = { 0xFFFF };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
     EXPECT(is_valid);
     EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 }
 
+
+TEST_CASE(Utf16Iterator) {
+  // Normal code units.
+  {
+    const int len = 4;
+    const uint16_t src[] = { 32, 33, 255, 256 };
+    Utf16::CodePointIterator it(src, len);
+    for (int i = 0; i < len; i++) {
+      EXPECT(it.Next());
+      EXPECT_EQ(src[i], it.Current());
+    }
+    EXPECT(!it.Next());
+  }
+
+  // Surrogate pairs.
+  {
+    const uint16_t src[] = { 0xd800, 0xdc00, 0xd800, 0xdc01 };
+    Utf16::CodePointIterator it(src, 4);
+    EXPECT(it.Next());
+    EXPECT_EQ(0x10000, it.Current());
+    EXPECT(it.Next());
+    EXPECT_EQ(0x10001, it.Current());
+    EXPECT(!it.Next());
+  }
+
+  // Surrogate characters not in pairs.
+  {
+    const uint16_t src[] = { 0xdc00, 0xd800 };
+    Utf16::CodePointIterator it(src, 2);
+    EXPECT(it.Next());
+    EXPECT_EQ(0xdc00, it.Current());
+    EXPECT(it.Next());
+    EXPECT_EQ(0xd800, it.Current());
+    EXPECT(!it.Next());
+  }
+}
+
 }  // namespace dart