Allow individual unicode surrogate strings

runtime/vm/unicode_test.cc

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/globals.h"
 #include "vm/unicode.h"
 #include "vm/unit_test.h"
 
 namespace dart {
 
@@ skipping 814 matching lines @@
     const char* src = "\xFC\x80\x80\x80\x80\x80";
     int32_t expected[] = { 0x0 };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0xFF, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
     EXPECT(!is_valid);
     EXPECT(memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.1 - Single UTF-16 surrogates
+  // UTF-8 suggests single surrogates are invalid, but both JS and
+  // Dart allow them and make use of them.
 
   // 5.1.1 - U+D800 = ed a0 80 = "\xED\xA0\x80"
   {
     const char* src = "\xED\xA0\x80";
     int32_t expected[] = { 0xD800 };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.1.2 - U+DB7F = ed ad bf = "\xED\xAD\xBF"
   {
     const char* src = "\xED\xAD\xBF";
     int32_t expected[] = { 0xDB7F };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.1.3 - U+DB80 = ed ae 80 = "\xED\xAE\x80"
   {
     const char* src = "\xED\xAE\x80";
     int32_t expected[] = { 0xDB80 };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.1.4 - U+DBFF = ed af bf = "\xED\xAF\xBF"
   {
     const char* src = "\xED\xAF\xBF";
     int32_t expected[] = { 0xDBFF };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.1.5 - U+DC00 = ed b0 80 = "\xED\xB0\x80"
   {
     const char* src = "\xED\xB0\x80";
     int32_t expected[] = { 0xDC00 };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.1.6 - U+DF80 = ed be 80 = "\xED\xBE\x80"
   {
     const char* src = "\xED\xBE\x80";
     int32_t expected[] = { 0xDF80 };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.1.7 - U+DFFF = ed bf bf = "\xED\xBF\xBF"
   {
     const char* src = "\xED\xBF\xBF";
     int32_t expected[] = { 0xDFFF };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.2 Paired UTF-16 surrogates
+  // Also not a valid string, but accepted in Dart, even if it doesn't make
+  // sense. e.g.
+  // var s =  new String.fromCharCodes([0xd800, 0xDC00]);
+  // print(s.runes);  // (65536) (0x10000)
+  // print(s.codeUnits); // [55296, 56320]
 
   // 5.2.1 - U+D800 U+DC00 = ed a0 80 ed b0 80 = "\xED\xA0\x80\xED\xB0\x80"
   {
     const char* src = "\xED\xA0\x80\xED\xB0\x80";
     int32_t expected[] = { 0xD800, 0xDC00 };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.2.2 - U+D800 U+DFFF = ed a0 80 ed bf bf = "\xED\xA0\x80\xED\xBF\xBF"
   {
     const char* src = "\xED\xA0\x80\xED\xBF\xBF";
     int32_t expected[] = { 0xD800, 0xDFFF };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.2.3 - U+DB7F U+DC00 = ed a0 80 ed bf bf = "\xED\xAD\xBF\xED\xB0\x80"
   {
     const char* src = "\xED\xAD\xBF\xED\xB0\x80";
     int32_t expected[] = { 0xDB7F, 0xDC00 };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.2.4 - U+DB7F U+DFFF = ed ad bf ed bf bf = "\xED\xAD\xBF\xED\xBF\xBF"
   {
     const char* src = "\xED\xAD\xBF\xED\xBF\xBF";
     int32_t expected[] = { 0xDB7F, 0xDFFF };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.2.5 - U+DB80 U+DC00 = ed ae 80 ed b0 80 = "\xED\xAE\x80\xED\xB0\x80"
   {
     const char* src = "\xED\xAE\x80\xED\xB0\x80";
     int32_t expected[] = { 0xDB80, 0xDC00 };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.2.6 - U+DB80 U+DFFF = ed ae 80 ed bf bf = "\xED\xAE\x80\xED\xBF\xBF"
   {
     const char* src = "\xED\xAE\x80\xED\xBF\xBF";
     int32_t expected[] = { 0xDB80, 0xDFFF };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.2.7 - U+DBFF U+DC00 = ed af bf ed b0 80 = "\xED\xAF\xBF\xED\xB0\x80"
   {
     const char* src = "\xED\xAF\xBF\xED\xB0\x80";
     int32_t expected[] = { 0xDBFF, 0xDC00 };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.2.8 - U+DBFF U+DFFF = ed af bf ed bf bf = "\xED\xAF\xBF\xED\xBF\xBF"
   {
     const char* src = "\xED\xAF\xBF\xED\xBF\xBF";
     int32_t expected[] = { 0xDBFF, 0xDFFF };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
-    EXPECT(!is_valid);
-    EXPECT(memcmp(expected, dst, sizeof(expected)));
+    EXPECT(is_valid);
+    EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.3 - Other illegal code positions
 
   // 5.3.1 - U+FFFE = ef bf be = "\xEF\xBF\xBE"
   {
     const char* src = "\xEF\xBF\xBE";
     int32_t expected[] = { 0xFFFE };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
     EXPECT(is_valid);
     EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 
   // 5.3.2 - U+FFFF = ef bf bf = "\xEF\xBF\xBF"
   {
     const char* src = "\xEF\xBF\xBF";
     int32_t expected[] = { 0xFFFF };
     int32_t dst[ARRAY_SIZE(expected)];
     memset(dst, 0, sizeof(dst));
     bool is_valid = Utf8::DecodeCStringToUTF32(src, dst, ARRAY_SIZE(dst));
     EXPECT(is_valid);
     EXPECT(!memcmp(expected, dst, sizeof(expected)));
   }
 }
 
 }  // namespace dart