Update UTF-8 decoder to detect more special cases.

test/cctest/test-parsing.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 681 matching lines @@
   static const int kAllUtf8CharsSize =
       (unibrow::Utf8::kMaxOneByteChar + 1) +
       (unibrow::Utf8::kMaxTwoByteChar - unibrow::Utf8::kMaxOneByteChar) * 2 +
       (unibrow::Utf8::kMaxThreeByteChar - unibrow::Utf8::kMaxTwoByteChar) * 3;
   static const unsigned kAllUtf8CharsSizeU =
       static_cast<unsigned>(kAllUtf8CharsSize);
 
   char buffer[kAllUtf8CharsSizeU];
   unsigned cursor = 0;
   for (int i = 0; i <= kMaxUC16Char; i++) {
-    cursor += unibrow::Utf8::Encode(buffer + cursor,
-                                    i,
-                                    unibrow::Utf16::kNoPreviousCharacter);
+    cursor += unibrow::Utf8::Encode(buffer + cursor, i,
+                                    unibrow::Utf16::kNoPreviousCharacter, true);
   }
   DCHECK(cursor == kAllUtf8CharsSizeU);
 
   i::Utf8ToUtf16CharacterStream stream(reinterpret_cast<const i::byte*>(buffer),
                                        kAllUtf8CharsSizeU);
+  int32_t bad = unibrow::Utf8::kBadChar;
   for (int i = 0; i <= kMaxUC16Char; i++) {
     CHECK_EQU(i, stream.pos());
     int32_t c = stream.Advance();
-    CHECK_EQ(i, c);
+    if (i >= 0xd800 && i <= 0xdfff) {
+      CHECK_EQ(bad, c);
+    } else {
+      CHECK_EQ(i, c);
+    }
     CHECK_EQU(i + 1, stream.pos());
   }
   for (int i = kMaxUC16Char; i >= 0; i--) {
     CHECK_EQU(i + 1, stream.pos());
     stream.PushBack(i);
     CHECK_EQU(i, stream.pos());
   }
   int i = 0;
   while (stream.pos() < kMaxUC16CharU) {
     CHECK_EQU(i, stream.pos());
     int progress = static_cast<int>(stream.SeekForward(12));
     i += progress;
     int32_t c = stream.Advance();
-    if (i <= kMaxUC16Char) {
+    if (i >= 0xd800 && i <= 0xdfff) {
+      CHECK_EQ(bad, c);
+    } else if (i <= kMaxUC16Char) {
       CHECK_EQ(i, c);
     } else {
       CHECK_EQ(-1, c);
     }
     i += 1;
     CHECK_EQU(i, stream.pos());
   }
 }
 
 #undef CHECK_EQU
@@ skipping 168 matching lines @@
         }
         input_offset = 3;
         // 4 bytes of UTF-8 turn into 2 UTF-16 code units.
         character_length -= 2;
       } else if (c >= 0xe0) {
         if ((c & 0xf) == 0 && ((s[i + 1] & 0x20) == 0)) {
           // This 3 byte sequence could have been coded as a 2 byte sequence.
           // Record a single kBadChar for the first byte and continue.
           continue;
         }
+        if (c == 0xed) {
+          unsigned char d = s[i + 1];
+          if ((d < 0x80) || (d > 0x9f)) {
+            // This 3 byte sequence is part of a surrogate pair which is not
+            // supported by UTF-8. Record a single kBadChar for the first byte
+            // and continue.
+            continue;
+          }
+        }
         input_offset = 2;
         // 3 bytes of UTF-8 turn into 1 UTF-16 code unit.
         output_adjust = 2;
       } else {
         if ((c & 0x1e) == 0) {
           // This 2 byte sequence could have been coded as a 1 byte sequence.
           // Record a single kBadChar for the first byte and continue.
           continue;
         }
         input_offset = 1;
@@ skipping 5663 matching lines @@
     "[a, ...]",
     "[..., ]",
     "[..., ...]",
     "[ (...a)]",
     NULL};
   // clang-format on
   static const ParserFlag always_flags[] = {kAllowHarmonySpreadArrays};
   RunParserSyncTest(context_data, data, kError, NULL, 0, always_flags,
                     arraysize(always_flags));
 }