Preserve the contents of Dart strings with unmatched surrogate halfs by avoiding a UTF16 -> UTF8

runtime/platform/json.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "platform/json.h"
 
 #include "platform/assert.h"
 #include "platform/globals.h"
 #include "platform/utils.h"
 #include "vm/os.h"
@@ skipping 509 matching lines @@
 
 
 void TextBuffer::AddChar(char ch) {
   EnsureCapacity(sizeof(ch));
   buf_[msg_len_] = ch;
   msg_len_++;
   buf_[msg_len_] = '\0';
 }
 
 
-void TextBuffer::AddUTF8(uint32_t ch) {
-  static const uint32_t kMaxOneByteChar   = 0x7F;
-  static const uint32_t kMaxTwoByteChar   = 0x7FF;
-  static const uint32_t kMaxThreeByteChar = 0xFFFF;
-  static const uint32_t kMaxFourByteChar  = 0x10FFFF;
-  static const uint32_t kMask = ~(1 << 6);
-
-  if (ch <= kMaxOneByteChar) {
-    EnsureCapacity(1);
-    buf_[msg_len_++] = ch;
-    buf_[msg_len_] = '\0';
-    return;
-  }
-  if (ch <= kMaxTwoByteChar) {
-    EnsureCapacity(2);
-    buf_[msg_len_++] = 0xC0 | (ch >> 6);
-    buf_[msg_len_++] = 0x80 | (ch & kMask);
-    buf_[msg_len_] = '\0';
-    return;
-  }
-  if (ch <= kMaxThreeByteChar) {
-    EnsureCapacity(3);
-    buf_[msg_len_++] = 0xE0 | (ch >> 12);
-    buf_[msg_len_++] = 0x80 | ((ch >> 6) & kMask);
-    buf_[msg_len_++] = 0x80 | (ch & kMask);
-    buf_[msg_len_] = '\0';
-    return;
-  }
-  ASSERT(ch <= kMaxFourByteChar);
-  EnsureCapacity(4);
-  buf_[msg_len_++] = 0xF0 | (ch >> 18);
-  buf_[msg_len_++] = 0x80 | ((ch >> 12) & kMask);
-  buf_[msg_len_++] = 0x80 | ((ch >> 6) & kMask);
-  buf_[msg_len_++] = 0x80 | (ch & kMask);
-  buf_[msg_len_] = '\0';
-}
-
-
 intptr_t TextBuffer::Printf(const char* format, ...) {
   va_list args;
   va_start(args, format);
   intptr_t remaining = buf_size_ - msg_len_;
   ASSERT(remaining >= 0);
   intptr_t len = OS::VSNPrint(buf_ + msg_len_, remaining, format, args);
   va_end(args);
   if (len >= remaining) {
     EnsureCapacity(len);
     remaining = buf_size_ - msg_len_;
     ASSERT(remaining > len);
     va_list args2;
     va_start(args2, format);
     intptr_t len2 = OS::VSNPrint(buf_ + msg_len_, remaining, format, args2);
     va_end(args2);
     ASSERT(len == len2);
   }
   msg_len_ += len;
   buf_[msg_len_] = '\0';
   return len;
 }
 
 
-void TextBuffer::AddEscapedChar(uint32_t cp) {
-  switch (cp) {
+// Write a UTF-16 code unit as

[koda, 2014/09/10 21:42:07]

Finish this comment.

[rmacnak, 2014/09/10 21:49:59]

Done.

+void TextBuffer::EscapeAndAddCodeUnit(uint32_t codeunit) {

[turnidge, 2014/09/10 19:55:28]

Can you take a look at json_test and see if it needs a bit of beefing up?  I'm
seeing the JSON_JSONStream_EscapedString test in there, but it does seem to
exercise the unicode path.

[rmacnak, 2014/09/10 21:38:49]

Added.

+  switch (codeunit) {
     case '"':
       Printf("%s", "\\\"");
       break;
     case '\\':
       Printf("%s", "\\\\");
       break;
     case '/':
       Printf("%s", "\\/");
       break;
     case '\b':
       Printf("%s", "\\b");
       break;
     case '\f':
       Printf("%s", "\\f");
       break;
     case '\n':
       Printf("%s", "\\n");
       break;
     case '\r':
       Printf("%s", "\\r");
       break;
     case '\t':
       Printf("%s", "\\t");
       break;
     default:
-      if (cp < 0x20) {
+      if (codeunit < 0x20) {
         // Encode character as \u00HH.
-        uint32_t digit2 = (cp >> 4) & 0xf;
-        uint32_t digit3 = (cp & 0xf);
+        uint32_t digit2 = (codeunit >> 4) & 0xf;
+        uint32_t digit3 = (codeunit & 0xf);
         Printf("\\u00%c%c",
                digit2 > 9 ? 'A' + (digit2 - 10) : '0' + digit2,
                digit3 > 9 ? 'A' + (digit3 - 10) : '0' + digit3);
-      } else {
-        AddUTF8(cp);
+      } else if (codeunit > 127) {
+        // Encode character as \uHHHH.
+        uint32_t digit1 = (codeunit >> 12) & 0xf;
+        uint32_t digit2 = (codeunit >> 8) & 0xf;
+        uint32_t digit3 = (codeunit >> 4) & 0xf;
+        uint32_t digit4 = (codeunit & 0xf);
+        Printf("\\u%c%c%c%c",
+               digit1 > 9 ? 'A' + (digit1 - 10) : '0' + digit1,
+               digit2 > 9 ? 'A' + (digit2 - 10) : '0' + digit2,
+               digit3 > 9 ? 'A' + (digit3 - 10) : '0' + digit3,
+               digit4 > 9 ? 'A' + (digit4 - 10) : '0' + digit4);
+      } else  {
+        AddChar(codeunit);
       }
   }
 }
 
 
 void TextBuffer::AddString(const char* s) {
   Printf("%s", s);
 }
 
 
 void TextBuffer::AddEscapedString(const char* s) {
   intptr_t len = strlen(s);
   for (int i = 0; i < len; i++) {
-    AddEscapedChar(s[i]);
+    EscapeAndAddCodeUnit(s[i]);
   }
 }
 
 
 void TextBuffer::EnsureCapacity(intptr_t len) {
   intptr_t remaining = buf_size_ - msg_len_;
   if (remaining <= len) {
     const int kBufferSpareCapacity = 64;  // Somewhat arbitrary.
     // TODO(turnidge): do we need to guard against overflow or other
     // security issues here? Text buffers are used by the debugger
     // to send user-controlled data (e.g. values of string variables) to
     // the debugger front-end.
     intptr_t new_size = buf_size_ + len + kBufferSpareCapacity;
     char* new_buf = reinterpret_cast<char*>(realloc(buf_, new_size));
     ASSERT(new_buf != NULL);
     buf_ = new_buf;
     buf_size_ = new_size;
   }
 }
 
 }  // namespace dart