Fix input and output to handle UTF16 surrogate pairs.

src/unicode-inl.h

 // Copyright 2007-2010 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 60 matching lines @@
     } else {
       entries_[c & kMask] = CacheEntry(c, 0);
       return 0;
     }
   } else {
     return length;
   }
 }
 
 
-unsigned Utf8::Encode(char* str, uchar c) {
+unsigned Utf8::Encode(char* str, uchar c, int previous) {
   static const int kMask = ~(1 << 6);
   if (c <= kMaxOneByteChar) {
     str[0] = c;
     return 1;
   } else if (c <= kMaxTwoByteChar) {
     str[0] = 0xC0 | (c >> 6);
     str[1] = 0x80 | (c & kMask);
     return 2;
   } else if (c <= kMaxThreeByteChar) {
+    if (Utf16::IsTrailSurrogate(c) &&
+        previous != kNoPreviousCharacter &&

[rossberg, 2012/03/07 13:32:47]

Isn't that implied by Utf16::IsLeadSurrogate(previous)?

[Erik Corry, 2012/03/11 19:29:22]

No, but it should be.  Fixed.

+        Utf16::IsLeadSurrogate(previous)) {
+      return Encode(str - 3,
+                    Utf16::CombineSurrogatePair(previous, c),
+                    Utf8::kNoPreviousCharacter) - 3;

[rossberg, 2012/03/07 13:32:47]

kSizeOfUnmatchedSurrogate instead of 3 perhaps (here and above).

[Erik Corry, 2012/03/11 19:29:22]

Done.

+    }
     str[0] = 0xE0 | (c >> 12);
     str[1] = 0x80 | ((c >> 6) & kMask);
     str[2] = 0x80 | (c & kMask);
     return 3;
   } else {
     str[0] = 0xF0 | (c >> 18);
     str[1] = 0x80 | ((c >> 12) & kMask);
     str[2] = 0x80 | ((c >> 6) & kMask);
     str[3] = 0x80 | (c & kMask);
     return 4;
   }
 }
 
 
 uchar Utf8::ValueOf(const byte* bytes, unsigned length, unsigned* cursor) {
   if (length <= 0) return kBadChar;
   byte first = bytes[0];
   // Characters between 0000 and 0007F are encoded as a single character
   if (first <= kMaxOneByteChar) {
     *cursor += 1;
     return first;
   }
   return CalculateValue(bytes, length, cursor);
 }
 
-unsigned Utf8::Length(uchar c) {
+unsigned Utf8::Length(uchar c, int previous) {
   if (c <= kMaxOneByteChar) {
     return 1;
   } else if (c <= kMaxTwoByteChar) {
     return 2;
   } else if (c <= kMaxThreeByteChar) {
+    if (Utf16::IsTrailSurrogate(c) &&
+        previous != kNoPreviousCharacter &&

[rossberg, 2012/03/07 13:32:47]

See above.

[Erik Corry, 2012/03/11 19:29:22]

Done.

+        Utf16::IsLeadSurrogate(previous)) {
+      return 1;

[rossberg, 2012/03/07 13:32:47]

This is 4 - 3 already counted, I suppose.

[Erik Corry, 2012/03/11 19:29:22]

Fixed

+    }
     return 3;
   } else {
     return 4;
   }
 }
 
 uchar CharacterStream::GetNext() {
   uchar result = DecodeCharacter(buffer_, &cursor_);
   if (remaining_ == 1) {
     cursor_ = 0;
@@ skipping 97 matching lines @@
 
 template <unsigned s>
 Utf8InputBuffer<s>::Utf8InputBuffer(const char* data, unsigned length)
     : InputBuffer<Utf8, Buffer<const char*>, s>(Buffer<const char*>(data,
                                                                     length)) {
 }
 
 }  // namespace unibrow
 
 #endif  // V8_UNICODE_INL_H_