String:WriteUtf8: Add REPLACE_INVALID_UTF8 option

src/unicode.h

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 84 matching lines @@
 
 class UnicodeData {
  private:
   friend class Test;
   static int GetByteCount();
   static const uchar kMaxCodePoint;
 };
 
 class Utf16 {
  public:
+  static inline bool IsSurrogatePair(int lead, int trail) {
+    return IsLeadSurrogate(lead) && IsTrailSurrogate(trail);
+  }
   static inline bool IsLeadSurrogate(int code) {
     if (code == kNoPreviousCharacter) return false;
     return (code & 0xfc00) == 0xd800;
   }
   static inline bool IsTrailSurrogate(int code) {
     if (code == kNoPreviousCharacter) return false;
     return (code & 0xfc00) == 0xdc00;
   }
 
   static inline int CombineSurrogatePair(uchar lead, uchar trail) {
@@ skipping 24 matching lines @@
   // Returns 0 if character does not convert to single latin-1 character
   // or if the character doesn't not convert back to latin-1 via inverse
   // operation (upper to lower, etc).
   static inline uint16_t ConvertNonLatin1ToLatin1(uint16_t);
 };
 
 class Utf8 {
  public:
   static inline uchar Length(uchar chr, int previous);
   static inline unsigned EncodeOneByte(char* out, uint8_t c);
-  static inline unsigned Encode(
-      char* out, uchar c, int previous);
+  static inline unsigned Encode(char* out,
+                                uchar c,
+                                int previous,
+                                bool allow_invalid);

[dcarney, 2014/01/13 09:19:56]

needs default value

[haimuiba, 2014/01/15 10:52:34]

Done.

   static uchar CalculateValue(const byte* str,
                               unsigned length,
                               unsigned* cursor);
+
+  // The unicode replacement character, used to signal invalid unicode
+  // sequences (e.g. an orphan surrogate) when converting to a UTF-8 encoding.
   static const uchar kBadChar = 0xFFFD;
   static const unsigned kMaxEncodedSize   = 4;
   static const unsigned kMaxOneByteChar   = 0x7f;
   static const unsigned kMaxTwoByteChar   = 0x7ff;
   static const unsigned kMaxThreeByteChar = 0xffff;
   static const unsigned kMaxFourByteChar  = 0x1fffff;
 
   // A single surrogate is coded as a 3 byte UTF-8 sequence, but two together
   // that match are coded as a 4 byte UTF-8 sequence.
   static const unsigned kBytesSavedByCombiningSurrogates = 2;
   static const unsigned kSizeOfUnmatchedSurrogate = 3;
+  // The maximum size a single UTF-16 code unit may take up when encoded as
+  // UTF-8.
+  static const unsigned kMax16BitCodeUnitSize  = 3;
   static inline uchar ValueOf(const byte* str,
                               unsigned length,
                               unsigned* cursor);
 };
 
 
 class Utf8DecoderBase {
  public:
   // Initialization done in subclass.
   inline Utf8DecoderBase();
@@ skipping 92 matching lines @@
   static const int kMaxWidth = 1;
   static int Convert(uchar c,
                      uchar n,
                      uchar* result,
                      bool* allow_caching_ptr);
 };
 
 }  // namespace unibrow
 
 #endif  // V8_UNICODE_H_