String:WriteUtf8: Add REPLACE_INVALID_UTF8 option

src/unicode.h

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 28 matching lines @@
 
 typedef unsigned int uchar;
 typedef unsigned char byte;
 
 /**
  * The max length of the result of converting the case of a single
  * character.
  */
 const int kMaxMappingSize = 4;
 
+/**
+ * The unicode replacement character, used to signal invalid unicode sequences
+ * (e.g. an orphan surrogate) when converting to a UTF encoding.
+ */
+const int kReplacementCharacter = 0xFFFD;

[dcarney, 2014/01/04 15:56:45]

should probably be in the Utf8 class

[haimuiba, 2014/01/06 05:40:18]

I don't feel strongly about it, but technically it's not directly related to
UTF-8, but rather provided for all unicode encoders/decoders that might need it.

+
 template <class T, int size = 256>
 class Predicate {
  public:
   inline Predicate() { }
   inline bool get(uchar c);
  private:
   friend class Test;
   bool CalculateValue(uchar c);
   struct CacheEntry {
     inline CacheEntry() : code_point_(0), value_(0) { }
@@ skipping 36 matching lines @@
 
 class UnicodeData {
  private:
   friend class Test;
   static int GetByteCount();
   static const uchar kMaxCodePoint;
 };
 
 class Utf16 {
  public:
+  static inline bool IsSurrogatePair(int lead, int trail) {
+    return IsLeadSurrogate(lead) && IsTrailSurrogate(trail);
+  }
   static inline bool IsLeadSurrogate(int code) {
     if (code == kNoPreviousCharacter) return false;
     return (code & 0xfc00) == 0xd800;
   }
   static inline bool IsTrailSurrogate(int code) {
     if (code == kNoPreviousCharacter) return false;
     return (code & 0xfc00) == 0xdc00;
   }
 
   static inline int CombineSurrogatePair(uchar lead, uchar trail) {
@@ skipping 24 matching lines @@
   // Returns 0 if character does not convert to single latin-1 character
   // or if the character doesn't not convert back to latin-1 via inverse
   // operation (upper to lower, etc).
   static inline uint16_t ConvertNonLatin1ToLatin1(uint16_t);
 };
 
 class Utf8 {
  public:
   static inline uchar Length(uchar chr, int previous);
   static inline unsigned EncodeOneByte(char* out, uint8_t c);
-  static inline unsigned Encode(
-      char* out, uchar c, int previous);
+  static inline unsigned Encode(char* str, uchar c, bool replace_surrogates);

[dcarney, 2014/01/04 15:56:45]

this is not backwards compatible.  All existing call sites which you did not cut
over are casting int to bool for the third argument

[haimuiba, 2014/01/06 05:40:18]

Is there a good reason to keep compatibility, rather than changing the call
sites?

   static uchar CalculateValue(const byte* str,
                               unsigned length,
                               unsigned* cursor);
   static const uchar kBadChar = 0xFFFD;
   static const unsigned kMaxEncodedSize   = 4;
   static const unsigned kMaxOneByteChar   = 0x7f;
   static const unsigned kMaxTwoByteChar   = 0x7ff;
   static const unsigned kMaxThreeByteChar = 0xffff;
   static const unsigned kMaxFourByteChar  = 0x1fffff;
 
@@ skipping 106 matching lines @@
   static const int kMaxWidth = 1;
   static int Convert(uchar c,
                      uchar n,
                      uchar* result,
                      bool* allow_caching_ptr);
 };
 
 }  // namespace unibrow
 
 #endif  // V8_UNICODE_H_