String:WriteUtf8: Add REPLACE_INVALID_UTF8 option

src/unicode.h

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 28 matching lines @@
 
 typedef unsigned int uchar;
 typedef unsigned char byte;
 
 /**
  * The max length of the result of converting the case of a single
  * character.
  */
 const int kMaxMappingSize = 4;
 
+/**
+ * The unicode replacement character, used to signal invalid unicode sequences
+ * (e.g. an orphan surrogate) when converting to a UTF encoding.

[dcarney, 2014/01/10 16:49:55]

typo - UTF-8

[haimuiba, 2014/01/13 07:48:21]

Done.

+ */
+const int kReplacementCharacter = 0xFFFD;

[dcarney, 2014/01/10 16:49:55]

this should be in Utf8, but see below

[haimuiba, 2014/01/13 07:48:21]

Done.

+
 template <class T, int size = 256>
 class Predicate {
  public:
   inline Predicate() { }
   inline bool get(uchar c);
  private:
   friend class Test;
   bool CalculateValue(uchar c);
   struct CacheEntry {
     inline CacheEntry() : code_point_(0), value_(0) { }
@@ skipping 36 matching lines @@
 
 class UnicodeData {
  private:
   friend class Test;
   static int GetByteCount();
   static const uchar kMaxCodePoint;
 };
 
 class Utf16 {
  public:
+  static inline bool IsSurrogatePair(int lead, int trail) {
+    return IsLeadSurrogate(lead) && IsTrailSurrogate(trail);
+  }
   static inline bool IsLeadSurrogate(int code) {
     if (code == kNoPreviousCharacter) return false;
     return (code & 0xfc00) == 0xd800;
   }
   static inline bool IsTrailSurrogate(int code) {
     if (code == kNoPreviousCharacter) return false;
     return (code & 0xfc00) == 0xdc00;
   }
 
   static inline int CombineSurrogatePair(uchar lead, uchar trail) {
@@ skipping 24 matching lines @@
   // Returns 0 if character does not convert to single latin-1 character
   // or if the character doesn't not convert back to latin-1 via inverse
   // operation (upper to lower, etc).
   static inline uint16_t ConvertNonLatin1ToLatin1(uint16_t);
 };
 
 class Utf8 {
  public:
   static inline uchar Length(uchar chr, int previous);
   static inline unsigned EncodeOneByte(char* out, uint8_t c);
-  static inline unsigned Encode(
-      char* out, uchar c, int previous);
+  static inline unsigned Encode(char* out,
+                                uchar c,
+                                int previous,
+                                bool allow_invalid);

[dcarney, 2014/01/10 16:49:55]

this either needs to be an enum to avoid passing true and false directly, or you
need to keep the bool but use a default value of true so existing call sites
don't change

[haimuiba, 2014/01/13 07:48:21]

Done by defaulting to true. I don't understand the requirement of not changing
existing call sites so, I assume this is just to minimize the diff of the patch?

   static uchar CalculateValue(const byte* str,
                               unsigned length,
                               unsigned* cursor);
   static const uchar kBadChar = 0xFFFD;

[dcarney, 2014/01/10 16:49:55]

hmmm, maybe you should just rename this variable here to kReplacementChar, since
that's what it is, and it might conflict with your change.
you'll obviously need to check usages of it, but i think i removed most of them
in the codebase

[haimuiba, 2014/01/13 07:48:21]

Done. Decided to keep the kBadChar name for now as it was still used in a few
places.

   static const unsigned kMaxEncodedSize   = 4;
   static const unsigned kMaxOneByteChar   = 0x7f;
   static const unsigned kMaxTwoByteChar   = 0x7ff;
   static const unsigned kMaxThreeByteChar = 0xffff;
   static const unsigned kMaxFourByteChar  = 0x1fffff;
 
   // A single surrogate is coded as a 3 byte UTF-8 sequence, but two together
   // that match are coded as a 4 byte UTF-8 sequence.
   static const unsigned kBytesSavedByCombiningSurrogates = 2;
   static const unsigned kSizeOfUnmatchedSurrogate = 3;
@@ skipping 102 matching lines @@
   static const int kMaxWidth = 1;
   static int Convert(uchar c,
                      uchar n,
                      uchar* result,
                      bool* allow_caching_ptr);
 };
 
 }  // namespace unibrow
 
 #endif  // V8_UNICODE_H_