Add basic support for Latin1 strings to the API.

include/v8.h

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 1051 matching lines @@
    *    terminator.  For WriteUtf8: The number of bytes copied to the buffer
    *    including the null terminator (if written).
    */
   enum WriteOptions {
     NO_OPTIONS = 0,
     HINT_MANY_WRITES_EXPECTED = 1,
     NO_NULL_TERMINATION = 2,
     PRESERVE_ASCII_NULL = 4
   };
 
+
+  enum StringEncoding {
+    INVALID_ENCODING  = 0,
+    UTF8_ENCODING     = 1,
+    LATIN1_ENCODING   = 2,
+    UTF16_ENCODING    = 3,
+
+    STRICT_ASCII_HINT = 1 << 16,
+    NOT_ASCII_HINT    = 1 << 17
+  };
+
+  static const int kStringEncodingMask = 3;
+  static const int kAsciiHintMask =
+      String::STRICT_ASCII_HINT | String::NOT_ASCII_HINT;
+
+  static const int kUndefinedLength = -1;
+
+
   // 16-bit character codes.
   V8EXPORT int Write(uint16_t* buffer,
                      int start = 0,
-                     int length = -1,
+                     int length = kUndefinedLength,
                      int options = NO_OPTIONS) const;
   // ASCII characters.
   V8EXPORT int WriteAscii(char* buffer,
                           int start = 0,
-                          int length = -1,
+                          int length = kUndefinedLength,
                           int options = NO_OPTIONS) const;
+  // Latin1 characters.
+  V8EXPORT int WriteLatin1(char* buffer,
+                           int start = 0,
+                           int length = kUndefinedLength,
+                           int options = NO_OPTIONS) const;
+
   // UTF-8 encoded characters.
   V8EXPORT int WriteUtf8(char* buffer,
-                         int length = -1,
+                         int length = kUndefinedLength,
                          int* nchars_ref = NULL,
                          int options = NO_OPTIONS) const;
 
   /**
    * A zero length string.
    */
   V8EXPORT static v8::Local<v8::String> Empty();
   inline static v8::Local<v8::String> Empty(Isolate* isolate);
 
   /**
@@ skipping 20 matching lines @@
      * control how allocated external string resources are disposed.
      */
     virtual void Dispose() { delete this; }
 
    private:
     // Disallow copying and assigning.
     ExternalStringResourceBase(const ExternalStringResourceBase&);
     void operator=(const ExternalStringResourceBase&);
 
     friend class v8::internal::Heap;
+    friend class v8::String;
   };
 
   /**
    * An ExternalStringResource is a wrapper around a two-byte string
    * buffer that resides outside V8's heap. Implement an
    * ExternalStringResource to manage the life cycle of the underlying
    * buffer.  Note that the string data must be immutable.
    */
   class V8EXPORT ExternalStringResource
       : public ExternalStringResourceBase {
@@ skipping 39 matching lines @@
     virtual ~ExternalAsciiStringResource() {}
     /** The string data from the underlying buffer.*/
     virtual const char* data() const = 0;
     /** The number of ASCII characters in the string.*/
     virtual size_t length() const = 0;
    protected:
     ExternalAsciiStringResource() {}
   };
 
   /**
+   * An ExternalLatin1StringResource is a wrapper around an Latin1-encoded
+   * string buffer that resides outside V8's heap.  For usage in V8, Latin1
+   * strings are converted to ASCII or two-byte string depending on whether
+   * the string contains non-ASCII characters.
+   */
+  class V8EXPORT ExternalLatin1StringResource
+      : public ExternalAsciiStringResource {
+  };
+
+  /**
    * Get the ExternalStringResource for an external string.  Returns
    * NULL if IsExternal() doesn't return true.
    */
   inline ExternalStringResource* GetExternalStringResource() const;
 
   /**
    * Get the ExternalAsciiStringResource for an external ASCII string.
    * Returns NULL if IsExternalAscii() doesn't return true.
    */
   V8EXPORT const ExternalAsciiStringResource* GetExternalAsciiStringResource()
       const;
 
+  /**
+   * If the string is external, return the its encoding (Latin1, UTF8 or UTF16)
+   * and possibly a hint on whether the content is ASCII.

[Erik Corry, 2012/08/10 11:28:51]

Comment seems wrong as it can't return a UTF8 encoding.

[Yang, 2012/08/10 13:14:45]

Done.

+   * Return String::INVALID_ENCODING otherwise.
+   */
+  inline int GetExternalStringEncoding() const;
+
+
+  /**
+   * Return the resource of the external string regardless of encoding.
+   * Call this only after having made sure that the string is indeed external!
+   */
+  inline ExternalStringResourceBase* GetExternalStringResourceBase() const;
+
   static inline String* Cast(v8::Value* obj);
 
   /**
-   * Allocates a new string from either UTF-8 encoded or ASCII data.
-   * The second parameter 'length' gives the buffer length.
-   * If the data is UTF-8 encoded, the caller must
-   * be careful to supply the length parameter.
-   * If it is not given, the function calls
-   * 'strlen' to determine the buffer length, it might be
-   * wrong if 'data' contains a null character.
+   * Allocates a new string from either UTF-8-, Latin1-encoded data.
+   * The second parameter 'length' gives the buffer length.  If the data is
+   * UTF-8 encoded, the caller must be careful to supply the length parameter.
+   * If it is not given, the function calls 'strlen' to determine the buffer
+   * length, it might be wrong if 'data' contains a null character.
+   * The third parameter specifies the encoding, which may include an hint
+   * whether the string contains ASCII characters.  In case of Latin1, the
+   * appropriate internal representation (UTF16 or ASCII) is chosen.
    */
-  V8EXPORT static Local<String> New(const char* data, int length = -1);
+  V8EXPORT static Local<String> New(const char* data,
+                                    int length = kUndefinedLength,
+                                    int encoding = UTF8_ENCODING);

[Erik Corry, 2012/08/10 11:28:51]

No point in making the second argument optional.  There's no way to call this
with only 1 argument because in that case the encoding is UTF8 and you should
have given a length.  Perhaps reorder the arguments?  Here and other places in
this file.

[Yang, 2012/08/10 13:14:45]

I see your point here.  But then I also feel a certain obligation to maintain
backward-compatibility... I could set the default encoding to LATIN1_ENCODING |
STRICT_ASCII_HINT, but this may still break things for embedders that don't
update their code.

 
   /** Allocates a new string from 16-bit character codes.*/

[Erik Corry, 2012/08/10 11:28:51]

Not your error, but:
16-bit character codes -> 16-bit UTF16 code units
Here and in other places

[Yang, 2012/08/10 13:14:45]

Done.

-  V8EXPORT static Local<String> New(const uint16_t* data, int length = -1);
+  V8EXPORT static Local<String> New(const uint16_t* data,
+                                    int length = kUndefinedLength);
 
   /** Creates a symbol. Returns one if it exists already.*/
-  V8EXPORT static Local<String> NewSymbol(const char* data, int length = -1);
+  V8EXPORT static Local<String> NewSymbol(const char* data,
+                                          int length = kUndefinedLength,
+                                          int encoding = UTF8_ENCODING);
 
   /**
    * Creates a new string by concatenating the left and the right strings
    * passed in as parameters.
    */
   V8EXPORT static Local<String> Concat(Handle<String> left,
                                        Handle<String> right);
 
   /**
    * Creates a new external string using the data defined in the given
@@ skipping 16 matching lines @@
    */
   V8EXPORT bool MakeExternal(ExternalStringResource* resource);
 
   /**
    * Creates a new external string using the ASCII data defined in the given
    * resource. When the external string is no longer live on V8's heap the
    * resource will be disposed by calling its Dispose method. The caller of
    * this function should not otherwise delete or modify the resource. Neither
    * should the underlying buffer be deallocated or modified except through the
    * destructor of the external string resource.
-   */ V8EXPORT static Local<String> NewExternal(
+   */
+  V8EXPORT static Local<String> NewExternal(
       ExternalAsciiStringResource* resource);
 
   /**
    * Associate an external string resource with this string by transforming it
    * in place so that existing references to this string in the JavaScript heap
    * will use the external string resource. The external string resource's
    * character contents need to be equivalent to this string.
    * Returns true if the string has been changed to be an external string.
    * The string is not modified if the operation fails. See NewExternal for
    * information on the lifetime of the resource.
    */
   V8EXPORT bool MakeExternal(ExternalAsciiStringResource* resource);
 
+
+  /**
+   * Creates a new external string using the Latin1-encoded data defined in the
+   * given resource.  When the external string is no longer live on V8's heap
+   * the resource will be disposed by calling its Dispose method. The caller of
+   * this function should not otherwise delete or modify the resource. Neither
+   * should the underlying buffer be deallocated or modified except through the
+   * destructor of the external string resource.
+   * If the data contains non-ASCII character, the string is created as new
+   * string object on the V8 heap and the Dispose method is called on the

[Erik Corry, 2012/08/10 11:28:51]

is called ->is called immediately
?

[Yang, 2012/08/10 13:14:45]

Done.

+   * resource.  This is because V8 is unable to handle non-ASCII Latin1-encoded
+   * strings internally at this time.
+   */
+  V8EXPORT static Local<String> NewExternal(
+      ExternalLatin1StringResource* resource,
+      int encoding = String::LATIN1_ENCODING);
+
+  /**
+   * Associate an external string resource with this string by transforming it
+   * in place so that existing references to this string in the JavaScript heap
+   * will use the external string resource. The external string resource's
+   * character contents need to be equivalent to this string.
+   * Returns true if the string has been changed to be an external string.
+   * The string is not modified if the operation fails. See NewExternal for
+   * information on the lifetime of the resource.
+   * If the string to be externalized contains non-ASCII characters, use
+   * the two-byte ExternalStringResource instead.
+   */
+  V8EXPORT bool MakeExternal(ExternalLatin1StringResource* resource);

[Erik Corry, 2012/08/10 11:28:51]

Seems like this should be an ExternalASCIIStringResource.

[Yang, 2012/08/10 13:14:45]

It's actually intended to be like this.  I'm torn between offering a
MakeExternal for Latin1 string resources that refuses to externalize for
non-Latin1 strings (in which case the embedder has to resort to using two-byte
string resources if he *really* wants to externalize), or directly expose to the
embedder that V8 still uses ASCII strings internally, forcing him to use
ExternalAsciiStringResource here.

The former has the advantage that if we decide to support Latin1 natively in V8,
we can simply reimplement ExternalLatin1StringResource and the corresponding
MakeExternal.  It would then also accept non-ASCII Latin1 string resources when
externalizing.  This all would remain transparent to the embedder, i.e. he
doesn't have to change his code since it already assumes that V8 can handle
Latin1.

+
+
   /**
    * Returns true if this string can be made external.
    */
   V8EXPORT bool CanMakeExternal();
 
   /** Creates an undetectable string from the supplied ASCII or UTF-8 data.*/
   V8EXPORT static Local<String> NewUndetectable(const char* data,
-                                                int length = -1);
+                                                int length = kUndefinedLength,
+                                                int encoding = UTF8_ENCODING);
 
   /** Creates an undetectable string from the supplied 16-bit character codes.*/
   V8EXPORT static Local<String> NewUndetectable(const uint16_t* data,
-                                                int length = -1);
+                                                int length = kUndefinedLength);
 
   /**
    * Converts an object to a UTF-8-encoded character array.  Useful if
    * you want to print the object.  If conversion to a string fails
    * (e.g. due to an exception in the toString() method of the object)
    * then the length() method returns 0 and the * operator returns
    * NULL.
    */
   class V8EXPORT Utf8Value {
    public:
@@ skipping 50 matching lines @@
    private:
     uint16_t* str_;
     int length_;
 
     // Disallow copying and assigning.
     Value(const Value&);
     void operator=(const Value&);
   };
 
  private:
-  V8EXPORT void VerifyExternalStringResource(ExternalStringResource* val) const;
+  V8EXPORT void VerifyExternalStringEncoding(int encoding) const;
+  V8EXPORT void VerifyExternalStringResourceBase(
+      ExternalStringResourceBase* val) const;
   V8EXPORT static void CheckCast(v8::Value* obj);
 };
 
 
 /**
  * A JavaScript number value (ECMA-262, 4.3.20)
  */
 class Number : public Primitive {
  public:
   V8EXPORT double Value() const;
@@ skipping 2596 matching lines @@
   // the implementation of v8.
   static const int kHeapObjectMapOffset = 0;
   static const int kMapInstanceTypeOffset = 1 * kApiPointerSize + kApiIntSize;
   static const int kStringResourceOffset = 3 * kApiPointerSize;
 
   static const int kOddballKindOffset = 3 * kApiPointerSize;
   static const int kForeignAddressOffset = kApiPointerSize;
   static const int kJSObjectHeaderSize = 3 * kApiPointerSize;
   static const int kFullStringRepresentationMask = 0x07;
   static const int kExternalTwoByteRepresentationTag = 0x02;
+  static const int kExternalAsciiRepresentationTag = 0x06;
+  static const int kExternalAsciiDataHintMask = 0x08;
+  static const int kExternalAsciiDataHintTag = 0x08;
 
   static const int kIsolateStateOffset = 0;
   static const int kIsolateEmbedderDataOffset = 1 * kApiPointerSize;
   static const int kIsolateRootsOffset = 3 * kApiPointerSize;
   static const int kUndefinedValueRootIndex = 5;
   static const int kNullValueRootIndex = 7;
   static const int kTrueValueRootIndex = 8;
   static const int kFalseValueRootIndex = 9;
   static const int kEmptySymbolRootIndex = 112;
 
@@ skipping 37 matching lines @@
   static inline void* GetExternalPointer(internal::Object* obj) {
     if (HasSmiTag(obj)) {
       return GetExternalPointerFromSmi(obj);
     } else if (GetInstanceType(obj) == kForeignType) {
       return ReadField<void*>(obj, kForeignAddressOffset);
     } else {
       return NULL;
     }
   }
 
-  static inline bool IsExternalTwoByteString(int instance_type) {
-    int representation = (instance_type & kFullStringRepresentationMask);
-    return representation == kExternalTwoByteRepresentationTag;
-  }
-
   static inline bool IsInitialized(v8::Isolate* isolate) {
     uint8_t* addr = reinterpret_cast<uint8_t*>(isolate) + kIsolateStateOffset;
     return *reinterpret_cast<int*>(addr) == 1;
   }
 
   static inline void SetEmbedderData(v8::Isolate* isolate, void* data) {
     uint8_t* addr = reinterpret_cast<uint8_t*>(isolate) +
         kIsolateEmbedderDataOffset;
     *reinterpret_cast<void**>(addr) = data;
   }
@@ skipping 255 matching lines @@
 Local<String> String::Empty(Isolate* isolate) {
   typedef internal::Object* S;
   typedef internal::Internals I;
   if (!I::IsInitialized(isolate)) return Empty();
   S* slot = I::GetRoot(isolate, I::kEmptySymbolRootIndex);
   return Local<String>(reinterpret_cast<String*>(slot));
 }
 
 
 String::ExternalStringResource* String::GetExternalStringResource() const {
+  String::ExternalStringResource* result = NULL;
+  if ((GetExternalStringEncoding() & kStringEncodingMask) ==

[Erik Corry, 2012/08/10 11:28:51]

This calls GetExternalStringEncoding which gets one of 4 answers, then we throw
some information away so that there are only 2 answers and do an 'if' on it.  I
think it would be faster and simpler to just get the representation of the
string directly and test that in the 'if'.

[Yang, 2012/08/10 13:14:45]

Done.

+      UTF16_ENCODING) {
+    result = reinterpret_cast<String::ExternalStringResource*>(
+                 GetExternalStringResourceBase());
+  }
+  return result;
+}
+
+
+int String::GetExternalStringEncoding() const {
   typedef internal::Object O;
   typedef internal::Internals I;
   O* obj = *reinterpret_cast<O**>(const_cast<String*>(this));
-  String::ExternalStringResource* result;
-  if (I::IsExternalTwoByteString(I::GetInstanceType(obj))) {
-    void* value = I::ReadField<void*>(obj, I::kStringResourceOffset);
-    result = reinterpret_cast<String::ExternalStringResource*>(value);
-  } else {
-    result = NULL;
+  static const int kRepresentationAndHintMask =
+      I::kFullStringRepresentationMask | I::kExternalAsciiDataHintMask;
+
+  int encoding;
+  switch (I::GetInstanceType(obj) & kRepresentationAndHintMask) {
+  case I::kExternalTwoByteRepresentationTag | I::kExternalAsciiDataHintTag:

[Erik Corry, 2012/08/10 11:28:51]

Indentation looks wrong here.

[Yang, 2012/08/10 13:14:45]

Done.

+    encoding = UTF16_ENCODING | STRICT_ASCII_HINT;
+    break;
+  case I::kExternalTwoByteRepresentationTag:
+    encoding = UTF16_ENCODING | NOT_ASCII_HINT;
+    break;
+  case I::kExternalAsciiRepresentationTag:
+    encoding = LATIN1_ENCODING | STRICT_ASCII_HINT;
+    break;
+  default:
+    encoding = INVALID_ENCODING;
+    break;
   }
 #ifdef V8_ENABLE_CHECKS
-  VerifyExternalStringResource(result);
+  VerifyExternalStringEncoding(encoding);
+#endif
+  return encoding;
+}
+
+
+String::ExternalStringResourceBase* String::GetExternalStringResourceBase()
+    const {
+  typedef internal::Object O;
+  typedef internal::Internals I;
+  O* obj = *reinterpret_cast<O**>(const_cast<String*>(this));
+  void* value = I::ReadField<void*>(obj, I::kStringResourceOffset);
+  ExternalStringResourceBase* result =
+      reinterpret_cast<String::ExternalStringResourceBase*>(value);
+#ifdef V8_ENABLE_CHECKS
+  VerifyExternalStringResourceBase(result);
 #endif
   return result;
 }
 
 
 bool Value::IsUndefined() const {
 #ifdef V8_ENABLE_CHECKS
   return FullIsUndefined();
 #else
   return QuickIsUndefined();
@@ skipping 214 matching lines @@
 
 
 }  // namespace v8
 
 
 #undef V8EXPORT
 #undef TYPE_CHECK
 
 
 #endif  // V8_H_