Add basic support for Latin1 to the API.

include/v8.h

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 1051 matching lines @@
    *    terminator.  For WriteUtf8: The number of bytes copied to the buffer
    *    including the null terminator (if written).
    */
   enum WriteOptions {
     NO_OPTIONS = 0,
     HINT_MANY_WRITES_EXPECTED = 1,
     NO_NULL_TERMINATION = 2,
     PRESERVE_ASCII_NULL = 4
   };
 
-  // 16-bit character codes.
+
+  enum StringEncoding {
+    INVALID_ENCODING  = 0,
+    UTF8_ENCODING     = 1,

[Erik Corry, 2012/08/21 12:21:34]

The names are not UTF8 or UTF16, but rather UTF-8 and UTF-16.  These should
therefore be called UTF_8_ENCODING and UTF_16_ENCODING.  Also this needs fixing
several places in the comments.

[Yang, 2012/08/21 13:06:20]

Done.

+    LATIN1_ENCODING   = 2,
+    UTF16_ENCODING    = 3,
+
+    STRICT_ASCII_HINT = 1 << 16,

[Erik Corry, 2012/08/21 12:21:34]

I think STRICT_ASCII_HINT should just be called ASCII_HINT

[Yang, 2012/08/21 13:06:20]

Done.

+    NOT_ASCII_HINT    = 1 << 17
+  };
+
+  static const int kStringEncodingMask = 3;
+  static const int kAsciiHintMask =
+      String::STRICT_ASCII_HINT | String::NOT_ASCII_HINT;
+
+  static const int kUndefinedLength = -1;
+
+
+  // 16-bit UTF16 code units.
   V8EXPORT int Write(uint16_t* buffer,
                      int start = 0,
-                     int length = -1,
+                     int length = kUndefinedLength,
                      int options = NO_OPTIONS) const;
   // ASCII characters.
   V8EXPORT int WriteAscii(char* buffer,
                           int start = 0,
-                          int length = -1,
+                          int length = kUndefinedLength,
                           int options = NO_OPTIONS) const;
+  // Latin1 characters.

[Erik Corry, 2012/08/21 12:21:34]

This one doesn't support PRESERVE_ASCII_NULL, or rather it is always on.  That
should be noted.  May also apply to WriteUtf8.

[Yang, 2012/08/21 13:06:20]

Done.

+  V8EXPORT int WriteLatin1(char* buffer,
+                           int start = 0,
+                           int length = kUndefinedLength,
+                           int options = NO_OPTIONS) const;
+
   // UTF-8 encoded characters.
   V8EXPORT int WriteUtf8(char* buffer,
-                         int length = -1,
+                         int length = kUndefinedLength,
                          int* nchars_ref = NULL,
                          int options = NO_OPTIONS) const;
 
   /**
    * A zero length string.
    */
   V8EXPORT static v8::Local<v8::String> Empty();
   inline static v8::Local<v8::String> Empty(Isolate* isolate);
 
   /**
@@ skipping 20 matching lines @@
      * control how allocated external string resources are disposed.
      */
     virtual void Dispose() { delete this; }
 
    private:
     // Disallow copying and assigning.
     ExternalStringResourceBase(const ExternalStringResourceBase&);
     void operator=(const ExternalStringResourceBase&);
 
     friend class v8::internal::Heap;
+    friend class v8::String;
   };
 
   /**
    * An ExternalStringResource is a wrapper around a two-byte string
    * buffer that resides outside V8's heap. Implement an
    * ExternalStringResource to manage the life cycle of the underlying
    * buffer.  Note that the string data must be immutable.
    */
   class V8EXPORT ExternalStringResource
       : public ExternalStringResourceBase {
@@ skipping 39 matching lines @@
     virtual ~ExternalAsciiStringResource() {}
     /** The string data from the underlying buffer.*/
     virtual const char* data() const = 0;
     /** The number of ASCII characters in the string.*/
     virtual size_t length() const = 0;
    protected:
     ExternalAsciiStringResource() {}
   };
 
   /**
+   * An ExternalLatin1StringResource is a wrapper around an Latin1-encoded
+   * string buffer that resides outside V8's heap.  For usage in V8, Latin1
+   * strings are converted to ASCII or two-byte string depending on whether

[Erik Corry, 2012/08/21 12:21:34]

string -> strings,

[Yang, 2012/08/21 13:06:20]

Done.

+   * the string contains non-ASCII characters.
+   */
+  class V8EXPORT ExternalLatin1StringResource
+      : public ExternalAsciiStringResource {
+  };
+
+  /**
    * Get the ExternalStringResource for an external string.  Returns
    * NULL if IsExternal() doesn't return true.
    */
   inline ExternalStringResource* GetExternalStringResource() const;
 
   /**
    * Get the ExternalAsciiStringResource for an external ASCII string.
    * Returns NULL if IsExternalAscii() doesn't return true.
    */
   V8EXPORT const ExternalAsciiStringResource* GetExternalAsciiStringResource()
       const;
 
+  /**
+   * If the string is external, return the its encoding (Latin1 or UTF16)

[Erik Corry, 2012/08/21 12:21:34]

the its -> its

[Yang, 2012/08/21 13:06:20]

Done.

+   * and possibly a hint on whether the content is ASCII.
+   * Return String::INVALID_ENCODING otherwise.
+   */
+  inline int GetExternalStringEncoding() const;
+
+
+  /**
+   * Return the resource of the external string regardless of encoding.
+   * Call this only after having made sure that the string is indeed external!
+   */
+  inline ExternalStringResourceBase* GetExternalStringResourceBase() const;
+
   static inline String* Cast(v8::Value* obj);
 
   /**
-   * Allocates a new string from either UTF-8 encoded or ASCII data.
-   * The second parameter 'length' gives the buffer length.
-   * If the data is UTF-8 encoded, the caller must
-   * be careful to supply the length parameter.
-   * If it is not given, the function calls
-   * 'strlen' to determine the buffer length, it might be
-   * wrong if 'data' contains a null character.
+   * Allocates a new string from either UTF-8-, Latin1-encoded data.

[Erik Corry, 2012/08/21 12:21:34]

"-," should be "or "

[Yang, 2012/08/21 13:06:20]

Done.

+   * The second parameter 'length' gives the buffer length.  If the data is

[Erik Corry, 2012/08/21 12:21:34]

is UTF-8 encoded -> may contain zero bytes

[Yang, 2012/08/21 13:06:20]

Done.

+   * UTF-8 encoded, the caller must be careful to supply the length parameter.
+   * If it is not given, the function calls 'strlen' to determine the buffer
+   * length, it might be wrong if 'data' contains a null character.
+   * The third parameter specifies the encoding, which may include an hint
+   * whether the string contains ASCII characters.  In case of Latin1, the

[Erik Corry, 2012/08/21 12:21:34]

In case of -> In the case of

[Yang, 2012/08/21 13:06:20]

Done.

+   * appropriate internal representation (UTF16 or ASCII) is chosen.
    */
-  V8EXPORT static Local<String> New(const char* data, int length = -1);
+  V8EXPORT static Local<String> New(const char* data,
+                                    int length = kUndefinedLength,
+                                    int encoding = UTF8_ENCODING);

[Erik Corry, 2012/08/21 12:21:34]

Surely this should be a StringEncoding and not an int.

[Yang, 2012/08/21 13:06:20]

The encoding here may additionally contain NOT_ASCII_HINT or ASCII_HINT,
therefore I can't simply use an enum, since e.g. (ASCII_HINT | UTF-8-ENCODING)
is not a valid element of the enum.

 
-  /** Allocates a new string from 16-bit character codes.*/
-  V8EXPORT static Local<String> New(const uint16_t* data, int length = -1);
+  /** Allocates a new string from 16-bit UTF16 code units.*/
+  V8EXPORT static Local<String> New(const uint16_t* data,
+                                    int length = kUndefinedLength);
 
   /** Creates a symbol. Returns one if it exists already.*/
-  V8EXPORT static Local<String> NewSymbol(const char* data, int length = -1);
+  V8EXPORT static Local<String> NewSymbol(const char* data,
+                                          int length = kUndefinedLength,
+                                          int encoding = UTF8_ENCODING);
 
   /**
    * Creates a new string by concatenating the left and the right strings
    * passed in as parameters.
    */
   V8EXPORT static Local<String> Concat(Handle<String> left,
                                        Handle<String> right);
 
   /**
    * Creates a new external string using the data defined in the given
@@ skipping 16 matching lines @@
    */
   V8EXPORT bool MakeExternal(ExternalStringResource* resource);
 
   /**
    * Creates a new external string using the ASCII data defined in the given
    * resource. When the external string is no longer live on V8's heap the
    * resource will be disposed by calling its Dispose method. The caller of
    * this function should not otherwise delete or modify the resource. Neither
    * should the underlying buffer be deallocated or modified except through the
    * destructor of the external string resource.
-   */ V8EXPORT static Local<String> NewExternal(
+   */
+  V8EXPORT static Local<String> NewExternal(
       ExternalAsciiStringResource* resource);
 
   /**
    * Associate an external string resource with this string by transforming it
    * in place so that existing references to this string in the JavaScript heap
    * will use the external string resource. The external string resource's
    * character contents need to be equivalent to this string.
    * Returns true if the string has been changed to be an external string.
    * The string is not modified if the operation fails. See NewExternal for
    * information on the lifetime of the resource.
    */
   V8EXPORT bool MakeExternal(ExternalAsciiStringResource* resource);
 
+
+  /**
+   * Creates a new external string using the Latin1-encoded data defined in the
+   * given resource.  When the external string is no longer live on V8's heap
+   * the resource will be disposed by calling its Dispose method. The caller of
+   * this function should not otherwise delete or modify the resource. Neither
+   * should the underlying buffer be deallocated or modified except through the
+   * destructor of the external string resource.
+   * If the data contains non-ASCII character, the string is created as new

[Erik Corry, 2012/08/21 12:21:34]

contains -> contains a
as new -> as a new

[Yang, 2012/08/21 13:06:20]

Done.

+   * string object on the V8 heap and the Dispose method is called on the
+   * resource immediately.  This is because V8 is currently unable to handle

[Erik Corry, 2012/08/21 12:21:34]

is currently unable -> is unable

[Yang, 2012/08/21 13:06:20]

Done.

+   * non-ASCII Latin1-encoded strings internally.
+   */
+  V8EXPORT static Local<String> NewExternal(
+      ExternalLatin1StringResource* resource,
+      int encoding = String::LATIN1_ENCODING);
+
+
   /**
    * Returns true if this string can be made external.
    */
   V8EXPORT bool CanMakeExternal();
 
   /** Creates an undetectable string from the supplied ASCII or UTF-8 data.*/
   V8EXPORT static Local<String> NewUndetectable(const char* data,
-                                                int length = -1);
+                                                int length = kUndefinedLength,
+                                                int encoding = UTF8_ENCODING);
 
-  /** Creates an undetectable string from the supplied 16-bit character codes.*/
+  /** Creates an undetectable string from the supplied 16-bit UTF16 code units.
+   */
   V8EXPORT static Local<String> NewUndetectable(const uint16_t* data,
-                                                int length = -1);
+                                                int length = kUndefinedLength);
 
   /**
    * Converts an object to a UTF-8-encoded character array.  Useful if
    * you want to print the object.  If conversion to a string fails
    * (e.g. due to an exception in the toString() method of the object)
    * then the length() method returns 0 and the * operator returns
    * NULL.
    */
   class V8EXPORT Utf8Value {
    public:
@@ skipping 50 matching lines @@
    private:
     uint16_t* str_;
     int length_;
 
     // Disallow copying and assigning.
     Value(const Value&);
     void operator=(const Value&);
   };
 
  private:
-  V8EXPORT void VerifyExternalStringResource(ExternalStringResource* val) const;
+  V8EXPORT void VerifyExternalStringEncoding(int encoding) const;
+  V8EXPORT void VerifyExternalStringResourceBase(
+      ExternalStringResourceBase* val) const;
   V8EXPORT static void CheckCast(v8::Value* obj);
 };
 
 
 /**
  * A JavaScript number value (ECMA-262, 4.3.20)
  */
 class Number : public Primitive {
  public:
   V8EXPORT double Value() const;
@@ skipping 2596 matching lines @@
   // the implementation of v8.
   static const int kHeapObjectMapOffset = 0;
   static const int kMapInstanceTypeOffset = 1 * kApiPointerSize + kApiIntSize;
   static const int kStringResourceOffset = 3 * kApiPointerSize;
 
   static const int kOddballKindOffset = 3 * kApiPointerSize;
   static const int kForeignAddressOffset = kApiPointerSize;
   static const int kJSObjectHeaderSize = 3 * kApiPointerSize;
   static const int kFullStringRepresentationMask = 0x07;
   static const int kExternalTwoByteRepresentationTag = 0x02;
+  static const int kExternalAsciiRepresentationTag = 0x06;
+  static const int kExternalAsciiDataHintMask = 0x08;
+  static const int kExternalAsciiDataHintTag = 0x08;
 
   static const int kIsolateStateOffset = 0;
   static const int kIsolateEmbedderDataOffset = 1 * kApiPointerSize;
   static const int kIsolateRootsOffset = 3 * kApiPointerSize;
   static const int kUndefinedValueRootIndex = 5;
   static const int kNullValueRootIndex = 7;
   static const int kTrueValueRootIndex = 8;
   static const int kFalseValueRootIndex = 9;
   static const int kEmptySymbolRootIndex = 112;
 
@@ skipping 37 matching lines @@
   static inline void* GetExternalPointer(internal::Object* obj) {
     if (HasSmiTag(obj)) {
       return GetExternalPointerFromSmi(obj);
     } else if (GetInstanceType(obj) == kForeignType) {
       return ReadField<void*>(obj, kForeignAddressOffset);
     } else {
       return NULL;
     }
   }
 
-  static inline bool IsExternalTwoByteString(int instance_type) {
-    int representation = (instance_type & kFullStringRepresentationMask);
-    return representation == kExternalTwoByteRepresentationTag;
-  }
-
   static inline bool IsInitialized(v8::Isolate* isolate) {
     uint8_t* addr = reinterpret_cast<uint8_t*>(isolate) + kIsolateStateOffset;
     return *reinterpret_cast<int*>(addr) == 1;
   }
 
   static inline void SetEmbedderData(v8::Isolate* isolate, void* data) {
     uint8_t* addr = reinterpret_cast<uint8_t*>(isolate) +
         kIsolateEmbedderDataOffset;
     *reinterpret_cast<void**>(addr) = data;
   }
@@ skipping 257 matching lines @@
   typedef internal::Internals I;
   if (!I::IsInitialized(isolate)) return Empty();
   S* slot = I::GetRoot(isolate, I::kEmptySymbolRootIndex);
   return Local<String>(reinterpret_cast<String*>(slot));
 }
 
 
 String::ExternalStringResource* String::GetExternalStringResource() const {
   typedef internal::Object O;
   typedef internal::Internals I;
+  String::ExternalStringResource* result = NULL;
   O* obj = *reinterpret_cast<O**>(const_cast<String*>(this));
-  String::ExternalStringResource* result;
-  if (I::IsExternalTwoByteString(I::GetInstanceType(obj))) {
-    void* value = I::ReadField<void*>(obj, I::kStringResourceOffset);
-    result = reinterpret_cast<String::ExternalStringResource*>(value);
-  } else {
-    result = NULL;
+  if ((I::GetInstanceType(obj) & I::kFullStringRepresentationMask) ==
+      I::kExternalTwoByteRepresentationTag) {
+    result = reinterpret_cast<String::ExternalStringResource*>(
+                 GetExternalStringResourceBase());
+  }
+  return result;
+}
+
+
+int String::GetExternalStringEncoding() const {
+  typedef internal::Object O;
+  typedef internal::Internals I;
+  O* obj = *reinterpret_cast<O**>(const_cast<String*>(this));
+  static const int kRepresentationAndHintMask =
+      I::kFullStringRepresentationMask | I::kExternalAsciiDataHintMask;
+
+  int encoding;
+  switch (I::GetInstanceType(obj) & kRepresentationAndHintMask) {
+    case I::kExternalTwoByteRepresentationTag | I::kExternalAsciiDataHintTag:
+      encoding = UTF16_ENCODING | STRICT_ASCII_HINT;
+      break;
+    case I::kExternalTwoByteRepresentationTag:
+      encoding = UTF16_ENCODING | NOT_ASCII_HINT;
+      break;
+    case I::kExternalAsciiRepresentationTag:
+      encoding = LATIN1_ENCODING | STRICT_ASCII_HINT;
+      break;
+    default:
+      encoding = INVALID_ENCODING;
+      break;
   }
 #ifdef V8_ENABLE_CHECKS
-  VerifyExternalStringResource(result);
+  VerifyExternalStringEncoding(encoding);
+#endif
+  return encoding;
+}
+
+
+String::ExternalStringResourceBase* String::GetExternalStringResourceBase()
+    const {
+  typedef internal::Object O;
+  typedef internal::Internals I;
+  O* obj = *reinterpret_cast<O**>(const_cast<String*>(this));
+  void* value = I::ReadField<void*>(obj, I::kStringResourceOffset);
+  ExternalStringResourceBase* result =
+      reinterpret_cast<String::ExternalStringResourceBase*>(value);
+#ifdef V8_ENABLE_CHECKS
+  VerifyExternalStringResourceBase(result);
 #endif
   return result;
 }
 
 
 bool Value::IsUndefined() const {
 #ifdef V8_ENABLE_CHECKS
   return FullIsUndefined();
 #else
   return QuickIsUndefined();
@@ skipping 214 matching lines @@
 
 
 }  // namespace v8
 
 
 #undef V8EXPORT
 #undef TYPE_CHECK
 
 
 #endif  // V8_H_