Add basic support for Latin1 strings to the API.

test/cctest/test-api.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 691 matching lines @@
     HEAP->CollectAllAvailableGarbage();
     CHECK_EQ(0, TestAsciiResourceWithDisposeControl::dispose_count);
   }
   i::Isolate::Current()->compilation_cache()->Clear();
   HEAP->CollectAllAvailableGarbage();
   CHECK_EQ(1, TestAsciiResourceWithDisposeControl::dispose_calls);
   CHECK_EQ(1, TestAsciiResourceWithDisposeControl::dispose_count);
 }
 
 
+static void TestNewLatin1String(int encoding1, int encoding2) {
+  const char* chars1 = "ASCII 123";
+  const char* chars1js = "'ASCII 123'";
+  int str1_len = strlen(chars1);
+  const char* chars2 = "Non-ASCII \xAB\xCD\xEF";
+  const char* chars2js = "'Non-ASCII \\u00ab\\u00cd\\u00ef'";
+  int str2_len = strlen(chars2);
+
+  Local<String> str1 = String::New(chars1, str1_len, encoding1);
+  Local<String> str2 = String::New(chars2, str2_len, encoding2);
+  Local<String> str1_compare = CompileRun(chars1js)->ToString();
+  Local<String> str2_compare = CompileRun(chars2js)->ToString();
+
+  if (encoding1 & String::NOT_ASCII_HINT) {
+    CHECK(v8::Utils::OpenHandle(*str1)->IsSeqTwoByteString());
+  } else {
+    CHECK(v8::Utils::OpenHandle(*str1)->IsSeqAsciiString());
+  }
+  CHECK(v8::Utils::OpenHandle(*str1_compare)->IsSeqAsciiString());
+  CHECK(v8::Utils::OpenHandle(*str2)->IsSeqTwoByteString());
+  CHECK(v8::Utils::OpenHandle(*str2_compare)->IsSeqTwoByteString());
+
+  CHECK(str1_compare->Equals(str1));
+  CHECK(str2_compare->Equals(str2));
+}
+
+
+TEST(CreateLatin1String) {
+  v8::HandleScope scope;
+  LocalContext env;
+
+  int latin1 = String::LATIN1_ENCODING;
+  int l_noascii = String::LATIN1_ENCODING | String::NOT_ASCII_HINT;
+  int l_ascii = String::LATIN1_ENCODING | String::STRICT_ASCII_HINT;
+
+  TestNewLatin1String(latin1, latin1);
+  TestNewLatin1String(l_ascii, latin1);
+  TestNewLatin1String(l_noascii, l_noascii);
+}
+
+
+TEST(ExternalStringEncoding) {
+  v8::HandleScope scope;
+  LocalContext env;
+  int counter = 0;
+
+  { HandleScope scope;
+    uint16_t* two_byte_ascii = AsciiToTwoByteString("two byte ascii");
+    uint16_t* two_byte = AsciiToTwoByteString("two byte non-ascii \x99");
+    char* ascii = i::StrDup("ascii");
+
+    TestResource* two_byte_resource = new TestResource(two_byte, &counter);
+    TestResource* two_byte_ascii_resource =
+        new TestResource(two_byte_ascii, &counter);
+    TestAsciiResource* ascii_resource =
+        new TestAsciiResource(ascii, &counter);
+
+    Local<String> two_byte_external = String::NewExternal(two_byte_resource);
+    Local<String> two_byte_ascii_external =
+        String::NewExternal(two_byte_ascii_resource);
+    Local<String> ascii_external = String::NewExternal(ascii_resource);
+    Local<String> not_external = v8_str("not external");
+
+    CHECK_EQ(String::UTF16_ENCODING | String::NOT_ASCII_HINT,
+             two_byte_external->GetExternalStringEncoding());
+    CHECK_EQ(String::UTF16_ENCODING | String::STRICT_ASCII_HINT,
+             two_byte_ascii_external->GetExternalStringEncoding());
+    CHECK_EQ(String::LATIN1_ENCODING | String::STRICT_ASCII_HINT,
+             ascii_external->GetExternalStringEncoding());
+    CHECK_EQ(String::INVALID_ENCODING,
+             not_external->GetExternalStringEncoding());
+
+    CHECK_EQ(two_byte_resource, two_byte_external->GetExternalStringResource());
+    CHECK_EQ(two_byte_ascii_resource,
+             two_byte_ascii_external->GetExternalStringResourceBase());
+    CHECK_EQ(ascii_resource, ascii_external->GetExternalStringResourceBase());
+
+    CHECK_EQ(0, counter);
+  }
+
+  HEAP->CollectAllGarbage(i::Heap::kNoGCFlags);
+
+  CHECK_EQ(3, counter);
+}
+
+
+TEST(WriteLatin1String) {
+  HandleScope scope;
+  LocalContext env;
+  const char* latin1_ascii = "latin1 ascii";
+  const char* latin1 = "\x99 latin1 non-ascii \xF8";
+  const char* concat = "latin1 ascii\x99 latin1 non-ascii \xF8";
+  const char* sub = "latin1 non-ascii \xF8";
+
+  Local<String> latin1_ascii_string = String::New(latin1_ascii,
+                                                  String::kUndefinedLength,
+                                                  String::LATIN1_ENCODING);
+  Local<String> latin1_string = String::New(latin1,
+                                            String::kUndefinedLength,
+                                            String::LATIN1_ENCODING);
+  Local<String> concat_string = String::Concat(latin1_ascii_string,
+                                               latin1_string);
+  Local<String> sub_string = v8::Utils::ToLocal(
+      FACTORY->NewSubString(
+          v8::Utils::OpenHandle(*latin1_string), 2, latin1_string->Length()));
+
+  CHECK(v8::Utils::OpenHandle(*latin1_ascii_string)->IsSeqAsciiString());
+  CHECK(v8::Utils::OpenHandle(*latin1_string)->IsSeqTwoByteString());
+  CHECK(v8::Utils::OpenHandle(*concat_string)->IsConsString());
+  CHECK(v8::Utils::OpenHandle(*sub_string)->IsSlicedString());
+
+  char buffer[64];
+  CHECK_EQ(strlen(latin1_ascii), latin1_ascii_string->WriteLatin1(buffer));
+  CHECK_EQ(0, strcmp(latin1_ascii, buffer));
+  CHECK_EQ(strlen(latin1), latin1_string->WriteLatin1(buffer));
+  CHECK_EQ(0, strcmp(latin1, buffer));
+  CHECK_EQ(strlen(concat), concat_string->WriteLatin1(buffer));
+  CHECK_EQ(0, strcmp(concat, buffer));
+  CHECK_EQ(strlen(sub), sub_string->WriteLatin1(buffer));
+  CHECK_EQ(0, strcmp(sub, buffer));
+
+  memset(buffer, 0x1, sizeof(buffer));
+  CHECK_EQ(strlen(latin1),
+           latin1_string->WriteLatin1(buffer,
+                                      0,
+                                      String::kUndefinedLength,
+                                      String::NO_NULL_TERMINATION));
+  CHECK_EQ(0, strncmp(latin1, buffer, strlen(latin1)));
+  CHECK_NE(0, strcmp(latin1, buffer));
+  buffer[strlen(latin1)] = '\0';
+  CHECK_EQ(0, strcmp(latin1, buffer));
+
+  CHECK_EQ(strlen(latin1) - 2,
+           latin1_string->WriteLatin1(buffer, 2));
+  CHECK_EQ(0, strncmp(latin1 + 2, buffer, strlen(latin1)));
+}
+
+
+class TestLatin1Resource: public String::ExternalLatin1StringResource {
+ public:
+  explicit TestLatin1Resource(const char* data, int* counter = NULL)
+    : data_(data), length_(strlen(data)), counter_(counter) { }
+
+  ~TestLatin1Resource() {
+    i::DeleteArray(data_);
+    if (counter_ != NULL) ++*counter_;
+  }
+
+  const char* data() const {
+    return data_;
+  }
+
+  size_t length() const {
+    return length_;
+  }
+ private:
+  const char* data_;
+  size_t length_;
+  int* counter_;
+};
+
+
+TEST(ExternalLatin1String) {
+  HandleScope scope;
+  LocalContext env;
+  int counter = 0;
+
+  { HandleScope scope;
+    char* latin1_ascii_a = i::StrDup("latin1 ascii a");
+    char* latin1_ascii_b = i::StrDup("latin1 ascii b");
+    char* latin1_a = i::StrDup("latin non-ascii \xAA");
+    char* latin1_b = i::StrDup("latin non-ascii \xBB");
+
+    TestLatin1Resource* latin1_ascii_a_resource =
+        new TestLatin1Resource(latin1_ascii_a, &counter);
+    TestLatin1Resource* latin1_ascii_b_resource =
+        new TestLatin1Resource(latin1_ascii_b, &counter);
+    TestLatin1Resource* latin1_a_resource =
+        new TestLatin1Resource(latin1_a, &counter);
+    TestLatin1Resource* latin1_b_resource =
+        new TestLatin1Resource(latin1_b, &counter);
+
+    Local<String> latin1_ascii_a_external =
+        String::NewExternal(latin1_ascii_a_resource);
+    Local<String> latin1_ascii_b_external = String::NewExternal(
+        latin1_ascii_b_resource,
+        String::LATIN1_ENCODING | String::STRICT_ASCII_HINT);
+    CHECK_EQ(0, counter);
+
+    // Non-ascii latin1 strings are internalized immediately as two-byte
+    // string and the external resource is disposed.
+    Local<String> latin1_a_external = String::NewExternal(latin1_a_resource);
+    Local<String> latin1_b_external = String::NewExternal(
+        latin1_b_resource, String::LATIN1_ENCODING | String::NOT_ASCII_HINT);
+    CHECK(v8::Utils::OpenHandle(*latin1_a_external)->IsSeqTwoByteString());
+    CHECK(v8::Utils::OpenHandle(*latin1_b_external)->IsSeqTwoByteString());
+    CHECK_EQ(2, counter);
+
+    CHECK_EQ(latin1_ascii_a_external->GetExternalStringEncoding(),
+             (v8::String::LATIN1_ENCODING | v8::String::STRICT_ASCII_HINT));
+    CHECK_EQ(latin1_ascii_b_external->GetExternalStringEncoding(),
+             (v8::String::LATIN1_ENCODING | v8::String::STRICT_ASCII_HINT));
+    CHECK_EQ(latin1_a_external->GetExternalStringEncoding(),
+             v8::String::INVALID_ENCODING);
+    CHECK_EQ(latin1_b_external->GetExternalStringEncoding(),
+             v8::String::INVALID_ENCODING);
+
+    CHECK_EQ(latin1_ascii_a_resource,
+             latin1_ascii_a_external->GetExternalStringResourceBase());
+    CHECK_EQ(latin1_ascii_b_resource,
+             latin1_ascii_b_external->GetExternalStringResourceBase());
+  }
+
+  HEAP->CollectAllGarbage(i::Heap::kNoGCFlags);
+  CHECK_EQ(4, counter);
+}
+
+
+TEST(ExternalizeLatin1String) {
+  HandleScope scope;
+  LocalContext env;
+  int counter = 0;
+
+  { HandleScope scope;
+    Local<String> latin1_a_ascii = String::New("latin1 a ascii");
+    Local<String> latin1_b_ascii = String::New("latin1 b ascii");
+    Local<String> latin1_a = String::New("latin1 non-ascii \xAA",
+                                         String::kUndefinedLength,
+                                         String::LATIN1_ENCODING);
+    Local<String> latin1_b = String::New("latin1 non-ascii \xBB",
+                                         String::kUndefinedLength,
+                                         String::LATIN1_ENCODING);
+
+    CHECK(v8::Utils::OpenHandle(*latin1_a_ascii)->IsSeqAsciiString());
+    CHECK(v8::Utils::OpenHandle(*latin1_b_ascii)->IsSeqAsciiString());
+    CHECK(v8::Utils::OpenHandle(*latin1_a)->IsSeqTwoByteString());
+    CHECK(v8::Utils::OpenHandle(*latin1_b)->IsSeqTwoByteString());
+
+    // Run GC twice to put those strings into old space for externalizing.
+    HEAP->CollectGarbage(i::NEW_SPACE);
+    HEAP->CollectGarbage(i::NEW_SPACE);
+
+    char* latin1_a_ascii_chars = i::NewArray<char>(64);
+    uint16_t* latin1_b_ascii_chars = i::NewArray<uint16_t>(64);
+    char* latin1_a_chars = i::NewArray<char>(64);
+    uint16_t* latin1_b_chars = i::NewArray<uint16_t>(64);
+
+    latin1_a_ascii->WriteLatin1(latin1_a_ascii_chars);
+    latin1_b_ascii->Write(latin1_b_ascii_chars);
+    latin1_a->WriteLatin1(latin1_a_chars);
+    latin1_b->Write(latin1_b_chars);
+
+    TestLatin1Resource* latin1_a_ascii_resource =
+        new TestLatin1Resource(latin1_a_ascii_chars, &counter);
+    TestResource* latin1_b_ascii_resource =
+        new TestResource(latin1_b_ascii_chars, &counter);
+    TestLatin1Resource* latin1_a_resource =
+        new TestLatin1Resource(latin1_a_chars, &counter);
+    TestResource* latin1_b_resource =
+        new TestResource(latin1_b_chars, &counter);
+
+    CHECK(latin1_a_ascii->MakeExternal(latin1_a_ascii_resource));
+    CHECK(latin1_a_ascii->IsExternalAscii());
+    CHECK_EQ(latin1_a_ascii->GetExternalStringEncoding(),
+             (v8::String::LATIN1_ENCODING | v8::String::STRICT_ASCII_HINT));
+    CHECK_EQ(latin1_a_ascii_resource,
+             latin1_a_ascii->GetExternalStringResourceBase());
+    CHECK(latin1_a_ascii->Equals(String::New("latin1 a ascii")));
+
+    CHECK(latin1_b_ascii->MakeExternal(latin1_b_ascii_resource));
+    CHECK(latin1_b_ascii->IsExternal());
+    CHECK_EQ(latin1_b_ascii->GetExternalStringEncoding(),
+             (v8::String::UTF16_ENCODING | v8::String::STRICT_ASCII_HINT));
+    CHECK_EQ(latin1_b_ascii_resource,
+             latin1_b_ascii->GetExternalStringResourceBase());
+    CHECK(latin1_b_ascii->Equals(String::New("latin1 b ascii")));
+
+    CHECK(!latin1_a->MakeExternal(latin1_a_resource));

[Yang, 2012/08/10 13:14:45]

This would be the use case of MakeExternal(ExternalLatin1StringResource*).
Currently, we simply refuse to externalize non-ASCII strings if provided with an
external Latin1 string resource. If necessary, we could change that behavior to
accept those strings once V8 can deal with Latin1 strings internally.

With this, the embedder can get rid of usage of ExternalAsciiStringResource
entirely.

+    i::DeleteArray(latin1_a_chars);
+
+    CHECK(latin1_b->MakeExternal(latin1_b_resource));
+    CHECK(latin1_b->IsExternal());
+    CHECK_EQ(latin1_b->GetExternalStringEncoding(),
+             (v8::String::UTF16_ENCODING | v8::String::NOT_ASCII_HINT));
+    CHECK_EQ(latin1_b_resource,
+             latin1_b->GetExternalStringResourceBase());
+    CHECK(latin1_b->Equals(String::New("latin1 non-ascii \xBB",
+                                       String::kUndefinedLength,
+                                       String::LATIN1_ENCODING)));
+  }
+
+  HEAP->CollectAllGarbage(i::Heap::kNoGCFlags);
+  CHECK_EQ(3, counter);
+}
+
+
 THREADED_TEST(StringConcat) {
   {
-    v8::HandleScope scope;
+    HandleScope scope;
     LocalContext env;
     const char* one_byte_string_1 = "function a_times_t";
     const char* two_byte_string_1 = "wo_plus_b(a, b) {return ";
     const char* one_byte_extern_1 = "a * 2 + b;} a_times_two_plus_b(4, 8) + ";
     const char* two_byte_extern_1 = "a_times_two_plus_b(4, 8) + ";
     const char* one_byte_string_2 = "a_times_two_plus_b(4, 8) + ";
     const char* two_byte_string_2 = "a_times_two_plus_b(4, 8) + ";
     const char* two_byte_extern_2 = "a_times_two_plus_b(1, 2);";
     Local<String> left = v8_str(one_byte_string_1);
 
@@ skipping 16383 matching lines @@
   v8::HandleScope scope;
   LocalContext context;
 
   // Compile a try-finally clause where the finally block causes a GC
   // while there still is a message pending for external reporting.
   TryCatch try_catch;
   try_catch.SetVerbose(true);
   CompileRun("try { throw new Error(); } finally { gc(); }");
   CHECK(try_catch.HasCaught());
 }