Use ICU case conversion/transliterator for case conversion behind a flag

src/runtime/runtime-i18n.cc

 // Copyright 2014 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 
 #ifdef V8_I18N_SUPPORT
 #include "src/runtime/runtime-utils.h"
 
 #include "src/api.h"
 #include "src/api-natives.h"
@@ skipping 11 matching lines @@
 #include "unicode/dcfmtsym.h"
 #include "unicode/decimfmt.h"
 #include "unicode/dtfmtsym.h"
 #include "unicode/dtptngen.h"
 #include "unicode/locid.h"
 #include "unicode/numfmt.h"
 #include "unicode/numsys.h"
 #include "unicode/rbbi.h"
 #include "unicode/smpdtfmt.h"
 #include "unicode/timezone.h"
+#include "unicode/translit.h"
 #include "unicode/uchar.h"
 #include "unicode/ucol.h"
 #include "unicode/ucurr.h"
 #include "unicode/uloc.h"
+#include "unicode/unistr.h"
 #include "unicode/unum.h"
 #include "unicode/uversion.h"
 
 
 namespace v8 {
 namespace internal {
 
 RUNTIME_FUNCTION(Runtime_CanonicalizeLanguageTag) {
   HandleScope scope(isolate);
   Factory* factory = isolate->factory();
@@ skipping 696 matching lines @@
   } else if (status >= UBRK_WORD_LETTER && status < UBRK_WORD_LETTER_LIMIT) {
     return *isolate->factory()->NewStringFromStaticChars("letter");
   } else if (status >= UBRK_WORD_KANA && status < UBRK_WORD_KANA_LIMIT) {
     return *isolate->factory()->NewStringFromStaticChars("kana");
   } else if (status >= UBRK_WORD_IDEO && status < UBRK_WORD_IDEO_LIMIT) {
     return *isolate->factory()->NewStringFromStaticChars("ideo");
   } else {
     return *isolate->factory()->NewStringFromStaticChars("unknown");
   }
 }
+
+namespace {
+inline void LocaleConvertCaseHelper(icu::UnicodeString* s, bool is_to_upper,
+                                    const icu::Locale& locale) {
+  if (is_to_upper)
+    s->toUpper(locale);
+  else
+    s->toLower(locale);
+}
+void ConvertCaseWithTransliterator(icu::UnicodeString* input,
+                                   const char* transliterator_id) {
+  UErrorCode status = U_ZERO_ERROR;
+  base::SmartPointer<icu::Transliterator> translit(
+      icu::Transliterator::createInstance(
+          icu::UnicodeString(transliterator_id, -1, US_INV), UTRANS_FORWARD,
+          status));
+  if (U_FAILURE(status)) return;
+  translit->transliterate(*input);
+}
+
+MUST_USE_RESULT Object* LocaleConvertCase(Handle<String> s, Isolate* isolate,
+                                          bool is_to_upper, int locale_id) {
+  static const char* conversion_locales[] = {
+      "az", "el", "lt", "tr",
+  };
+  RUNTIME_ASSERT(locale_id >= -1 &&
+                 locale_id < static_cast<int>(arraysize(conversion_locales)));
+  int32_t length = s->length();
+  icu::UnicodeString converted;
+
+  DisallowHeapAllocation no_gc;

[jungshik at Google, 2016/04/13 17:34:34]

Getting this out of an isolated block to avoid an additional buffer copy [1]
does not work (I forgot to try a debug build and thought that I can get away
with it) because there's a heap allocation down the road. 

I'll see if there's a way around it after going back to the previous patch set. 

A question to v8 folks: what's the impact of |DisallowHeapAllocation| on heap
allocation in a third_party library (e.g. ICU)? Can bad things happen? BTW, does
SmartArrayPointer<uc16) behave differently when it's inside
|DisallowHeapAllocation| block?  

[1] I also have to make sure that additional buffer copy is indeed the reason
for slow down. 

[Dan Ehrenberg, 2016/04/13 17:43:43]

I believe DisallowHeapAllocation's function is to add a DCHECK that fires when
allocation occurs/may occur from C++ (grep around for
DCHECK(AllowHeapAllocation::IsAllowed()); ). It won't actually change the nature
of allocations.

+  DCHECK(s->IsFlat());
+  String::FlatContent flat = s->GetFlatContent();
+
+  const UChar* src;
+  base::SmartArrayPointer<uc16> sap;
+  if (flat.IsOneByte()) {
+    sap = s->ToWideCString();
+    src = reinterpret_cast<const UChar*>(sap.get());
+  } else {
+    src = reinterpret_cast<const UChar*>(flat.ToUC16Vector().start());
+  }
+  converted = icu::UnicodeString(false, src, length);
+
+  if (locale_id == -1) {
+    LocaleConvertCaseHelper(&converted, is_to_upper, icu::Locale::getRoot());
+  } else if (V8_UNLIKELY(locale_id == 1 && is_to_upper)) {
+    // TODO(jshin): Once http://bugs.icu-project.org/trac/ticket/10582 is
+    // fixed, remove this special-casing for uppercasing in Greek(el) locale.
+    // This is ~500 times slower than using the case conversion API.
+    ConvertCaseWithTransliterator(&converted, "el-Upper");
+  } else {
+    LocaleConvertCaseHelper(&converted, is_to_upper,
+                            icu::Locale(conversion_locales[locale_id]));
+  }
+
+  Handle<String> result;
+  ASSIGN_RETURN_FAILURE_ON_EXCEPTION(
+      isolate, result,
+      isolate->factory()->NewStringFromTwoByte(Vector<const uint16_t>(
+          reinterpret_cast<const uint16_t*>(converted.getBuffer()),
+          converted.length())));
+  return *result;
+}
+
+inline bool IsASCIIUpper(uint16_t ch) { return ch >= 'A' && ch <= 'Z'; }
+
+inline uint16_t ToASCIILower(uint16_t ch) {
+  return ch | ((ch >= 'A' && ch <= 'Z') << 5);
+}
+
+inline uint16_t ToASCIIUpper(uint16_t ch) {
+  return ch & ~((ch >= 'a' && ch <= 'z') << 5);
+}
+
+}  // namespace
+
+RUNTIME_FUNCTION(Runtime_StringToLowerCaseI18N) {
+  HandleScope scope(isolate);
+  DCHECK_EQ(args.length(), 1);
+  CONVERT_ARG_HANDLE_CHECKED(String, s, 0);
+
+  int length = s->length();
+  s = String::Flatten(s);
+  // First scan the string for uppercase and non-ASCII characters:
+  if (s->HasOnlyOneByteChars()) {
+    unsigned first_index_to_lower = length;
+    for (int index = 0; index < length; ++index) {
+      // Blink specializes this path for one-byte strings, so it
+      // does not need to do a generic get, but can do the equivalent
+      // of SeqOneByteStringGet.
+      uint16_t ch = s->Get(index);
+      if (V8_UNLIKELY(IsASCIIUpper(ch) || ch & ~0x7F)) {
+        first_index_to_lower = index;
+        break;
+      }
+    }
+
+    // Nothing to do if the string is all ASCII with no uppercase.
+    if (first_index_to_lower == length) return *s;
+
+    // We depend here on the invariant that the length of a Latin1
+    // string is invariant under ToLowerCase, and the result always
+    // fits in the Latin1 range (untrue for ToUpperCase, and might
+    // be untrue in some locales, but this is the root locale)
+    Handle<SeqOneByteString> result;
+    ASSIGN_RETURN_FAILURE_ON_EXCEPTION(
+        isolate, result, isolate->factory()->NewRawOneByteString(length));
+    if (s->IsSeqOneByteString()) {
+      SeqOneByteString* source = SeqOneByteString::cast(*s);
+      CopyChars(result->GetChars(), source->GetChars(), first_index_to_lower);
+    } else {
+      // Do we have to worry about External{One,Two}ByteString?
+      DCHECK(s->IsSeqTwoByteString());
+      SeqTwoByteString* source = SeqTwoByteString::cast(*s);
+      CopyChars(result->GetChars(), source->GetChars(), first_index_to_lower);
+    }
+
+    for (int index = first_index_to_lower; index < length; ++index) {
+      uint16_t ch = s->Get(index);
+      result->SeqOneByteStringSet(
+          index, V8_UNLIKELY(ch & ~0x7F) ? static_cast<uint16_t>(u_tolower(ch))
+                                         : ToASCIILower(ch));
+    }
+
+    return *result;
+  }
+
+  // Blink had an additional case here for ASCII 2-byte strings, but
+  // that is subsumed by the above code (assuming there isn't a false
+  // negative for HasOnlyOneByteChars).
+
+  // Do a slower implementation for cases that include non-ASCII characters.
+  return LocaleConvertCase(s, isolate, false, -1);
+}
+
+RUNTIME_FUNCTION(Runtime_StringToUpperCaseI18N) {
+  HandleScope scope(isolate);
+  DCHECK_EQ(args.length(), 1);
+  CONVERT_ARG_HANDLE_CHECKED(String, s, 0);
+
+  // This function could be optimized for no-op cases the way lowercase
+  // counterpart is, but in empirical testing, few actual calls to upper()
+  // are no-ops. So, it wouldn't be worth the extra time for pre-scanning.
+
+  int32_t length = s->length();
+  s = String::Flatten(s);
+  static const uint16_t sharp_s = 0x00DFu;
+
+  if (s->HasOnlyOneByteChars()) {
+    Handle<SeqOneByteString> result;
+    ASSIGN_RETURN_FAILURE_ON_EXCEPTION(
+        isolate, result, isolate->factory()->NewRawOneByteString(length));
+
+    // Do a faster loop for the case where all the characters are ASCII.
+    uint16_t ored = 0;
+    for (int index = 0; index < length; ++index) {
+      uint16_t ch = s->Get(index);
+      ored |= ch;
+      result->SeqOneByteStringSet(index, ToASCIIUpper(ch));
+    }
+    if (!(ored & ~0x7F)) return *result;
+
+    // Do a slower implementation for cases that include non-ASCII Latin-1
+    // characters.
+    int sharp_s_count = 0;
+
+    // There are two special cases.
+    //  1. latin-1 characters when converted to upper case are 16 bit
+    //  characters.
+    //  2. Lower case sharp-S converts to "SS" (two characters)
+    for (int32_t index = 0; index < length; ++index) {
+      uint16_t ch = s->Get(index);
+      if (V8_UNLIKELY(ch == sharp_s)) {
+        ++sharp_s_count;
+        continue;
+      }
+      uint16_t upper = static_cast<uint16_t>(u_toupper(static_cast<UChar>(ch)));
+      if (V8_UNLIKELY(upper > 0xff)) {
+        // Since this upper-cased character does not fit in an 8-bit string, we
+        // need to take the 16-bit path.
+        return LocaleConvertCase(s, isolate, true, -1);
+      }
+      result->SeqOneByteStringSet(index, upper);
+    }
+
+    if (sharp_s_count == 0) return *result;
+
+    // We have sharp_s_count sharp-s characters, but none of the other special
+    // characters.
+    ASSIGN_RETURN_FAILURE_ON_EXCEPTION(
+        isolate, result,
+        isolate->factory()->NewRawOneByteString(length + sharp_s_count));
+    for (int32_t index = 0, dest_index = 0; index < length; ++index) {
+      uint16_t ch = s->Get(index);
+      if (ch == sharp_s) {
+        result->SeqOneByteStringSet(dest_index++, 'S');
+        result->SeqOneByteStringSet(dest_index++, 'S');
+      } else {
+        uint16_t upper =
+            static_cast<uint16_t>(u_toupper(static_cast<UChar>(ch)));
+        result->SeqOneByteStringSet(dest_index++, upper);
+      }
+    }
+
+    return *result;
+  }
+
+  return LocaleConvertCase(s, isolate, true, -1);
+}
+
+RUNTIME_FUNCTION(Runtime_StringLocaleConvertCase) {
+  HandleScope scope(isolate);
+  DCHECK_EQ(args.length(), 3);
+  CONVERT_ARG_HANDLE_CHECKED(String, s, 0);
+  CONVERT_BOOLEAN_ARG_CHECKED(is_upper, 1);
+  CONVERT_NUMBER_CHECKED(int, lang_id, Int32, args[2]);
+
+  return LocaleConvertCase(s, isolate, is_upper, lang_id);
+}
+
 }  // namespace internal
 }  // namespace v8
 
 #endif  // V8_I18N_SUPPORT