Use ICU case conversion/transliterator for case conversion behind a flag

src/runtime/runtime-i18n.cc

 // Copyright 2014 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 
 #ifdef V8_I18N_SUPPORT
 #include "src/runtime/runtime-utils.h"
 
 #include "src/api.h"
 #include "src/api-natives.h"
@@ skipping 11 matching lines @@
 #include "unicode/dcfmtsym.h"
 #include "unicode/decimfmt.h"
 #include "unicode/dtfmtsym.h"
 #include "unicode/dtptngen.h"
 #include "unicode/locid.h"
 #include "unicode/numfmt.h"
 #include "unicode/numsys.h"
 #include "unicode/rbbi.h"
 #include "unicode/smpdtfmt.h"
 #include "unicode/timezone.h"
+#include "unicode/translit.h"
 #include "unicode/uchar.h"
 #include "unicode/ucol.h"
 #include "unicode/ucurr.h"
 #include "unicode/uloc.h"
+#include "unicode/unistr.h"
 #include "unicode/unum.h"
 #include "unicode/uversion.h"
 
 
 namespace v8 {
 namespace internal {
 
 RUNTIME_FUNCTION(Runtime_CanonicalizeLanguageTag) {
   HandleScope scope(isolate);
   Factory* factory = isolate->factory();
@@ skipping 696 matching lines @@
   } else if (status >= UBRK_WORD_LETTER && status < UBRK_WORD_LETTER_LIMIT) {
     return *isolate->factory()->NewStringFromStaticChars("letter");
   } else if (status >= UBRK_WORD_KANA && status < UBRK_WORD_KANA_LIMIT) {
     return *isolate->factory()->NewStringFromStaticChars("kana");
   } else if (status >= UBRK_WORD_IDEO && status < UBRK_WORD_IDEO_LIMIT) {
     return *isolate->factory()->NewStringFromStaticChars("ideo");
   } else {
     return *isolate->factory()->NewStringFromStaticChars("unknown");
   }
 }
+
+namespace {
+void ConvertCaseWithTransliterator(icu::UnicodeString* input,
+                                   const char* transliterator_id) {
+  UErrorCode status = U_ZERO_ERROR;
+  base::SmartPointer<icu::Transliterator> translit(
+      icu::Transliterator::createInstance(
+          icu::UnicodeString(transliterator_id, -1, US_INV), UTRANS_FORWARD,
+          status));
+  if (U_FAILURE(status)) return;
+  translit->transliterate(*input);
+}
+
+MUST_USE_RESULT Object* LocaleConvertCase(Handle<String> s, Isolate* isolate,
+                                          bool is_to_upper, int locale_id) {
+  static const char* conversion_locales[] = {
+      "az", "el", "lt", "tr",
+  };
+  RUNTIME_ASSERT(locale_id >= -1 &&
+                 locale_id < static_cast<int>(arraysize(conversion_locales)));
+  int32_t src_length = s->length();
+  const UChar* src = nullptr;
+
+  base::SmartArrayPointer<uc16> sap;
+  if (s->IsOneByteRepresentationUnderneath()) {
+    sap = s->ToWideCString();
+    src = reinterpret_cast<const UChar*>(sap.get());
+  }
+
+  // Greek (id == 1) uppercasing has to be done via transliteration.
+  // TODO(jshin): Drop this special-casing once ICU's regular case conversion
+  // API supports Greek uppercasing. See
+  // http://bugs.icu-project.org/trac/ticket/10582 .
+  // ICU's C API for transliteration is nasty and we just use C++ API.
+  if (V8_UNLIKELY(locale_id == 1 && is_to_upper)) {

[Dan Ehrenberg, 2016/04/20 22:01:30]

Maybe make an enum for these ids, like

enum LocaleID {
  ROOT = -1,
  AZERI = 0,
  GREEK = 1,
  ...
};

static const char* conversion_locales[] = {
  ...,
  [GREEK] = "el",
  ...
};

[jungshik at Google, 2016/04/21 20:39:17]

Is it ok to use a C99 feature?  I have to add '-Wno-c99-extensions' to gyp file.


This is related to your comment in i18n.js. If a string is used for locale, I
don't have to worry about this. As I replied there, I have a perf concern (maybe
I have to measure). 

+    icu::UnicodeString converted;
+    {
+      DisallowHeapAllocation no_gc;
+      String::FlatContent flat = s->GetFlatContent();
+      if (src == nullptr) {
+        DCHECK(flat.IsTwoByte());
+        src = reinterpret_cast<const UChar*>(flat.ToUC16Vector().start());
+      }
+      // Starts with the source string and will be replaced by the converted
+      // result.
+      converted.fastCopyFrom(icu::UnicodeString(false, src, src_length));
+      ConvertCaseWithTransliterator(&converted, "el-Upper");
+    }
+    Handle<String> result;
+    ASSIGN_RETURN_FAILURE_ON_EXCEPTION(
+        isolate, result,
+        isolate->factory()->NewStringFromTwoByte(Vector<const uint16_t>(
+            reinterpret_cast<const uint16_t*>(converted.getBuffer()),
+            converted.length())));
+    return *result;
+  }
+
+  typedef int32_t (*case_conversion_fn)(
+      UChar * dest, int32_t destCapacity, const UChar* src, int32_t srcLength,
+      const char* locale, UErrorCode* pErrorCode);
+  case_conversion_fn fn = is_to_upper ? u_strToUpper : u_strToLower;

[Dan Ehrenberg, 2016/04/20 22:01:30]

Nit: Since this is just used locally, maybe a good case for auto?

[jungshik at Google, 2016/04/21 20:39:17]

Thanks. Done

+  const char* locale = locale_id == -1 ? "" : conversion_locales[locale_id];

[Dan Ehrenberg, 2016/04/20 22:01:30]

The use of the root locale seems appropriate here for
String.prototype.toUpperCase. However, if the default locale is different from
the root locale with respect to case mapping, do we know that it will be
included in the set of four languages which is included in this code?

[jungshik at Google, 2016/04/21 20:39:17]

Again, "" means root locale and NULL means the default locale. :-)

As toLocale{U,L}Case without an argument, it's the default locale. And, if the
default locale happens to be one of 4, that's already taken care in JS code. 

+
+  int32_t dest_length = src_length;
+  UErrorCode error;
+  Handle<SeqTwoByteString> result;
+
+  // This is not a real loop. It'll be executed only once (no overflow) or
+  // twice (overflow).

[Dan Ehrenberg, 2016/04/20 22:01:30]

Any way we could include a DCHECK to ensure it doesn't run more than twice?

[jungshik at Google, 2016/04/21 20:39:17]

This is the second time I got that question in a month :-). (the first one was
for a  sqlite CL). This is a pretty common pattern used multiple times in both
Chromium and ICU itself.  Anyway, I'm  turning do-while to for loop (i=0 ; i <
2; ++i). 

+  do {
+    result =
+        isolate->factory()->NewRawTwoByteString(dest_length).ToHandleChecked();
+    base::SmartArrayPointer<uc16> sap;
+    DisallowHeapAllocation no_gc;
+    String::FlatContent flat = s->GetFlatContent();
+    // For OneByteString, |src| is already obtained with |sap| outside the loop.
+    if (flat.IsTwoByte())
+      src = reinterpret_cast<const UChar*>(flat.ToUC16Vector().start());
+    error = U_ZERO_ERROR;
+    dest_length = fn(reinterpret_cast<UChar*>(result->GetChars()), dest_length,
+                     src, src_length, locale, &error);
+  } while (error == U_BUFFER_OVERFLOW_ERROR);
+
+  // In most cases, the output will fill the destination buffer completely
+  // leading to an unterminated string (U_STRING_NOT_TERMINATED_WARNING).
+  // Only in rare cases, it'll be shorter than the destination buffer and
+  // |result| has to be truncated.
+  DCHECK(U_SUCCESS(error));
+  // dest_length == result->length()
+  if (V8_LIKELY(error == U_STRING_NOT_TERMINATED_WARNING)) return *result;
+  if (U_SUCCESS(error)) {
+    // dest_length < result->length()
+    return *Handle<SeqTwoByteString>::cast(
+        SeqString::Truncate(result, dest_length));

[Dan Ehrenberg, 2016/04/20 22:01:29]

Do you have a test which hits this case?

[jungshik at Google, 2016/04/21 20:39:17]

intl/general/case-mapping.js has several (Greek and Turkic drop diacritic
marks). test262/intl402/Strings/* has some, too. 

+  }
+  return *s;
+}
+
+inline bool IsASCIIUpper(uint16_t ch) { return ch >= 'A' && ch <= 'Z'; }
+
+inline uint16_t ToLatin1Lower(uint16_t ch) {
+  return ch |
+         (((ch >= 'A' && ch <= 'Z') || (ch >= 0xC0 && ch <= 0xDE && ch != 0xD7))
+          << 5);
+}
+
+inline uint16_t ToASCIIUpper(uint16_t ch) {
+  return ch & ~((ch >= 'a' && ch <= 'z') << 5);
+}
+
+// Does not work for U+00DF (sharp-s), U+00B5 (micron), U+00FF.
+inline uint16_t ToLatin1Upper(uint16_t ch) {
+  DCHECK(ch != 0xDF && ch != 0xB5 && ch != 0xFF);
+  return ch &
+         ~(((ch >= 'a' && ch <= 'z') || (((ch & 0xE0) == 0xE0) && ch != 0xE7))
+           << 5);
+}
+
+template <typename Char>
+bool ToUpperFastASCII(const Vector<const Char>& src,
+                      Handle<SeqOneByteString> result) {
+  // Do a faster loop for the case where all the characters are ASCII.
+  uint16_t ored = 0;
+  int32_t index = 0;
+  for (auto it = src.begin(); it != src.end(); ++it) {
+    uint16_t ch = static_cast<uint16_t>(*it);
+    ored |= ch;
+    result->SeqOneByteStringSet(index++, ToASCIIUpper(ch));
+  }
+  return !(ored & ~0x7F);
+}
+
+const uint16_t sharp_s = 0xDF;
+
+template <typename Char>
+bool ToUpperOneByte(const Vector<const Char>& src,
+                    Handle<SeqOneByteString> result, int* sharp_s_count) {
+  // Still pretty-fast path for the input with non-ASCII Latin-1 characters.
+
+  // There are two special cases.
+  //  1. U+00B5 and U+00FF are mapped to a character beyond U+00FF.
+  //  2. Lower case sharp-S converts to "SS" (two characters)
+  *sharp_s_count = 0;
+  int32_t index = 0;
+  for (auto it = src.begin(); it != src.end(); ++it) {
+    uint16_t ch = static_cast<uint16_t>(*it);
+    if (V8_UNLIKELY(ch == sharp_s)) {
+      ++(*sharp_s_count);
+      continue;
+    }
+    if (V8_UNLIKELY(ch == 0xB5 || ch == 0xFF)) {
+      // Since this upper-cased character does not fit in an 8-bit string, we
+      // need to take the 16-bit path.
+      return false;
+    }
+    result->SeqOneByteStringSet(index++, ToLatin1Upper(ch));
+  }
+
+  return true;
+}
+
+template <typename Char>
+void ToUpperWithSharpS(const Vector<const Char>& src,
+                       Handle<SeqOneByteString> result) {
+  int32_t dest_index = 0;
+  for (auto it = src.begin(); it != src.end(); ++it) {
+    uint16_t ch = static_cast<uint16_t>(*it);
+    if (ch == sharp_s) {
+      result->SeqOneByteStringSet(dest_index++, 'S');
+      result->SeqOneByteStringSet(dest_index++, 'S');
+    } else {
+      result->SeqOneByteStringSet(dest_index++, ToLatin1Upper(ch));
+    }
+  }
+}
+
+}  // namespace
+
+RUNTIME_FUNCTION(Runtime_StringToLowerCaseI18N) {
+  HandleScope scope(isolate);
+  DCHECK_EQ(args.length(), 1);
+  CONVERT_ARG_HANDLE_CHECKED(String, s, 0);
+
+  int length = s->length();
+  s = String::Flatten(s);
+  // First scan the string for uppercase and non-ASCII characters:
+  if (s->HasOnlyOneByteChars()) {
+    unsigned first_index_to_lower = length;
+    for (int index = 0; index < length; ++index) {
+      // Blink specializes this path for one-byte strings, so it
+      // does not need to do a generic get, but can do the equivalent
+      // of SeqOneByteStringGet.
+      uint16_t ch = s->Get(index);
+      if (V8_UNLIKELY(IsASCIIUpper(ch) || ch & ~0x7F)) {
+        first_index_to_lower = index;
+        break;
+      }
+    }
+
+    // Nothing to do if the string is all ASCII with no uppercase.
+    if (first_index_to_lower == length) return *s;
+
+    // We depend here on the invariant that the length of a Latin1
+    // string is invariant under ToLowerCase, and the result always
+    // fits in the Latin1 range (untrue for ToUpperCase, and might
+    // be untrue in some locales, but this is the root locale)
+    Handle<SeqOneByteString> result;
+    ASSIGN_RETURN_FAILURE_ON_EXCEPTION(
+        isolate, result, isolate->factory()->NewRawOneByteString(length));
+
+    DisallowHeapAllocation no_gc;
+    String::FlatContent flat = s->GetFlatContent();
+    if (flat.IsOneByte()) {
+      const uint8_t* src = flat.ToOneByteVector().start();
+      CopyChars(result->GetChars(), src, first_index_to_lower);
+      for (int index = first_index_to_lower; index < length; ++index) {
+        uint16_t ch = static_cast<uint16_t>(src[index]);
+        result->SeqOneByteStringSet(index, ToLatin1Lower(ch));
+      }
+    } else {
+      const uint16_t* src = flat.ToUC16Vector().start();
+      CopyChars(result->GetChars(), src, first_index_to_lower);
+      for (int index = first_index_to_lower; index < length; ++index) {
+        uint16_t ch = src[index];
+        result->SeqOneByteStringSet(index, ToLatin1Lower(ch));
+      }
+    }
+
+    return *result;
+  }
+
+  // Blink had an additional case here for ASCII 2-byte strings, but
+  // that is subsumed by the above code (assuming there isn't a false
+  // negative for HasOnlyOneByteChars).
+
+  // Do a slower implementation for cases that include non-ASCII characters.
+  return LocaleConvertCase(s, isolate, false, -1);
+}
+
+RUNTIME_FUNCTION(Runtime_StringToUpperCaseI18N) {
+  HandleScope scope(isolate);
+  DCHECK_EQ(args.length(), 1);
+  CONVERT_ARG_HANDLE_CHECKED(String, s, 0);
+
+  // This function could be optimized for no-op cases the way lowercase
+  // counterpart is, but in empirical testing, few actual calls to upper()
+  // are no-ops. So, it wouldn't be worth the extra time for pre-scanning.
+
+  int32_t length = s->length();
+  s = String::Flatten(s);
+
+  if (s->HasOnlyOneByteChars()) {
+    Handle<SeqOneByteString> result;
+    ASSIGN_RETURN_FAILURE_ON_EXCEPTION(
+        isolate, result, isolate->factory()->NewRawOneByteString(length));
+
+    int sharp_s_count;
+    bool is_result_single_byte;
+    {
+      DisallowHeapAllocation no_gc;
+      String::FlatContent flat = s->GetFlatContent();
+      // If it was ok to slow down ASCII-only input slightly, ToUpperFastASCII
+      // could be removed  because ToUpperOneByte is pretty fast now (it
+      // does not call ICU API any more.).
+      if (flat.IsOneByte()) {
+        Vector<const uint8_t> src = flat.ToOneByteVector();
+        if (ToUpperFastASCII(src, result)) return *result;
+        is_result_single_byte = ToUpperOneByte(src, result, &sharp_s_count);
+      } else {
+        DCHECK(flat.IsTwoByte());
+        Vector<const uint16_t> src = flat.ToUC16Vector();
+        if (ToUpperFastASCII(src, result)) return *result;
+        is_result_single_byte = ToUpperOneByte(src, result, &sharp_s_count);
+      }
+    }
+
+    // Go to the full Unicode path if there are characters whose uppercase
+    // is beyond the Latin-1 range (cannot be represented in OneByteString).
+    if (V8_UNLIKELY(!is_result_single_byte))
+      return LocaleConvertCase(s, isolate, true, -1);
+
+    if (sharp_s_count == 0) return *result;
+
+    // We have sharp_s_count sharp-s characters, but the result is still
+    // in the Latin-1 range.
+    ASSIGN_RETURN_FAILURE_ON_EXCEPTION(
+        isolate, result,
+        isolate->factory()->NewRawOneByteString(length + sharp_s_count));
+    DisallowHeapAllocation no_gc;
+    String::FlatContent flat = s->GetFlatContent();
+    if (flat.IsOneByte())
+      ToUpperWithSharpS(flat.ToOneByteVector(), result);
+    else
+      ToUpperWithSharpS(flat.ToUC16Vector(), result);
+
+    return *result;
+  }
+
+  return LocaleConvertCase(s, isolate, true, -1);
+}
+
+RUNTIME_FUNCTION(Runtime_StringLocaleConvertCase) {
+  HandleScope scope(isolate);
+  DCHECK_EQ(args.length(), 3);
+  CONVERT_ARG_HANDLE_CHECKED(String, s, 0);
+  CONVERT_BOOLEAN_ARG_CHECKED(is_upper, 1);
+  CONVERT_NUMBER_CHECKED(int, lang_id, Int32, args[2]);
+
+  return LocaleConvertCase(s, isolate, is_upper, lang_id);
+}
+
 }  // namespace internal
 }  // namespace v8
 
 #endif  // V8_I18N_SUPPORT