Use ICU case conversion/transliterator for case conversion behind a flag

src/js/i18n.js

Index: src/js/i18n.js
diff --git a/src/js/i18n.js b/src/js/i18n.js
index 845289a91f10cf1416c4cff4b6698e27f412d6be..7d623926e75a98abbb5e265cb27f889ddc92d5f9 100644
--- a/src/js/i18n.js
+++ b/src/js/i18n.js
@@ -144,6 +144,13 @@ var AVAILABLE_LOCALES = {
  */
 var DEFAULT_ICU_LOCALE = UNDEFINED;
 
+function GetDefaultICULocaleJS() {
+  if (IS_UNDEFINED(DEFAULT_ICU_LOCALE)) {
+    DEFAULT_ICU_LOCALE = %GetDefaultICULocale();
+  }
+  return DEFAULT_ICU_LOCALE;
+}
+
 /**
  * Unicode extension regular expression.
  */
@@ -448,11 +455,7 @@ function lookupMatcher(service, requestedLocales) {
   }
 
   // Didn't find a match, return default.
-  if (IS_UNDEFINED(DEFAULT_ICU_LOCALE)) {
-    DEFAULT_ICU_LOCALE = %GetDefaultICULocale();
-  }
-
-  return {'locale': DEFAULT_ICU_LOCALE, 'extension': '', 'position': -1};
+  return {'locale': GetDefaultICULocaleJS(), 'extension': '', 'position': -1};
 }
 
 
@@ -722,21 +725,23 @@ function toTitleCaseTimezoneLocation(location) {
  */
 function canonicalizeLanguageTag(localeID) {
   // null is typeof 'object' so we have to do extra check.
-  if (typeof localeID !== 'string' && typeof localeID !== 'object' ||
+  if ((typeof localeID !== 'string' && typeof localeID !== 'object') ||

[Dan Ehrenberg, 2016/04/20 22:01:29]

Nit: As long as you're changing this code, could you use IS_STRING(localeID),
and IS_RECEIVER(localeID) to be more idiomatic (and slightly more correct)?

[jungshik at Google, 2016/04/21 20:39:17]

Done.

       IS_NULL(localeID)) {
     throw MakeTypeError(kLanguageID);
   }
 
+  // Optimize for the most common case; a language code alone in
+  // the canonical form/lowercase (e.g. "en", "fil").
+  if (typeof localeID === 'string' &&

[Dan Ehrenberg, 2016/04/20 22:01:29]

IS_STRING

[jungshik at Google, 2016/04/21 20:39:16]

Done.

+      !IS_NULL(InternalRegExpMatch(/^[a-z]{2,3}$/, localeID)))
+    return localeID;
+
   var localeString = GlobalString(localeID);
 
   if (isValidLanguageTag(localeString) === false) {
     throw MakeRangeError(kInvalidLanguageTag, localeString);
   }
 
-  // This call will strip -kn but not -kn-true extensions.
-  // ICU bug filled - http://bugs.icu-project.org/trac/ticket/9265.
-  // TODO(cira): check if -u-kn-true-kc-true-kh-true still throws after
-  // upgrade to ICU 4.9.
   var tag = %CanonicalizeLanguageTag(localeString);
   if (tag === 'invalid-tag') {
     throw MakeRangeError(kInvalidLanguageTag, localeString);
@@ -1992,6 +1997,36 @@ function cachedOrNewService(service, locales, options, defaults) {
   return new savedObjects[service](locales, useOptions);
 }
 
+function getCaseConversionLanguageId(locales) {

[Dan Ehrenberg, 2016/04/20 22:01:29]

Generally, the v8 convention is CamelCase with an initial capital letter. Let's
use that for new code.

[jungshik at Google, 2016/04/21 20:39:16]

Done.

+  var language;
+  // Optimize for the most common two cases. initializeLocaleList() can handle
+  // them as well, but it's rather slow accounting for over 60% of
+  // toLocale{U,L}Case() and about 40% of toLocale{U,L}Case("<locale>").
+  if (IS_UNDEFINED(locales)) {
+    language = GetDefaultICULocaleJS();
+  } else if (typeof locales === 'string') {

[Dan Ehrenberg, 2016/04/20 22:01:29]

IS_STRING

[jungshik at Google, 2016/04/21 20:39:16]

Done.

+    language = canonicalizeLanguageTag(locales);
+  } else {
+    var locales = initializeLocaleList(locales);
+    language = locales.length > 0 ? locales[0] : GetDefaultICULocaleJS();
+  }
+
+  // StringSplit is slwoer than this.

[Dan Ehrenberg, 2016/04/20 22:01:29]

slower

Consider factoring this out into a utility function.

[jungshik at Google, 2016/04/21 20:39:16]

Typo fixed. 
 
You mean  "var pos ..  ; if (pos != -1) .... " ? Like this? 

function GetLanguagePart(locale) {
  // StringSplit is slower than this.
  var pos = %_Call(StringIndexOf, locale, '-');
  return pos == -1 ? locale :
      %_Call(StringSubstring, locale, 0, pos);
}

+  var pos = %_Call(StringIndexOf, language, '-');
+  if (pos != -1)
+    language = %_Call(StringSubstring, language, 0, pos);
+
+  var CUSTOM_CASE_LANGUAGES = ['az', 'el', 'lt', 'tr'];

[Dan Ehrenberg, 2016/04/20 22:01:29]

Could we somehow query this from ICU? I thought there were more languages that
had custom case conversion, like French for removing accents over vowels when
going uppercase.

Is there a way to avoid having this parallel array in JS and C++? For example,
what if we just pass the string into the runtime, and let that interpret it.

[jungshik at Google, 2016/04/21 20:39:17]

Some typesetting systems in the past couldn't deal with uppercase with
diacritics, but that's not the case any more. The French Academy is very clear
about what to do about them. (do not drop diacritics). Well, some native French
speakers get confused, too. :-) 

Anyway, what languages are subject to special casings comes from the Unicode
data files, but 
CLDR/ICU does not have that list exposed via API.  

Unicode's SpecialCasing.txt has 'tr', 'az' and 'lt'. 'el' is not there yet.
ECMAScript 402 only lists 'tr', 'az' and 'lt' at the moment citing
SpecialCasing.txt at unicode.org (I'll file a bug against the spec), but 'el'
certainly needs a tailored case mapping. (Blink and Gecko do that, already for
CSS text-transform) per strong demand from Greek speakers (Mozilla's
implementation is more sophisticated than CLDR/ICU's. CLDR/ICU is looking into
revising it). 
I hate parallel arrays, too !! OTOH, I want to avoid passing a string from JS to
C++ unless there's an efficient way to wrap and unwrap it (almost as efficient
as passing an integer).   

StringNormalize does the same (passing integer instead of 'NFC', 'NFKC', "NFD',
'NFKD'). If there's an efficient way to pass one of a small list of strings
between JS and runtime, I'd for sure use that. 

+  return %_Call(ArrayIndexOf, CUSTOM_CASE_LANGUAGES, language);
+}
+
+function localeConvertCase(s, locales, isToUpper) {

[Dan Ehrenberg, 2016/04/20 22:01:29]

LocaleConvertCase

[jungshik at Google, 2016/04/21 20:39:16]

Done.

+  var caseConversionLanguageId = getCaseConversionLanguageId(locales);
+  if (caseConversionLanguageId == -1)
+    return isToUpper ? %StringToUpperCaseI18N(s) : %StringToLowerCaseI18N(s);
+  return %StringLocaleConvertCase(s, isToUpper, caseConversionLanguageId);
+}
+
 /**
  * Compares this and that, and returns less than 0, 0 or greater than 0 value.
  * Overrides the built-in method.
@@ -2044,6 +2079,45 @@ OverrideFunction(GlobalString.prototype, 'normalize', function() {
   }
 );
 
+OverrideFunction(GlobalString.prototype, 'toLowerCase', function() {
+    if (!IS_UNDEFINED(new.target)) {
+      throw MakeTypeError(kOrdinaryFunctionCalledAsConstructor);
+    }
+    CHECK_OBJECT_COERCIBLE(this, "String.prototype.toLowerCase");
+    var s = TO_STRING(this);
+    return %StringToLowerCaseI18N(s);

[Dan Ehrenberg, 2016/04/20 22:01:29]

ECMA262 seems to specify using the root locale in
https://tc39.github.io/ecma262/#sec-string.prototype.tolowercase ; however, the
implementation seems to pass "" to ICU, which would use the current default
locale, right?

[jungshik at Google, 2016/04/21 20:39:16]

Right.
"" for root locale and NULL for the default locale :-)

@param locale    The locale to consider, or "" for the root locale or NULL for
the default locale.

+  }
+);
+
+OverrideFunction(GlobalString.prototype, 'toUpperCase', function() {
+    if (!IS_UNDEFINED(new.target)) {
+      throw MakeTypeError(kOrdinaryFunctionCalledAsConstructor);
+    }
+    CHECK_OBJECT_COERCIBLE(this, "String.prototype.toLowerCase");
+    var s = TO_STRING(this);
+    return %StringToUpperCaseI18N(s);
+  }
+);
+
+OverrideFunction(GlobalString.prototype, 'toLocaleLowerCase', function() {
+    if (!IS_UNDEFINED(new.target)) {
+      throw MakeTypeError(kOrdinaryFunctionCalledAsConstructor);
+    }
+    CHECK_OBJECT_COERCIBLE(this, "String.prototype.toLocaleLowerCase");
+    return localeConvertCase(TO_STRING(this), arguments[0], false);
+  }
+);
+
+
+OverrideFunction(GlobalString.prototype, 'toLocaleUpperCase', function() {
+    if (!IS_UNDEFINED(new.target)) {
+      throw MakeTypeError(kOrdinaryFunctionCalledAsConstructor);
+    }
+    CHECK_OBJECT_COERCIBLE(this, "String.prototype.toLocaleUpperCase");
+    return localeConvertCase(TO_STRING(this), arguments[0], true);
+  }
+);
+
 
 /**
  * Formats a Number object (this) using locale and options values.