Use ICU case conversion/transliterator for case conversion behind a flag

src/js/i18n.js

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // ECMAScript 402 API implementation.
 
 /**
  * Intl object is a single object that has some named properties,
  * all of which are constructors.
  */
@@ skipping 126 matching lines @@
   'numberformat': UNDEFINED,
   'dateformat': UNDEFINED,
   'breakiterator': UNDEFINED
 };
 
 /**
  * Caches default ICU locale.
  */
 var DEFAULT_ICU_LOCALE = UNDEFINED;
 
+function GetDefaultICULocaleJS() {
+  if (IS_UNDEFINED(DEFAULT_ICU_LOCALE)) {
+    DEFAULT_ICU_LOCALE = %GetDefaultICULocale();
+  }
+  return DEFAULT_ICU_LOCALE;
+}
+
 /**
  * Unicode extension regular expression.
  */
 var UNICODE_EXTENSION_RE = UNDEFINED;
 
 function GetUnicodeExtensionRE() {
   if (IS_UNDEFINED(UNDEFINED)) {
     UNICODE_EXTENSION_RE = new GlobalRegExp('-u(-[a-z0-9]{2,8})+', 'g');
   }
   return UNICODE_EXTENSION_RE;
@@ skipping 284 matching lines @@
       // Truncate locale if possible.
       var pos = %_Call(StringLastIndexOf, locale, '-');
       if (pos === -1) {
         break;
       }
       locale = %_Call(StringSubstring, locale, 0, pos);
     } while (true);
   }
 
   // Didn't find a match, return default.
-  if (IS_UNDEFINED(DEFAULT_ICU_LOCALE)) {
-    DEFAULT_ICU_LOCALE = %GetDefaultICULocale();
-  }
-
-  return {'locale': DEFAULT_ICU_LOCALE, 'extension': '', 'position': -1};
+  return {'locale': GetDefaultICULocaleJS(), 'extension': '', 'position': -1};
 }
 
 
 /**
  * Returns best matched supported locale and extension info using
  * implementation dependend algorithm.
  */
 function bestFitMatcher(service, requestedLocales) {
   // TODO(cira): implement better best fit algorithm.
   return lookupMatcher(service, requestedLocales);
@@ skipping 249 matching lines @@
     }
   }
   return result;
 }
 
 /**
  * Canonicalizes the language tag, or throws in case the tag is invalid.
  */
 function canonicalizeLanguageTag(localeID) {
   // null is typeof 'object' so we have to do extra check.
-  if (typeof localeID !== 'string' && typeof localeID !== 'object' ||
+  if ((typeof localeID !== 'string' && typeof localeID !== 'object') ||

[Dan Ehrenberg, 2016/04/20 22:01:29]

Nit: As long as you're changing this code, could you use IS_STRING(localeID),
and IS_RECEIVER(localeID) to be more idiomatic (and slightly more correct)?

[jungshik at Google, 2016/04/21 20:39:17]

Done.

       IS_NULL(localeID)) {
     throw MakeTypeError(kLanguageID);
   }
 
+  // Optimize for the most common case; a language code alone in
+  // the canonical form/lowercase (e.g. "en", "fil").
+  if (typeof localeID === 'string' &&

[Dan Ehrenberg, 2016/04/20 22:01:29]

IS_STRING

[jungshik at Google, 2016/04/21 20:39:16]

Done.

+      !IS_NULL(InternalRegExpMatch(/^[a-z]{2,3}$/, localeID)))
+    return localeID;
+
   var localeString = GlobalString(localeID);
 
   if (isValidLanguageTag(localeString) === false) {
     throw MakeRangeError(kInvalidLanguageTag, localeString);
   }
 
-  // This call will strip -kn but not -kn-true extensions.
-  // ICU bug filled - http://bugs.icu-project.org/trac/ticket/9265.
-  // TODO(cira): check if -u-kn-true-kc-true-kh-true still throws after
-  // upgrade to ICU 4.9.
   var tag = %CanonicalizeLanguageTag(localeString);
   if (tag === 'invalid-tag') {
     throw MakeRangeError(kInvalidLanguageTag, localeString);
   }
 
   return tag;
 }
 
 
 /**
@@ skipping 1235 matching lines @@
   var useOptions = (IS_UNDEFINED(defaults)) ? options : defaults;
   if (IS_UNDEFINED(locales) && IS_UNDEFINED(options)) {
     if (IS_UNDEFINED(defaultObjects[service])) {
       defaultObjects[service] = new savedObjects[service](locales, useOptions);
     }
     return defaultObjects[service];
   }
   return new savedObjects[service](locales, useOptions);
 }
 
+function getCaseConversionLanguageId(locales) {

[Dan Ehrenberg, 2016/04/20 22:01:29]

Generally, the v8 convention is CamelCase with an initial capital letter. Let's
use that for new code.

[jungshik at Google, 2016/04/21 20:39:16]

Done.

+  var language;
+  // Optimize for the most common two cases. initializeLocaleList() can handle
+  // them as well, but it's rather slow accounting for over 60% of
+  // toLocale{U,L}Case() and about 40% of toLocale{U,L}Case("<locale>").
+  if (IS_UNDEFINED(locales)) {
+    language = GetDefaultICULocaleJS();
+  } else if (typeof locales === 'string') {

[Dan Ehrenberg, 2016/04/20 22:01:29]

IS_STRING

[jungshik at Google, 2016/04/21 20:39:16]

Done.

+    language = canonicalizeLanguageTag(locales);
+  } else {
+    var locales = initializeLocaleList(locales);
+    language = locales.length > 0 ? locales[0] : GetDefaultICULocaleJS();
+  }
+
+  // StringSplit is slwoer than this.

[Dan Ehrenberg, 2016/04/20 22:01:29]

slower

Consider factoring this out into a utility function.

[jungshik at Google, 2016/04/21 20:39:16]

Typo fixed. 
 
You mean  "var pos ..  ; if (pos != -1) .... " ? Like this? 

function GetLanguagePart(locale) {
  // StringSplit is slower than this.
  var pos = %_Call(StringIndexOf, locale, '-');
  return pos == -1 ? locale :
      %_Call(StringSubstring, locale, 0, pos);
}

+  var pos = %_Call(StringIndexOf, language, '-');
+  if (pos != -1)
+    language = %_Call(StringSubstring, language, 0, pos);
+
+  var CUSTOM_CASE_LANGUAGES = ['az', 'el', 'lt', 'tr'];

[Dan Ehrenberg, 2016/04/20 22:01:29]

Could we somehow query this from ICU? I thought there were more languages that
had custom case conversion, like French for removing accents over vowels when
going uppercase.

Is there a way to avoid having this parallel array in JS and C++? For example,
what if we just pass the string into the runtime, and let that interpret it.

[jungshik at Google, 2016/04/21 20:39:17]

Some typesetting systems in the past couldn't deal with uppercase with
diacritics, but that's not the case any more. The French Academy is very clear
about what to do about them. (do not drop diacritics). Well, some native French
speakers get confused, too. :-) 

Anyway, what languages are subject to special casings comes from the Unicode
data files, but 
CLDR/ICU does not have that list exposed via API.  

Unicode's SpecialCasing.txt has 'tr', 'az' and 'lt'. 'el' is not there yet.
ECMAScript 402 only lists 'tr', 'az' and 'lt' at the moment citing
SpecialCasing.txt at unicode.org (I'll file a bug against the spec), but 'el'
certainly needs a tailored case mapping. (Blink and Gecko do that, already for
CSS text-transform) per strong demand from Greek speakers (Mozilla's
implementation is more sophisticated than CLDR/ICU's. CLDR/ICU is looking into
revising it). 
I hate parallel arrays, too !! OTOH, I want to avoid passing a string from JS to
C++ unless there's an efficient way to wrap and unwrap it (almost as efficient
as passing an integer).   

StringNormalize does the same (passing integer instead of 'NFC', 'NFKC', "NFD',
'NFKD'). If there's an efficient way to pass one of a small list of strings
between JS and runtime, I'd for sure use that. 

+  return %_Call(ArrayIndexOf, CUSTOM_CASE_LANGUAGES, language);
+}
+
+function localeConvertCase(s, locales, isToUpper) {

[Dan Ehrenberg, 2016/04/20 22:01:29]

LocaleConvertCase

[jungshik at Google, 2016/04/21 20:39:16]

Done.

+  var caseConversionLanguageId = getCaseConversionLanguageId(locales);
+  if (caseConversionLanguageId == -1)
+    return isToUpper ? %StringToUpperCaseI18N(s) : %StringToLowerCaseI18N(s);
+  return %StringLocaleConvertCase(s, isToUpper, caseConversionLanguageId);
+}
+
 /**
  * Compares this and that, and returns less than 0, 0 or greater than 0 value.
  * Overrides the built-in method.
  */
 OverrideFunction(GlobalString.prototype, 'localeCompare', function(that) {
     if (!IS_UNDEFINED(new.target)) {
       throw MakeTypeError(kOrdinaryFunctionCalledAsConstructor);
     }
 
     if (IS_NULL_OR_UNDEFINED(this)) {
@@ skipping 32 matching lines @@
     var normalizationForm = %_Call(ArrayIndexOf, NORMALIZATION_FORMS, form);
     if (normalizationForm === -1) {
       throw MakeRangeError(kNormalizationForm,
           %_Call(ArrayJoin, NORMALIZATION_FORMS, ', '));
     }
 
     return %StringNormalize(s, normalizationForm);
   }
 );
 
+OverrideFunction(GlobalString.prototype, 'toLowerCase', function() {
+    if (!IS_UNDEFINED(new.target)) {
+      throw MakeTypeError(kOrdinaryFunctionCalledAsConstructor);
+    }
+    CHECK_OBJECT_COERCIBLE(this, "String.prototype.toLowerCase");
+    var s = TO_STRING(this);
+    return %StringToLowerCaseI18N(s);

[Dan Ehrenberg, 2016/04/20 22:01:29]

ECMA262 seems to specify using the root locale in
https://tc39.github.io/ecma262/#sec-string.prototype.tolowercase ; however, the
implementation seems to pass "" to ICU, which would use the current default
locale, right?

[jungshik at Google, 2016/04/21 20:39:16]

Right.
"" for root locale and NULL for the default locale :-)

@param locale    The locale to consider, or "" for the root locale or NULL for
the default locale.

+  }
+);
+
+OverrideFunction(GlobalString.prototype, 'toUpperCase', function() {
+    if (!IS_UNDEFINED(new.target)) {
+      throw MakeTypeError(kOrdinaryFunctionCalledAsConstructor);
+    }
+    CHECK_OBJECT_COERCIBLE(this, "String.prototype.toLowerCase");
+    var s = TO_STRING(this);
+    return %StringToUpperCaseI18N(s);
+  }
+);
+
+OverrideFunction(GlobalString.prototype, 'toLocaleLowerCase', function() {
+    if (!IS_UNDEFINED(new.target)) {
+      throw MakeTypeError(kOrdinaryFunctionCalledAsConstructor);
+    }
+    CHECK_OBJECT_COERCIBLE(this, "String.prototype.toLocaleLowerCase");
+    return localeConvertCase(TO_STRING(this), arguments[0], false);
+  }
+);
+
+
+OverrideFunction(GlobalString.prototype, 'toLocaleUpperCase', function() {
+    if (!IS_UNDEFINED(new.target)) {
+      throw MakeTypeError(kOrdinaryFunctionCalledAsConstructor);
+    }
+    CHECK_OBJECT_COERCIBLE(this, "String.prototype.toLocaleUpperCase");
+    return localeConvertCase(TO_STRING(this), arguments[0], true);
+  }
+);
+
 
 /**
  * Formats a Number object (this) using locale and options values.
  * If locale or options are omitted, defaults are used.
  */
 OverrideFunction(GlobalNumber.prototype, 'toLocaleString', function() {
     if (!IS_UNDEFINED(new.target)) {
       throw MakeTypeError(kOrdinaryFunctionCalledAsConstructor);
     }
 
@@ skipping 75 matching lines @@
     }
 
     var locales = arguments[0];
     var options = arguments[1];
     return toLocaleDateTime(
         this, locales, options, 'time', 'time', 'dateformattime');
   }
 );
 
 })