Update ICU to 58.1

components/url_formatter/url_formatter.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/url_formatter/url_formatter.h"
 
 #include <algorithm>
 #include <utility>
 
 #include "base/lazy_instance.h"
@@ skipping 232 matching lines @@
 
   // Returns true if |label| is safe to display as Unicode. In the event of
   // library failure, all IDN inputs will be treated as unsafe.
   bool Check(base::StringPiece16 label);
 
  private:
   void SetAllowedUnicodeSet(UErrorCode* status);
 
   USpoofChecker* checker_;
   icu::UnicodeSet deviation_characters_;
-  icu::UnicodeSet latin_letters_;
   icu::UnicodeSet non_ascii_latin_letters_;
+  icu::UnicodeSet kana_letters_exceptions_;
 
   DISALLOW_COPY_AND_ASSIGN(IDNSpoofChecker);
 };
 
 base::LazyInstance<IDNSpoofChecker>::Leaky g_idn_spoof_checker =
     LAZY_INSTANCE_INITIALIZER;
 base::ThreadLocalStorage::StaticSlot tls_index = TLS_INITIALIZER;
 
 void OnThreadTermination(void* regex_matcher) {
   delete reinterpret_cast<icu::RegexMatcher*>(regex_matcher);
@@ skipping 17 matching lines @@
   // Japanese(Hiragana + Katakana + Han), and Korean(Hangul + Han), only one
   // script other than Common and Inherited can be mixed with Latin. Cyrillic
   // and Greek are not allowed to mix with Latin.
   // See http://www.unicode.org/reports/tr39/#Restriction_Level_Detection
   uspoof_setRestrictionLevel(checker_, USPOOF_MODERATELY_RESTRICTIVE);
 
   // Restrict allowed characters in IDN labels and turn on USPOOF_CHAR_LIMIT.
   SetAllowedUnicodeSet(&status);
 
   // Enable the return of auxillary (non-error) information.
+  // We used to disable WHOLE_SCRIPT_CONFUSABLE check explicitly, but as of
+  // ICU 58.1, WSC is a no-op in a single string check API.
   int32_t checks = uspoof_getChecks(checker_, &status) | USPOOF_AUX_INFO;
-
-  // Disable WHOLE_SCRIPT_CONFUSABLE check. The check has a marginal value when
-  // used against a single string as opposed to comparing a pair of strings. In
-  // addition, it would also flag a number of common labels including the IDN
-  // TLD for Russian.
-  // A possible alternative would be to turn on the check and block a label
-  // only under the following conditions, but it'd better be done on the
-  // server-side (e.g. SafeBrowsing):
-  // 1. The label is whole-script confusable.
-  // 2. And the skeleton of the label matches the skeleton of one of top
-  // domain labels. See http://unicode.org/reports/tr39/#Confusable_Detection
-  // for the definition of skeleton.
-  // 3. And the label is different from the matched top domain label in #2.
-  checks &= ~USPOOF_WHOLE_SCRIPT_CONFUSABLE;
-
   uspoof_setChecks(checker_, checks, &status);
 
   // Four characters handled differently by IDNA 2003 and IDNA 2008. UTS46
   // transitional processing treats them as IDNA 2003 does; maps U+00DF and
   // U+03C2 and drops U+200[CD].
   deviation_characters_ =
       icu::UnicodeSet(UNICODE_STRING_SIMPLE("[\\u00df\\u03c2\\u200c\\u200d]"),
                       status);
   deviation_characters_.freeze();
 
-  latin_letters_ =
-      icu::UnicodeSet(UNICODE_STRING_SIMPLE("[:Latin:]"), status);
-  latin_letters_.freeze();
-
   // Latin letters outside ASCII. 'Script_Extensions=Latin' is not necessary
   // because additional characters pulled in with scx=Latn are not included in
   // the allowed set.
   non_ascii_latin_letters_ = icu::UnicodeSet(
       UNICODE_STRING_SIMPLE("[[:Latin:] - [a-zA-Z]]"), status);
   non_ascii_latin_letters_.freeze();
 
+  // These letters are parts of |dangerous_patterns_|.
+  kana_letters_exceptions_ = icu::UnicodeSet(UNICODE_STRING_SIMPLE(
+      "[\\u3078-\\u307a\\u30d8-\\u30da\\u30fb\\u30fc]"), status);
+  kana_letters_exceptions_.freeze();
+
   DCHECK(U_SUCCESS(status));
 }
 
 bool IDNSpoofChecker::Check(base::StringPiece16 label) {
   UErrorCode status = U_ZERO_ERROR;
   int32_t result = uspoof_check(checker_, label.data(),
                                 base::checked_cast<int32_t>(label.size()),
                                 NULL, &status);
   // If uspoof_check fails (due to library failure), or if any of the checks
   // fail, treat the IDN as unsafe.
@@ skipping 11 matching lines @@
   // "UTS 46 section 4 Processing step 4" applies validity criteria for
   // non-transitional processing (i.e. do not map deviation characters) to any
   // punycode labels regardless of whether transitional or non-transitional is
   // chosen. On the other hand, 'fu<sharp-s>' typed or copy and pasted
   // as Unicode would be canonicalized to 'fuss' by GURL and is displayed as
   // such. See http://crbug.com/595263 .
   if (deviation_characters_.containsSome(label_string))
     return false;
 
   // If there's no script mixing, the input is regarded as safe without any
-  // extra check.
-  result &= USPOOF_RESTRICTION_LEVEL_MASK;
-  if (result == USPOOF_ASCII || result == USPOOF_SINGLE_SCRIPT_RESTRICTIVE)
-    return true;
-
-  // When check is passed at 'highly restrictive' level, |label| is
-  // made up of one of the following script sets optionally mixed with Latin.
+  // extra check unless it contains Kana letter exceptions. Note that
+  // the following combinations of scripts are treated as a 'logical' single
+  // script.
   //  - Chinese: Han, Bopomofo, Common
   //  - Japanese: Han, Hiragana, Katakana, Common
   //  - Korean: Hangul, Han, Common
-  // Treat this case as a 'logical' single script unless Latin is mixed.
-  if (result == USPOOF_HIGHLY_RESTRICTIVE &&
-      latin_letters_.containsNone(label_string))
+  result &= USPOOF_RESTRICTION_LEVEL_MASK;
+  if (result == USPOOF_ASCII ||
+      (result == USPOOF_SINGLE_SCRIPT_RESTRICTIVE &&
+       kana_letters_exceptions_.containsNone(label_string)))
     return true;
 
   // Additional checks for |label| with multiple scripts, one of which is Latin.
   // Disallow non-ASCII Latin letters to mix with a non-Latin script.
   if (non_ascii_latin_letters_.containsSome(label_string))
     return false;
 
   if (!tls_index.initialized())
     tls_index.Initialize(&OnThreadTermination);
   icu::RegexMatcher* dangerous_pattern =
       reinterpret_cast<icu::RegexMatcher*>(tls_index.Get());
   if (!dangerous_pattern) {
     // Disallow the katakana no, so, zo, or n, as they may be mistaken for
     // slashes when they're surrounded by non-Japanese scripts (i.e. scripts
     // other than Katakana, Hiragana or Han). If {no, so, zo, n} next to a
     // non-Japanese script on either side is disallowed, legitimate cases like
     // '{vitamin in Katakana}b6' are blocked. Note that trying to block those
     // characters when used alone as a label is futile because those cases
     // would not reach here.
+    // Besides, disallow what used to be blocked by mixed-script-confusable(MSC)

[Peter Kasting, 2016/10/28 22:46:18]

Nit "Besides," -> "Also"; space before '('

+    // detection. ICU 58 does not detect MSC any more for a single input string.
+    // See http://bugs.icu-project.org/trac/ticket/12823 .
+    // - Disallow U+30FB(Katakana Middle Dot) and U+30FC(Hiragana-Katakana
+    //   Prolonged Sound) used out-of-context.
+    // - Disallow three Hiragana letters(U+307[8-A]) or Katakana letters
+    //   (U+30D[8-A]) that look exactly like each other when they're used in a
+    //   label otherwise entirely in Katakna or Hiragana.
+    // - Disallow U+0585 (Armenian Small Letter Oh) to be mixed with Latin.
     dangerous_pattern = new icu::RegexMatcher(
         icu::UnicodeString(
             "[^\\p{scx=kana}\\p{scx=hira}\\p{scx=hani}]"
             "[\\u30ce\\u30f3\\u30bd\\u30be]"
-            "[^\\p{scx=kana}\\p{scx=hira}\\p{scx=hani}]", -1, US_INV),
+            "[^\\p{scx=kana}\\p{scx=hira}\\p{scx=hani}]|"
+            "[^\\p{scx=kana}\\p{scx=hira}]\\u30fc|"
+            "\\u30fc[^\\p{scx=kana}\\p{scx=hira}]|"
+            "^[\\p{scx=kana}]+[\\u3078-\\u307a][\\p{scx=kana}]+$|"
+            "^[\\p{scx=hira}]+[\\u30d8-\\u30da][\\p{scx=hira}]+$|"
+            "[a-z]\\u30fb|\\u30fb[a-z]|"
+            "\\u0585.*[a-z]+|[a-z]+\\u0585", -1, US_INV),

[Peter Kasting, 2016/10/28 22:46:18]

Note, I do not speak regex enough to really review this, assuming you did it
right + smartly

         0, status);
     tls_index.Set(dangerous_pattern);
   }
   dangerous_pattern->reset(label_string);
   return !dangerous_pattern->find();
 }
 
 void IDNSpoofChecker::SetAllowedUnicodeSet(UErrorCode* status) {
   if (U_FAILURE(*status))
     return;
@@ skipping 383 matching lines @@
   return base::StartsWith(text, www, base::CompareCase::SENSITIVE)
       ? text.substr(www.length()) : text;
 }
 
 base::string16 StripWWWFromHost(const GURL& url) {
   DCHECK(url.is_valid());
   return StripWWW(base::ASCIIToUTF16(url.host_piece()));
 }
 
 }  // namespace url_formatter