Index: components/url_formatter/url_formatter.cc |
diff --git a/components/url_formatter/url_formatter.cc b/components/url_formatter/url_formatter.cc |
index ed790a9b73915018ecab0043b364f12eea6a3f3f..f3da52cb748435f3f75c4525550882ad9033aeaf 100644 |
--- a/components/url_formatter/url_formatter.cc |
+++ b/components/url_formatter/url_formatter.cc |
@@ -6,6 +6,7 @@ |
#include <algorithm> |
#include <utility> |
+#include <vector> |
#include "base/lazy_instance.h" |
#include "base/macros.h" |
@@ -15,13 +16,9 @@ |
#include "base/strings/utf_offset_string_conversions.h" |
#include "base/strings/utf_string_conversions.h" |
#include "base/threading/thread_local_storage.h" |
-#include "third_party/icu/source/common/unicode/schriter.h" |
+#include "components/url_formatter/idn_spoof_checker.h" |
#include "third_party/icu/source/common/unicode/uidna.h" |
-#include "third_party/icu/source/common/unicode/uniset.h" |
-#include "third_party/icu/source/common/unicode/uscript.h" |
-#include "third_party/icu/source/common/unicode/uvernum.h" |
-#include "third_party/icu/source/i18n/unicode/regex.h" |
-#include "third_party/icu/source/i18n/unicode/uspoof.h" |
+#include "third_party/icu/source/common/unicode/utypes.h" |
#include "url/gurl.h" |
#include "url/third_party/mozilla/url_parse.h" |
@@ -191,6 +188,9 @@ base::string16 FormatViewSourceUrl( |
return result; |
} |
+base::LazyInstance<IDNSpoofChecker>::Leaky g_idn_spoof_checker = |
+ LAZY_INSTANCE_INITIALIZER; |
+ |
// TODO(brettw): We may want to skip this step in the case of file URLs to |
// allow unicode UNC hostnames regardless of encodings. |
base::string16 IDNToUnicodeWithAdjustments( |
@@ -242,309 +242,12 @@ base::string16 IDNToUnicodeWithAdjustments( |
return out16; |
} |
-// A helper class for IDN Spoof checking, used to ensure that no IDN input is |
-// spoofable per Chromium's standard of spoofability. For a more thorough |
-// explanation of how spoof checking works in Chromium, see |
-// http://dev.chromium.org/developers/design-documents/idn-in-google-chrome . |
-class IDNSpoofChecker { |
- public: |
- IDNSpoofChecker(); |
- |
- // Returns true if |label| is safe to display as Unicode. When the TLD is |
- // ASCII, check if a label is entirely made of Cyrillic letters that look like |
- // Latin letters. In the event of library failure, all IDN inputs will be |
- // treated as unsafe. |
- bool Check(base::StringPiece16 label, bool is_tld_ascii); |
- |
- private: |
- void SetAllowedUnicodeSet(UErrorCode* status); |
- bool IsMadeOfLatinAlikeCyrillic(const icu::UnicodeString& label_string); |
- |
- USpoofChecker* checker_; |
- icu::UnicodeSet deviation_characters_; |
- icu::UnicodeSet non_ascii_latin_letters_; |
- icu::UnicodeSet kana_letters_exceptions_; |
- icu::UnicodeSet cyrillic_letters_; |
- icu::UnicodeSet cyrillic_letters_latin_alike_; |
- |
- DISALLOW_COPY_AND_ASSIGN(IDNSpoofChecker); |
-}; |
- |
-base::LazyInstance<IDNSpoofChecker>::Leaky g_idn_spoof_checker = |
- LAZY_INSTANCE_INITIALIZER; |
-base::ThreadLocalStorage::StaticSlot tls_index = TLS_INITIALIZER; |
- |
-void OnThreadTermination(void* regex_matcher) { |
- delete reinterpret_cast<icu::RegexMatcher*>(regex_matcher); |
-} |
- |
-IDNSpoofChecker::IDNSpoofChecker() { |
- UErrorCode status = U_ZERO_ERROR; |
- checker_ = uspoof_open(&status); |
- if (U_FAILURE(status)) { |
- checker_ = nullptr; |
- return; |
- } |
- |
- // At this point, USpoofChecker has all the checks enabled except |
- // for USPOOF_CHAR_LIMIT (USPOOF_{RESTRICTION_LEVEL, INVISIBLE, |
- // MIXED_SCRIPT_CONFUSABLE, WHOLE_SCRIPT_CONFUSABLE, MIXED_NUMBERS, ANY_CASE}) |
- // This default configuration is adjusted below as necessary. |
- |
- // Set the restriction level to moderate. It allows mixing Latin with another |
- // script (+ COMMON and INHERITED). Except for Chinese(Han + Bopomofo), |
- // Japanese(Hiragana + Katakana + Han), and Korean(Hangul + Han), only one |
- // script other than Common and Inherited can be mixed with Latin. Cyrillic |
- // and Greek are not allowed to mix with Latin. |
- // See http://www.unicode.org/reports/tr39/#Restriction_Level_Detection |
- uspoof_setRestrictionLevel(checker_, USPOOF_MODERATELY_RESTRICTIVE); |
- |
- // Restrict allowed characters in IDN labels and turn on USPOOF_CHAR_LIMIT. |
- SetAllowedUnicodeSet(&status); |
- |
- // Enable the return of auxillary (non-error) information. |
- // We used to disable WHOLE_SCRIPT_CONFUSABLE check explicitly, but as of |
- // ICU 58.1, WSC is a no-op in a single string check API. |
- int32_t checks = uspoof_getChecks(checker_, &status) | USPOOF_AUX_INFO; |
- uspoof_setChecks(checker_, checks, &status); |
- |
- // Four characters handled differently by IDNA 2003 and IDNA 2008. UTS46 |
- // transitional processing treats them as IDNA 2003 does; maps U+00DF and |
- // U+03C2 and drops U+200[CD]. |
- deviation_characters_ = |
- icu::UnicodeSet(UNICODE_STRING_SIMPLE("[\\u00df\\u03c2\\u200c\\u200d]"), |
- status); |
- deviation_characters_.freeze(); |
- |
- // Latin letters outside ASCII. 'Script_Extensions=Latin' is not necessary |
- // because additional characters pulled in with scx=Latn are not included in |
- // the allowed set. |
- non_ascii_latin_letters_ = icu::UnicodeSet( |
- UNICODE_STRING_SIMPLE("[[:Latin:] - [a-zA-Z]]"), status); |
- non_ascii_latin_letters_.freeze(); |
- |
- // These letters are parts of |dangerous_patterns_|. |
- kana_letters_exceptions_ = icu::UnicodeSet( |
- UNICODE_STRING_SIMPLE("[\\u3078-\\u307a\\u30d8-\\u30da\\u30fb-\\u30fe]"), |
- status); |
- kana_letters_exceptions_.freeze(); |
- |
- // These Cyrillic letters look like Latin. A domain label entirely made of |
- // these letters is blocked as a simplified whole-script-spoofable. |
- cyrillic_letters_latin_alike_ = |
- icu::UnicodeSet(icu::UnicodeString("[асԁеһіјӏорԛѕԝхуъЬҽпгѵѡ]"), status); |
- cyrillic_letters_latin_alike_.freeze(); |
- |
- cyrillic_letters_ = |
- icu::UnicodeSet(UNICODE_STRING_SIMPLE("[[:Cyrl:]]"), status); |
- cyrillic_letters_.freeze(); |
- |
- DCHECK(U_SUCCESS(status)); |
-} |
- |
-bool IDNSpoofChecker::Check(base::StringPiece16 label, bool is_tld_ascii) { |
- UErrorCode status = U_ZERO_ERROR; |
- int32_t result = uspoof_check(checker_, label.data(), |
- base::checked_cast<int32_t>(label.size()), |
- NULL, &status); |
- // If uspoof_check fails (due to library failure), or if any of the checks |
- // fail, treat the IDN as unsafe. |
- if (U_FAILURE(status) || (result & USPOOF_ALL_CHECKS)) |
- return false; |
- |
- icu::UnicodeString label_string(FALSE, label.data(), |
- base::checked_cast<int32_t>(label.size())); |
- |
- // A punycode label with 'xn--' prefix is not subject to the URL |
- // canonicalization and is stored as it is in GURL. If it encodes a deviation |
- // character (UTS 46; e.g. U+00DF/sharp-s), it should be still shown in |
- // punycode instead of Unicode. Without this check, xn--fu-hia for |
- // 'fu<sharp-s>' would be converted to 'fu<sharp-s>' for display because |
- // "UTS 46 section 4 Processing step 4" applies validity criteria for |
- // non-transitional processing (i.e. do not map deviation characters) to any |
- // punycode labels regardless of whether transitional or non-transitional is |
- // chosen. On the other hand, 'fu<sharp-s>' typed or copy and pasted |
- // as Unicode would be canonicalized to 'fuss' by GURL and is displayed as |
- // such. See http://crbug.com/595263 . |
- if (deviation_characters_.containsSome(label_string)) |
- return false; |
- |
- // If there's no script mixing, the input is regarded as safe without any |
- // extra check unless it contains Kana letter exceptions or it's made entirely |
- // of Cyrillic letters that look like Latin letters. Note that the following |
- // combinations of scripts are treated as a 'logical' single script. |
- // - Chinese: Han, Bopomofo, Common |
- // - Japanese: Han, Hiragana, Katakana, Common |
- // - Korean: Hangul, Han, Common |
- result &= USPOOF_RESTRICTION_LEVEL_MASK; |
- if (result == USPOOF_ASCII) return true; |
- if (result == USPOOF_SINGLE_SCRIPT_RESTRICTIVE && |
- kana_letters_exceptions_.containsNone(label_string)) { |
- // Check Cyrillic confusable only for ASCII TLDs. |
- return !is_tld_ascii || !IsMadeOfLatinAlikeCyrillic(label_string); |
- } |
- |
- // Additional checks for |label| with multiple scripts, one of which is Latin. |
- // Disallow non-ASCII Latin letters to mix with a non-Latin script. |
- if (non_ascii_latin_letters_.containsSome(label_string)) |
- return false; |
- |
- if (!tls_index.initialized()) |
- tls_index.Initialize(&OnThreadTermination); |
- icu::RegexMatcher* dangerous_pattern = |
- reinterpret_cast<icu::RegexMatcher*>(tls_index.Get()); |
- if (!dangerous_pattern) { |
- // Disallow the katakana no, so, zo, or n, as they may be mistaken for |
- // slashes when they're surrounded by non-Japanese scripts (i.e. scripts |
- // other than Katakana, Hiragana or Han). If {no, so, zo, n} next to a |
- // non-Japanese script on either side is disallowed, legitimate cases like |
- // '{vitamin in Katakana}b6' are blocked. Note that trying to block those |
- // characters when used alone as a label is futile because those cases |
- // would not reach here. |
- // Also disallow what used to be blocked by mixed-script-confusable (MSC) |
- // detection. ICU 58 does not detect MSC any more for a single input string. |
- // See http://bugs.icu-project.org/trac/ticket/12823 . |
- // TODO(jshin): adjust the pattern once the above ICU bug is fixed. |
- // - Disallow U+30FB (Katakana Middle Dot) and U+30FC (Hiragana-Katakana |
- // Prolonged Sound) used out-of-context. |
- // - Dislallow U+30FD/E (Katakana iteration mark/voiced iteration mark) |
- // unless they're preceded by a Katakana. |
- // - Disallow three Hiragana letters (U+307[8-A]) or Katakana letters |
- // (U+30D[8-A]) that look exactly like each other when they're used in a |
- // label otherwise entirely in Katakna or Hiragana. |
- // - Disallow U+0585 (Armenian Small Letter Oh) and U+0581 (Armenian Small |
- // Letter Co) to be next to Latin. |
- // - Disallow Latin 'o' and 'g' next to Armenian. |
- // - Disalow mixing of Latin and Canadian Syllabary. |
- dangerous_pattern = new icu::RegexMatcher( |
- icu::UnicodeString( |
- "[^\\p{scx=kana}\\p{scx=hira}\\p{scx=hani}]" |
- "[\\u30ce\\u30f3\\u30bd\\u30be]" |
- "[^\\p{scx=kana}\\p{scx=hira}\\p{scx=hani}]|" |
- "[^\\p{scx=kana}\\p{scx=hira}]\\u30fc|^\\u30fc|" |
- "[^\\p{scx=kana}][\\u30fd\\u30fe]|^[\\u30fd\\u30fe]|" |
- "^[\\p{scx=kana}]+[\\u3078-\\u307a][\\p{scx=kana}]+$|" |
- "^[\\p{scx=hira}]+[\\u30d8-\\u30da][\\p{scx=hira}]+$|" |
- "[a-z]\\u30fb|\\u30fb[a-z]|" |
- "^[\\u0585\\u0581]+[a-z]|[a-z][\\u0585\\u0581]+$|" |
- "[a-z][\\u0585\\u0581]+[a-z]|" |
- "^[og]+[\\p{scx=armn}]|[\\p{scx=armn}][og]+$|" |
- "[\\p{scx=armn}][og]+[\\p{scx=armn}]|" |
- "[\\p{sc=cans}].*[a-z]|[a-z].*[\\p{sc=cans}]", |
- -1, US_INV), |
- 0, status); |
- tls_index.Set(dangerous_pattern); |
- } |
- dangerous_pattern->reset(label_string); |
- return !dangerous_pattern->find(); |
-} |
- |
-bool IDNSpoofChecker::IsMadeOfLatinAlikeCyrillic( |
- const icu::UnicodeString& label_string) { |
- // Collect all the Cyrillic letters in |label_string| and see if they're |
- // a subset of |cyrillic_letters_latin_alike_|. |
- // A shortcut of defining cyrillic_letters_latin_alike_ to include [0-9] and |
- // [_-] and checking if the set contains all letters of |label_string| |
- // would work in most cases, but not if a label has non-letters outside |
- // ASCII. |
- icu::UnicodeSet cyrillic_in_label; |
- icu::StringCharacterIterator it(label_string); |
- for (it.setToStart(); it.hasNext();) { |
- const UChar32 c = it.next32PostInc(); |
- if (cyrillic_letters_.contains(c)) |
- cyrillic_in_label.add(c); |
- } |
- return !cyrillic_in_label.isEmpty() && |
- cyrillic_letters_latin_alike_.containsAll(cyrillic_in_label); |
-} |
- |
-void IDNSpoofChecker::SetAllowedUnicodeSet(UErrorCode* status) { |
- if (U_FAILURE(*status)) |
- return; |
- |
- // The recommended set is a set of characters for identifiers in a |
- // security-sensitive environment taken from UTR 39 |
- // (http://unicode.org/reports/tr39/) and |
- // http://www.unicode.org/Public/security/latest/xidmodifications.txt . |
- // The inclusion set comes from "Candidate Characters for Inclusion |
- // in idenfiers" of UTR 31 (http://www.unicode.org/reports/tr31). The list |
- // may change over the time and will be updated whenever the version of ICU |
- // used in Chromium is updated. |
- const icu::UnicodeSet* recommended_set = |
- uspoof_getRecommendedUnicodeSet(status); |
- icu::UnicodeSet allowed_set; |
- allowed_set.addAll(*recommended_set); |
- const icu::UnicodeSet* inclusion_set = uspoof_getInclusionUnicodeSet(status); |
- allowed_set.addAll(*inclusion_set); |
- |
- // Five aspirational scripts are taken from UTR 31 Table 6 at |
- // http://www.unicode.org/reports/tr31/#Aspirational_Use_Scripts . |
- // Not all the characters of aspirational scripts are suitable for |
- // identifiers. Therefore, only characters belonging to |
- // [:Identifier_Type=Aspirational:] (listed in 'Status/Type=Aspirational' |
- // section at |
- // http://www.unicode.org/Public/security/latest/xidmodifications.txt) are |
- // are added to the allowed set. The list has to be updated when a new |
- // version of Unicode is released. The current version is 9.0.0 and ICU 60 |
- // will have Unicode 10.0 data. |
-#if U_ICU_VERSION_MAJOR_NUM < 60 |
- const icu::UnicodeSet aspirational_scripts( |
- icu::UnicodeString( |
- // Unified Canadian Syllabics |
- "[\\u1401-\\u166C\\u166F-\\u167F" |
- // Mongolian |
- "\\u1810-\\u1819\\u1820-\\u1877\\u1880-\\u18AA" |
- // Unified Canadian Syllabics |
- "\\u18B0-\\u18F5" |
- // Tifinagh |
- "\\u2D30-\\u2D67\\u2D7F" |
- // Yi |
- "\\uA000-\\uA48C" |
- // Miao |
- "\\U00016F00-\\U00016F44\\U00016F50-\\U00016F7E" |
- "\\U00016F8F-\\U00016F9F]", |
- -1, US_INV), |
- *status); |
- allowed_set.addAll(aspirational_scripts); |
-#else |
-#error "Update aspirational_scripts per Unicode 10.0" |
-#endif |
- |
- // U+0338 is included in the recommended set, while U+05F4 and U+2027 are in |
- // the inclusion set. However, they are blacklisted as a part of Mozilla's |
- // IDN blacklist (http://kb.mozillazine.org/Network.IDN.blacklist_chars). |
- // U+2010 is in the inclusion set, but we drop it because it can be confused |
- // with an ASCII U+002D (Hyphen-Minus). |
- // U+0338 and U+2027 are dropped; the former can look like a slash when |
- // rendered with a broken font, and the latter can be confused with U+30FB |
- // (Katakana Middle Dot). U+05F4 (Hebrew Punctuation Gershayim) is kept, |
- // even though it can look like a double quotation mark. Using it in Hebrew |
- // should be safe. When used with a non-Hebrew script, it'd be filtered by |
- // other checks in place. |
- allowed_set.remove(0x338u); // Combining Long Solidus Overlay |
- allowed_set.remove(0x2010u); // Hyphen |
- allowed_set.remove(0x2027u); // Hyphenation Point |
- |
-#if defined(OS_MACOSX) |
- // The following characters are reported as present in the default macOS |
- // system UI font, but they render as blank. Remove them from the allowed |
- // set to prevent spoofing. |
- // Tibetan characters used for transliteration of ancient texts: |
- allowed_set.remove(0x0F8Cu); |
- allowed_set.remove(0x0F8Du); |
- allowed_set.remove(0x0F8Eu); |
- allowed_set.remove(0x0F8Fu); |
-#endif |
- |
- uspoof_setAllowedUnicodeSet(checker_, &allowed_set, status); |
-} |
- |
// Returns true if the given Unicode host component is safe to display to the |
// user. Note that this function does not deal with pure ASCII domain labels at |
// all even though it's possible to make up look-alike labels with ASCII |
// characters alone. |
bool IsIDNComponentSafe(base::StringPiece16 label, bool is_tld_ascii) { |
- return g_idn_spoof_checker.Get().Check(label, is_tld_ascii); |
+ return g_idn_spoof_checker.Get().SafeToDisplayAsUnicode(label, is_tld_ascii); |
} |
// A wrapper to use LazyInstance<>::Leaky with ICU's UIDNA, a C pointer to |