Implement a new IDN display policy

net/base/net_util_icu.cc

Index: net/base/net_util_icu.cc
diff --git a/net/base/net_util_icu.cc b/net/base/net_util_icu.cc
index 259baba33bd41ef781afe121579857c95b602dc2..ad40aeccadfba89c71d12d9107dd6998b558bf49 100644
--- a/net/base/net_util_icu.cc
+++ b/net/base/net_util_icu.cc
@@ -4,16 +4,13 @@
 
 #include "net/base/net_util.h"
 
-#include <map>
 #include <vector>
 
 #include "base/i18n/time_formatting.h"
 #include "base/json/string_escape.h"
 #include "base/lazy_instance.h"
 #include "base/logging.h"
-#include "base/memory/singleton.h"
-#include "base/stl_util.h"
-#include "base/strings/string_tokenizer.h"
+#include "base/memory/scoped_ptr.h"
 #include "base/strings/string_util.h"
 #include "base/strings/utf_offset_string_conversions.h"
 #include "base/strings/utf_string_conversions.h"
@@ -21,11 +18,9 @@
 #include "url/gurl.h"
 #include "third_party/icu/source/common/unicode/uidna.h"
 #include "third_party/icu/source/common/unicode/uniset.h"
-#include "third_party/icu/source/common/unicode/uscript.h"
-#include "third_party/icu/source/common/unicode/uset.h"
 #include "third_party/icu/source/i18n/unicode/datefmt.h"
 #include "third_party/icu/source/i18n/unicode/regex.h"
-#include "third_party/icu/source/i18n/unicode/ulocdata.h"
+#include "third_party/icu/source/i18n/unicode/uspoof.h"
 
 using base::Time;
 
@@ -35,241 +30,162 @@ namespace {
 
 typedef std::vector<size_t> Offsets;
 
-// Does some simple normalization of scripts so we can allow certain scripts
-// to exist together.
-// TODO(brettw) bug 880223: we should allow some other languages to be
-// oombined such as Chinese and Latin. We will probably need a more
-// complicated system of language pairs to have more fine-grained control.
-UScriptCode NormalizeScript(UScriptCode code) {
-  switch (code) {
-    case USCRIPT_KATAKANA:
-    case USCRIPT_HIRAGANA:
-    case USCRIPT_KATAKANA_OR_HIRAGANA:
-    case USCRIPT_HANGUL:  // This one is arguable.
-      return USCRIPT_HAN;
-    default:
-      return code;
-  }
-}
-
-bool IsIDNComponentInSingleScript(const base::char16* str, int str_len) {
-  UScriptCode first_script = USCRIPT_INVALID_CODE;
-  bool is_first = true;
-
-  int i = 0;
-  while (i < str_len) {
-    unsigned code_point;
-    U16_NEXT(str, i, str_len, code_point);
-
-    UErrorCode err = U_ZERO_ERROR;
-    UScriptCode cur_script = uscript_getScript(code_point, &err);
-    if (err != U_ZERO_ERROR)
-      return false;  // Report mixed on error.
-    cur_script = NormalizeScript(cur_script);
-
-    // TODO(brettw) We may have to check for USCRIPT_INHERENT as well.
-    if (is_first && cur_script != USCRIPT_COMMON) {
-      first_script = cur_script;
-      is_first = false;
-    } else {
-      if (cur_script != USCRIPT_COMMON && cur_script != first_script)
-        return false;
-    }
-  }
-  return true;
-}
+class IDNSpoofChecker {

[Ryan Sleevi, 2015/08/28 00:35:15]

Document

[jungshik at Google, 2015/09/01 19:47:07]

Done.

+ public:
+  IDNSpoofChecker();
+  bool check(const base::char16* label, int label_len);

[Ryan Sleevi, 2015/08/28 00:35:14]

naming nit: These should be called Check
DANGER: using "int" for buffer length's is quite dangerous! Would a
base::StringPiece16 work here (at least to control the danger). Alternatively,
would size_t work with a checked cast?

[Ryan Sleevi, 2015/08/28 00:35:15]

Document

[jungshik at Google, 2015/09/01 19:47:07]

Done.

[jungshik at Google, 2015/09/01 19:47:07]

Done.

[jungshik at Google, 2015/09/01 19:47:07]

I switched to StringPiece16 (I thought about it but didn't bother...). 

 
-// Check if the script of a language can be 'safely' mixed with
-// Latin letters in the ASCII range.
-bool IsCompatibleWithASCIILetters(const std::string& lang) {
-  // For now, just list Chinese, Japanese and Korean (positive list).
-  // An alternative is negative-listing (languages using Greek and
-  // Cyrillic letters), but it can be more dangerous.
-  return !lang.substr(0, 2).compare("zh") ||
-         !lang.substr(0, 2).compare("ja") ||
-         !lang.substr(0, 2).compare("ko");
-}
+ private:
+  USpoofChecker* checker_;
+  DISALLOW_COPY_AND_ASSIGN(IDNSpoofChecker);
+};
 
-typedef std::map<std::string, icu::UnicodeSet*> LangToExemplarSetMap;
+base::LazyInstance<IDNSpoofChecker>::Leaky g_idn_spoof_checker =
+    LAZY_INSTANCE_INITIALIZER;
 
-class LangToExemplarSet {
+class IDNSpoofCheckerExtra {

[Ryan Sleevi, 2015/08/28 00:35:15]

Document

[jungshik at Google, 2015/09/01 19:47:07]

Done.

  public:
-  static LangToExemplarSet* GetInstance() {
-    return Singleton<LangToExemplarSet>::get();
-  }
+  IDNSpoofCheckerExtra();
+  bool check(const base::char16* label, int label_len);

[Ryan Sleevi, 2015/08/28 00:35:15]

Same comments as above

[jungshik at Google, 2015/09/01 19:47:07]

Done.

 
  private:
-  LangToExemplarSetMap map;
-  LangToExemplarSet() { }
-  ~LangToExemplarSet() {
-    STLDeleteContainerPairSecondPointers(map.begin(), map.end());
-  }
-
-  friend class Singleton<LangToExemplarSet>;
-  friend struct DefaultSingletonTraits<LangToExemplarSet>;
-  friend bool GetExemplarSetForLang(const std::string&, icu::UnicodeSet**);
-  friend void SetExemplarSetForLang(const std::string&, icu::UnicodeSet*);
-
-  DISALLOW_COPY_AND_ASSIGN(LangToExemplarSet);
+  icu::UnicodeSet non_ascii_latin_;
+  icu::RegexPattern* dangerous_pattern_;
+  DISALLOW_COPY_AND_ASSIGN(IDNSpoofCheckerExtra);
 };
 
-bool GetExemplarSetForLang(const std::string& lang,
-                           icu::UnicodeSet** lang_set) {
-  const LangToExemplarSetMap& map = LangToExemplarSet::GetInstance()->map;
-  LangToExemplarSetMap::const_iterator pos = map.find(lang);
-  if (pos != map.end()) {
-    *lang_set = pos->second;
-    return true;
-  }
-  return false;
-}
+base::LazyInstance<IDNSpoofCheckerExtra>::Leaky g_idn_spoof_checker_extra =
+    LAZY_INSTANCE_INITIALIZER;
 
-void SetExemplarSetForLang(const std::string& lang,
-                           icu::UnicodeSet* lang_set) {
-  LangToExemplarSetMap& map = LangToExemplarSet::GetInstance()->map;
-  map.insert(std::make_pair(lang, lang_set));
+IDNSpoofChecker::IDNSpoofChecker() {
+  UErrorCode status = U_ZERO_ERROR;
+  checker_ = uspoof_open(&status);
+  DCHECK(U_SUCCESS(status)) << "spoof checker failed to open with error: "

[Ryan Sleevi, 2015/08/28 00:35:15]

Is DCHECK really appropriate? Does this indicate programmer error? Or fatal
library failure?

[jungshik at Google, 2015/09/01 19:47:07]

This should never happen unless for some reason, ICU data is not available or
ICU data is broken/malformed. Unfortunately, there are mysterious crash reports
that could only happen with ICU data missing/unavailable/broken on Android / iOS
(e.g. internal bug : b/23186531 ). 

If the ICU data is missing/malformed/.., we're hosed anyway and we can just use
CHECK here. 

Alternatively, we can move along (disabling IDN spoof check and displaying all
the IDNs in punycode) hoping that the rest of ICU is somehow all right. 

+                            << u_errorName(status);
+
+  // Use 'hightly restrictive' restritiction level to limit the script mixing
+  // to Latin + Han + {Hiragana + Katakana, Bopomofo, Hangul}.
+  // See http://www.unicode.org/reports/tr39/#Restriction_Level_Detection
+  // The default is highly restrictive so that it's not set explicitly.
+  // TODO(jshin): Firefox uses 'moderately restrictive' by default. Review
+  // using that, instead.
+  uspoof_setRestrictionLevel(checker_, USPOOF_HIGHLY_RESTRICTIVE);
+
+  // The recommended set and inclusion set come from
+  // http://unicode.org/reports/tr39/ and
+  // http://www.unicode.org/Public/security/latest/xidmodifications.txt
+  // The list can undergo some changes as a new version of Unicode is
+  // released and we update our copy of ICU.
+  const icu::UnicodeSet* recommended_set =
+      uspoof_getRecommendedUnicodeSet(&status);
+  icu::UnicodeSet allowed_set;
+  allowed_set.addAll(*recommended_set);
+  const icu::UnicodeSet* inclusion_set = uspoof_getInclusionUnicodeSet(&status);
+  allowed_set.addAll(*inclusion_set);
+
+  // From UAX 31 Table 6:
+  // http://www.unicode.org/reports/tr31/#Aspirational_Use_Scripts
+  const icu::UnicodeSet aspirational_scripts(
+      UNICODE_STRING_SIMPLE(
+          "[[:sc=Cans:][:sc=Plrd:][:sc=Mong:][:sc=Tfng:][:sc=Yiii:]]"),
+      status);
+  allowed_set.addAll(aspirational_scripts);
+
+  // Add 'Black Heart Suit' and 'Circled White Star'.
+  // TODO(jshin): How about other heart-like characters and Emoji (e.g.
+  // U+1F600) ?
+  allowed_set.add(0x2665u);
+  allowed_set.add(0x272au);
+
+  // Remove the following three characters listed in Mozilla's blacklist (
+  // http://kb.mozillazine.org/Network.IDN.blacklist_chars ) but
+  // not yet excluded from |allowed_set| up to this point:
+  // Combining Long Solidus Overlay, Hebrew Punctuation Gershayim, and
+  // Hyphenation Point
+  allowed_set.remove(0x338u);   // Combining Long Solidus Overlay
+  allowed_set.remove(0x5f4u);   // Hebrew Punctuation Gershayim
+  allowed_set.remove(0x2027u);  // Hyphenation Point
+
+  // TODO(jshin): Decide what to do with '+' and U+0020. For now, leave
+  // them out as Mozilla does.
+  uspoof_setAllowedUnicodeSet(checker_, &allowed_set, &status);
+
+  int32_t checks = uspoof_getChecks(checker_, &status);
+  // Do not allow mixed numbering systems (e.g. ASCII digits and
+  // Devanagari digits) or invisible characters or multiple occurrences
+  // there is a script mixing.
+  checks |= USPOOF_MIXED_NUMBERS | USPOOF_AUX_INFO;
+
+  // USPOOF_INVISBLE should be on by this point without being
+  // explicitly turned on.
+  DCHECK(checks & USPOOF_INVISIBLE);
+
+  // Disable whole-script-confusable check because even 'pax' (Latin)
+  // and b<u-umlaut>cher cannot pass the test because Cyrillic/Greek have
+  // confusable characters for all letters in them.
+  // TODO(jshin): Disabling this check has a downside. One way to alleviate
+  // is to check against a list of well known good domain names.
+  checks ^= USPOOF_WHOLE_SCRIPT_CONFUSABLE;
+
+  uspoof_setChecks(checker_, checks, &status);
+  DCHECK(U_SUCCESS(status));
 }
 
-static base::LazyInstance<base::Lock>::Leaky
-    g_lang_set_lock = LAZY_INSTANCE_INITIALIZER;
-
-// Returns true if all the characters in component_characters are used by
-// the language |lang|.
-bool IsComponentCoveredByLang(const icu::UnicodeSet& component_characters,
-                              const std::string& lang) {
-  CR_DEFINE_STATIC_LOCAL(
-      const icu::UnicodeSet, kASCIILetters, ('a', 'z'));
-  icu::UnicodeSet* lang_set = nullptr;
-  // We're called from both the UI thread and the history thread.
-  {
-    base::AutoLock lock(g_lang_set_lock.Get());
-    if (!GetExemplarSetForLang(lang, &lang_set)) {
-      UErrorCode status = U_ZERO_ERROR;
-      ULocaleData* uld = ulocdata_open(lang.c_str(), &status);
-      // TODO(jungshik) Turn this check on when the ICU data file is
-      // rebuilt with the minimal subset of locale data for languages
-      // to which Chrome is not localized but which we offer in the list
-      // of languages selectable for Accept-Languages. With the rebuilt ICU
-      // data, ulocdata_open never should fall back to the default locale.
-      // (issue 2078)
-      // DCHECK(U_SUCCESS(status) && status != U_USING_DEFAULT_WARNING);
-      if (U_SUCCESS(status) && status != U_USING_DEFAULT_WARNING) {
-        lang_set = reinterpret_cast<icu::UnicodeSet*>(ulocdata_getExemplarSet(
-            uld, nullptr, 0, ULOCDATA_ES_STANDARD, &status));
-        // On success, if |lang| is compatible with ASCII Latin letters, add
-        // them.
-        if (lang_set && IsCompatibleWithASCIILetters(lang))
-          lang_set->addAll(kASCIILetters);
-      }
+inline bool IDNSpoofChecker::check(const base::char16* label, int label_len) {
+  UErrorCode status = U_ZERO_ERROR;
+  int32_t results = uspoof_check(checker_, label, label_len, NULL, &status);
+  DCHECK(U_SUCCESS(status));

[Ryan Sleevi, 2015/08/28 00:35:15]

Shouldn't you actually handle if this can fail?

[jungshik at Google, 2015/09/01 19:47:07]

I agree. Done.

+  if (results & USPOOF_ALL_CHECKS)
+    return false;
 
-      if (!lang_set)
-        lang_set = new icu::UnicodeSet(1, 0);
+  // If there's no script mixing, the input passes without any extra check.
+  if (results == USPOOF_ASCII || results == USPOOF_SINGLE_SCRIPT_RESTRICTIVE)
+    return true;
 
-      lang_set->freeze();
-      SetExemplarSetForLang(lang, lang_set);
-      ulocdata_close(uld);
-    }
-  }
-  return !lang_set->isEmpty() && lang_set->containsAll(component_characters);
+  return g_idn_spoof_checker_extra.Get().check(label, label_len);
 }
 
-// Returns true if the given Unicode host component is safe to display to the
-// user.
-bool IsIDNComponentSafe(const base::char16* str,
-                        int str_len,
-                        const std::string& languages) {
-  // Most common cases (non-IDN) do not reach here so that we don't
-  // need a fast return path.
-  // TODO(jungshik) : Check if there's any character inappropriate
-  // (although allowed) for domain names.
-  // See http://www.unicode.org/reports/tr39/#IDN_Security_Profiles and
-  // http://www.unicode.org/reports/tr39/data/xidmodifications.txt
-  // For now, we borrow the list from Mozilla and tweaked it slightly.
-  // (e.g. Characters like U+00A0, U+3000, U+3002 are omitted because
-  //  they're gonna be canonicalized to U+0020 and full stop before
-  //  reaching here.)
-  // The original list is available at
-  // http://kb.mozillazine.org/Network.IDN.blacklist_chars and
-  // at http://mxr.mozilla.org/seamonkey/source/modules/libpref/src/init/all.js#703
-
+IDNSpoofCheckerExtra::IDNSpoofCheckerExtra() {
   UErrorCode status = U_ZERO_ERROR;
-#ifdef U_WCHAR_IS_UTF16
-  icu::UnicodeSet dangerous_characters(
-      icu::UnicodeString(
-          L"[[\\ \u00ad\u00bc\u00bd\u01c3\u0337\u0338"
-          L"\u05c3\u05f4\u06d4\u0702\u115f\u1160][\u2000-\u200b]"
-          L"[\u2024\u2027\u2028\u2029\u2039\u203a\u2044\u205f]"
-          L"[\u2154-\u2156][\u2159-\u215b][\u215f\u2215\u23ae"
-          L"\u29f6\u29f8\u2afb\u2afd][\u2ff0-\u2ffb][\u3014"
-          L"\u3015\u3033\u3164\u321d\u321e\u33ae\u33af\u33c6\u33df\ufe14"
-          L"\ufe15\ufe3f\ufe5d\ufe5e\ufeff\uff0e\uff06\uff61\uffa0\ufff9]"
-          L"[\ufffa-\ufffd]\U0001f50f\U0001f510\U0001f512\U0001f513]"),
-      status);
-  DCHECK(U_SUCCESS(status));
-  icu::RegexMatcher dangerous_patterns(icu::UnicodeString(
-      // Lone katakana no, so, or n
-      L"[^\\p{Katakana}][\u30ce\u30f3\u30bd][^\\p{Katakana}]"
-      // Repeating Japanese accent characters
-      L"|[\u3099\u309a\u309b\u309c][\u3099\u309a\u309b\u309c]"),
-      0, status);
-#else
-  icu::UnicodeSet dangerous_characters(icu::UnicodeString(
-      "[[\\u0020\\u00ad\\u00bc\\u00bd\\u01c3\\u0337\\u0338"
-      "\\u05c3\\u05f4\\u06d4\\u0702\\u115f\\u1160][\\u2000-\\u200b]"
-      "[\\u2024\\u2027\\u2028\\u2029\\u2039\\u203a\\u2044\\u205f]"
-      "[\\u2154-\\u2156][\\u2159-\\u215b][\\u215f\\u2215\\u23ae"
-      "\\u29f6\\u29f8\\u2afb\\u2afd][\\u2ff0-\\u2ffb][\\u3014"
-      "\\u3015\\u3033\\u3164\\u321d\\u321e\\u33ae\\u33af\\u33c6\\u33df\\ufe14"
-      "\\ufe15\\ufe3f\\ufe5d\\ufe5e\\ufeff\\uff0e\\uff06\\uff61\\uffa0\\ufff9]"
-      "[\\ufffa-\\ufffd]\\U0001f50f\\U0001f510\\U0001f512\\U0001f513]", -1,
-      US_INV), status);
-  DCHECK(U_SUCCESS(status));
-  icu::RegexMatcher dangerous_patterns(icu::UnicodeString(
-      // Lone katakana no, so, or n
-      "[^\\p{Katakana}][\\u30ce\\u30f3\\u30bd][^\\p{Katakana}]"
-      // Repeating Japanese accent characters
-      "|[\\u3099\\u309a\\u309b\\u309c][\\u3099\\u309a\\u309b\\u309c]"),
+  non_ascii_latin_ = icu::UnicodeSet(
+      UNICODE_STRING_SIMPLE("[[:sc=Latn:] - [a-zA-Z]]"), status);
+
+  dangerous_pattern_ = icu::RegexPattern::compile(
+      UNICODE_STRING_SIMPLE(
+          // Lone (out-of-context) katakana no, so, zo, or n
+          // They can be mistaken for a slash.
+          "[^\\p{Katakana}][\\u30ce\\u30f3\\u30bd\\u30be][^\\p{Katakana}]"
+          // Repeating Japanese accent characters. USPOOF_INVISIBLE
+          // only checks for a repeated occurence of the same combining
+          // mark, but we block a sequence of similary looking
+          // Japanese combining marks as well.
+          "|[\\u3099-\\u309c][\\u3099-\\u309c]"),
       0, status);
-#endif
   DCHECK(U_SUCCESS(status));
-  icu::UnicodeSet component_characters;
-  icu::UnicodeString component_string(str, str_len);
-  component_characters.addAll(component_string);
-  if (dangerous_characters.containsSome(component_characters))
-    return false;
+}
 
-  DCHECK(U_SUCCESS(status));
-  dangerous_patterns.reset(component_string);
-  if (dangerous_patterns.find())
+inline bool IDNSpoofCheckerExtra::check(const base::char16* label,
+                                        int label_len) {
+  // This is called only if script mixing is detected.
+  // Limit Latin letters that can be mixed with other scripts to
+  // ASCII-Latin instead  of any Latin.
+  icu::UnicodeString label_string(FALSE, label, label_len);
+  if (non_ascii_latin_.containsSome(label_string))
     return false;
 
-  // If the language list is empty, the result is completely determined
-  // by whether a component is a single script or not. This will block
-  // even "safe" script mixing cases like <Chinese, Latin-ASCII> that are
-  // allowed with |languages| (while it blocks Chinese + Latin letters with
-  // an accent as should be the case), but we want to err on the safe side
-  // when |languages| is empty.
-  if (languages.empty())
-    return IsIDNComponentInSingleScript(str, str_len);
-
-  // |common_characters| is made up of  ASCII numbers, hyphen, plus and
-  // underscore that are used across scripts and allowed in domain names.
-  // (sync'd with characters allowed in url_canon_host with square
-  // brackets excluded.) See kHostCharLookup[] array in url_canon_host.cc.
-  icu::UnicodeSet common_characters(UNICODE_STRING_SIMPLE("[[0-9]\\-_+\\ ]"),
-                                    status);
+  UErrorCode status = U_ZERO_ERROR;
+  scoped_ptr<icu::RegexMatcher> dangerous_pattern_matcher(
+      dangerous_pattern_->matcher(label_string, status));
   DCHECK(U_SUCCESS(status));
-  // Subtract common characters because they're always allowed so that
-  // we just have to check if a language-specific set contains
-  // the remainder.
-  component_characters.removeAll(common_characters);
-
-  base::StringTokenizer t(languages, ",");
-  while (t.GetNext()) {
-    if (IsComponentCoveredByLang(component_characters, t.token()))
-      return true;
-  }
-  return false;
+  return !dangerous_pattern_matcher->find();
+
+  // TODO(jshin): Check spoofing attempt against a list of 'good' domains
+}
+
+// Returns true if the given Unicode host component is safe to display to the
+// user.
+bool IsIDNComponentSafe(const base::char16* label, int label_len) {
+  return g_idn_spoof_checker.Get().check(label, label_len);
 }
 
 // A wrapper to use LazyInstance<>::Leaky with ICU's UIDNA, a C pointer to
@@ -305,16 +221,14 @@ struct UIDNAWrapper {
   UIDNA* value;
 };
 
-static base::LazyInstance<UIDNAWrapper>::Leaky
-    g_uidna = LAZY_INSTANCE_INITIALIZER;
+base::LazyInstance<UIDNAWrapper>::Leaky g_uidna = LAZY_INSTANCE_INITIALIZER;
 
-// Converts one component of a host (between dots) to IDN if safe. The result
-// will be APPENDED to the given output string and will be the same as the input
-// if it is not IDN or the IDN is unsafe to display.  Returns whether any
-// conversion was performed.
+// Converts one component (label) of a host (between dots) to Unicode if safe.
+// The result will be APPENDED to the given output string and will be the
+// same as the input if it is not Punycode or the IDN is unsafe to display.
+// Returns whether any conversion was performed.
 bool IDNToUnicodeOneComponent(const base::char16* comp,
                               size_t comp_len,
-                              const std::string& languages,
                               base::string16* out) {
   DCHECK(out);
   if (comp_len == 0)
@@ -345,8 +259,7 @@ bool IDNToUnicodeOneComponent(const base::char16* comp,
       // Converted successfully. Ensure that the converted component
       // can be safely displayed to the user.
       out->resize(original_length + output_length);
-      if (IsIDNComponentSafe(out->data() + original_length, output_length,
-                             languages))
+      if (IsIDNComponentSafe(out->data() + original_length, output_length))
         return true;
     }
 
@@ -360,16 +273,10 @@ bool IDNToUnicodeOneComponent(const base::char16* comp,
   return false;
 }
 
-// TODO(brettw) bug 734373: check the scripts for each host component and
-// don't un-IDN-ize if there is more than one. Alternatively, only IDN for
-// scripts that the user has installed. For now, just put the entire
-// path through IDN. Maybe this feature can be implemented in ICU itself?
-//
-// We may want to skip this step in the case of file URLs to allow unicode
-// UNC hostnames regardless of encodings.
+// TODO(brettw) We may want to skip this step in the case of file URLs to
+// allow unicode UNC hostnames regardless of encodings.
 base::string16 IDNToUnicodeWithAdjustments(
     const std::string& host,
-    const std::string& languages,
     base::OffsetAdjuster::Adjustments* adjustments) {
   if (adjustments)
     adjustments->clear();
@@ -395,8 +302,7 @@ base::string16 IDNToUnicodeWithAdjustments(
       if (component_end > component_start) {
         // Add the substring that we just found.
         converted_idn = IDNToUnicodeOneComponent(
-            input16.data() + component_start, component_length, languages,
-            &out16);
+            input16.data() + component_start, component_length, &out16);
       }
       size_t new_component_length = out16.length() - new_component_start;
 
@@ -436,7 +342,6 @@ void AdjustAllComponentsButScheme(int delta, url::Parsed* parsed) {
 // Helper for FormatUrlWithOffsets().
 base::string16 FormatViewSourceUrl(
     const GURL& url,
-    const std::string& languages,
     FormatUrlTypes format_types,
     UnescapeRule::Type unescape_rules,
     url::Parsed* new_parsed,
@@ -449,9 +354,10 @@ base::string16 FormatViewSourceUrl(
   // Format the underlying URL and record adjustments.
   const std::string& url_str(url.possibly_invalid_spec());
   adjustments->clear();
-  base::string16 result(base::ASCIIToUTF16(kViewSource) +
+  base::string16 result(
+      base::ASCIIToUTF16(kViewSource) +
       FormatUrlWithAdjustments(GURL(url_str.substr(kViewSourceLength)),
-                               languages, format_types, unescape_rules,
+                               std::string(), format_types, unescape_rules,
                                new_parsed, prefix_end, adjustments));
   // Revise |adjustments| by shifting to the offsets to prefix that the above
   // call to FormatUrl didn't get to see.
@@ -491,19 +397,14 @@ class AppendComponentTransform {
 
 class HostComponentTransform : public AppendComponentTransform {
  public:
-  explicit HostComponentTransform(const std::string& languages)
-      : languages_(languages) {
-  }
+  explicit HostComponentTransform() {}
 
  private:
   base::string16 Execute(
       const std::string& component_text,
       base::OffsetAdjuster::Adjustments* adjustments) const override {
-    return IDNToUnicodeWithAdjustments(component_text, languages_,
-                                       adjustments);
+    return IDNToUnicodeWithAdjustments(component_text, adjustments);
   }
-
-  const std::string& languages_;
 };
 
 class NonHostComponentTransform : public AppendComponentTransform {
@@ -573,7 +474,7 @@ void AppendFormattedComponent(const std::string& spec,
   }
 }
 
-}  // namespace
+}  // anonymous namespace
 
 const FormatUrlType kFormatUrlOmitNothing                     = 0;
 const FormatUrlType kFormatUrlOmitUsernamePassword            = 1 << 0;
@@ -584,7 +485,7 @@ const FormatUrlType kFormatUrlOmitAll = kFormatUrlOmitUsernamePassword |
 
 base::string16 IDNToUnicode(const std::string& host,
                             const std::string& languages) {
-  return IDNToUnicodeWithAdjustments(host, languages, NULL);
+  return IDNToUnicodeWithAdjustments(host, NULL);
 }
 
 std::string GetDirectoryListingEntry(const base::string16& name,
@@ -631,8 +532,8 @@ void AppendFormattedHost(const GURL& url,
                          const std::string& languages,
                          base::string16* output) {
   AppendFormattedComponent(url.possibly_invalid_spec(),
-      url.parsed_for_possibly_invalid_spec().host,
-      HostComponentTransform(languages), output, NULL, NULL);
+                           url.parsed_for_possibly_invalid_spec().host,
+                           HostComponentTransform(), output, NULL, NULL);
 }
 
 base::string16 FormatUrlWithOffsets(
@@ -645,7 +546,7 @@ base::string16 FormatUrlWithOffsets(
     std::vector<size_t>* offsets_for_adjustment) {
   base::OffsetAdjuster::Adjustments adjustments;
   const base::string16& format_url_return_value =
-      FormatUrlWithAdjustments(url, languages, format_types, unescape_rules,
+      FormatUrlWithAdjustments(url, std::string(), format_types, unescape_rules,
                                new_parsed, prefix_end, &adjustments);
   base::OffsetAdjuster::AdjustOffsets(adjustments, offsets_for_adjustment);
   if (offsets_for_adjustment) {
@@ -681,9 +582,8 @@ base::string16 FormatUrlWithAdjustments(
   if (url.SchemeIs(kViewSource) &&
       !base::StartsWith(url.possibly_invalid_spec(), kViewSourceTwice,
                         base::CompareCase::INSENSITIVE_ASCII)) {
-    return FormatViewSourceUrl(url, languages, format_types,
-                               unescape_rules, new_parsed, prefix_end,
-                               adjustments);
+    return FormatViewSourceUrl(url, format_types, unescape_rules, new_parsed,
+                               prefix_end, adjustments);
   }
 
   // We handle both valid and invalid URLs (this will give us the spec
@@ -753,7 +653,7 @@ base::string16 FormatUrlWithAdjustments(
     *prefix_end = static_cast<size_t>(url_string.length());
 
   // Host.
-  AppendFormattedComponent(spec, parsed.host, HostComponentTransform(languages),
+  AppendFormattedComponent(spec, parsed.host, HostComponentTransform(),
                            &url_string, &new_parsed->host, adjustments);
 
   // Port.
@@ -829,8 +729,9 @@ base::string16 FormatUrl(const GURL& url,
   Offsets offsets;
   if (offset_for_adjustment)
     offsets.push_back(*offset_for_adjustment);
-  base::string16 result = FormatUrlWithOffsets(url, languages, format_types,
-      unescape_rules, new_parsed, prefix_end, &offsets);
+  base::string16 result =
+      FormatUrlWithOffsets(url, std::string(), format_types, unescape_rules,
+                           new_parsed, prefix_end, &offsets);
   if (offset_for_adjustment)
     *offset_for_adjustment = offsets[0];
   return result;