Revert of Move net::FormatUrl and friends outside of //net and into //components

net/base/net_util_icu.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "net/base/net_util.h"
 
+#include <map>
+#include <vector>
+
 #include "base/i18n/time_formatting.h"
 #include "base/json/string_escape.h"
+#include "base/lazy_instance.h"
+#include "base/logging.h"
+#include "base/memory/singleton.h"
+#include "base/stl_util.h"
+#include "base/strings/string_tokenizer.h"
 #include "base/strings/string_util.h"
+#include "base/strings/utf_offset_string_conversions.h"
 #include "base/strings/utf_string_conversions.h"
-#include "net/base/escape.h"
+#include "base/time/time.h"
+#include "url/gurl.h"
+#include "third_party/icu/source/common/unicode/uidna.h"
+#include "third_party/icu/source/common/unicode/uniset.h"
+#include "third_party/icu/source/common/unicode/uscript.h"
+#include "third_party/icu/source/common/unicode/uset.h"
+#include "third_party/icu/source/i18n/unicode/datefmt.h"
+#include "third_party/icu/source/i18n/unicode/regex.h"
+#include "third_party/icu/source/i18n/unicode/ulocdata.h"
+
+using base::Time;
 
 namespace net {
 
+namespace {
+
+typedef std::vector<size_t> Offsets;
+
+// Does some simple normalization of scripts so we can allow certain scripts
+// to exist together.
+// TODO(brettw) bug 880223: we should allow some other languages to be
+// oombined such as Chinese and Latin. We will probably need a more
+// complicated system of language pairs to have more fine-grained control.
+UScriptCode NormalizeScript(UScriptCode code) {
+  switch (code) {
+    case USCRIPT_KATAKANA:
+    case USCRIPT_HIRAGANA:
+    case USCRIPT_KATAKANA_OR_HIRAGANA:
+    case USCRIPT_HANGUL:  // This one is arguable.
+      return USCRIPT_HAN;
+    default:
+      return code;
+  }
+}
+
+bool IsIDNComponentInSingleScript(const base::char16* str, int str_len) {
+  UScriptCode first_script = USCRIPT_INVALID_CODE;
+  bool is_first = true;
+
+  int i = 0;
+  while (i < str_len) {
+    unsigned code_point;
+    U16_NEXT(str, i, str_len, code_point);
+
+    UErrorCode err = U_ZERO_ERROR;
+    UScriptCode cur_script = uscript_getScript(code_point, &err);
+    if (err != U_ZERO_ERROR)
+      return false;  // Report mixed on error.
+    cur_script = NormalizeScript(cur_script);
+
+    // TODO(brettw) We may have to check for USCRIPT_INHERENT as well.
+    if (is_first && cur_script != USCRIPT_COMMON) {
+      first_script = cur_script;
+      is_first = false;
+    } else {
+      if (cur_script != USCRIPT_COMMON && cur_script != first_script)
+        return false;
+    }
+  }
+  return true;
+}
+
+// Check if the script of a language can be 'safely' mixed with
+// Latin letters in the ASCII range.
+bool IsCompatibleWithASCIILetters(const std::string& lang) {
+  // For now, just list Chinese, Japanese and Korean (positive list).
+  // An alternative is negative-listing (languages using Greek and
+  // Cyrillic letters), but it can be more dangerous.
+  return !lang.substr(0, 2).compare("zh") ||
+         !lang.substr(0, 2).compare("ja") ||
+         !lang.substr(0, 2).compare("ko");
+}
+
+typedef std::map<std::string, icu::UnicodeSet*> LangToExemplarSetMap;
+
+class LangToExemplarSet {
+ public:
+  static LangToExemplarSet* GetInstance() {
+    return Singleton<LangToExemplarSet>::get();
+  }
+
+ private:
+  LangToExemplarSetMap map;
+  LangToExemplarSet() { }
+  ~LangToExemplarSet() {
+    STLDeleteContainerPairSecondPointers(map.begin(), map.end());
+  }
+
+  friend class Singleton<LangToExemplarSet>;
+  friend struct DefaultSingletonTraits<LangToExemplarSet>;
+  friend bool GetExemplarSetForLang(const std::string&, icu::UnicodeSet**);
+  friend void SetExemplarSetForLang(const std::string&, icu::UnicodeSet*);
+
+  DISALLOW_COPY_AND_ASSIGN(LangToExemplarSet);
+};
+
+bool GetExemplarSetForLang(const std::string& lang,
+                           icu::UnicodeSet** lang_set) {
+  const LangToExemplarSetMap& map = LangToExemplarSet::GetInstance()->map;
+  LangToExemplarSetMap::const_iterator pos = map.find(lang);
+  if (pos != map.end()) {
+    *lang_set = pos->second;
+    return true;
+  }
+  return false;
+}
+
+void SetExemplarSetForLang(const std::string& lang,
+                           icu::UnicodeSet* lang_set) {
+  LangToExemplarSetMap& map = LangToExemplarSet::GetInstance()->map;
+  map.insert(std::make_pair(lang, lang_set));
+}
+
+static base::LazyInstance<base::Lock>::Leaky
+    g_lang_set_lock = LAZY_INSTANCE_INITIALIZER;
+
+// Returns true if all the characters in component_characters are used by
+// the language |lang|.
+bool IsComponentCoveredByLang(const icu::UnicodeSet& component_characters,
+                              const std::string& lang) {
+  CR_DEFINE_STATIC_LOCAL(
+      const icu::UnicodeSet, kASCIILetters, ('a', 'z'));
+  icu::UnicodeSet* lang_set = nullptr;
+  // We're called from both the UI thread and the history thread.
+  {
+    base::AutoLock lock(g_lang_set_lock.Get());
+    if (!GetExemplarSetForLang(lang, &lang_set)) {
+      UErrorCode status = U_ZERO_ERROR;
+      ULocaleData* uld = ulocdata_open(lang.c_str(), &status);
+      // TODO(jungshik) Turn this check on when the ICU data file is
+      // rebuilt with the minimal subset of locale data for languages
+      // to which Chrome is not localized but which we offer in the list
+      // of languages selectable for Accept-Languages. With the rebuilt ICU
+      // data, ulocdata_open never should fall back to the default locale.
+      // (issue 2078)
+      // DCHECK(U_SUCCESS(status) && status != U_USING_DEFAULT_WARNING);
+      if (U_SUCCESS(status) && status != U_USING_DEFAULT_WARNING) {
+        lang_set = reinterpret_cast<icu::UnicodeSet*>(ulocdata_getExemplarSet(
+            uld, nullptr, 0, ULOCDATA_ES_STANDARD, &status));
+        // On success, if |lang| is compatible with ASCII Latin letters, add
+        // them.
+        if (lang_set && IsCompatibleWithASCIILetters(lang))
+          lang_set->addAll(kASCIILetters);
+      }
+
+      if (!lang_set)
+        lang_set = new icu::UnicodeSet(1, 0);
+
+      lang_set->freeze();
+      SetExemplarSetForLang(lang, lang_set);
+      ulocdata_close(uld);
+    }
+  }
+  return !lang_set->isEmpty() && lang_set->containsAll(component_characters);
+}
+
+// Returns true if the given Unicode host component is safe to display to the
+// user.
+bool IsIDNComponentSafe(const base::char16* str,
+                        int str_len,
+                        const std::string& languages) {
+  // Most common cases (non-IDN) do not reach here so that we don't
+  // need a fast return path.
+  // TODO(jungshik) : Check if there's any character inappropriate
+  // (although allowed) for domain names.
+  // See http://www.unicode.org/reports/tr39/#IDN_Security_Profiles and
+  // http://www.unicode.org/reports/tr39/data/xidmodifications.txt
+  // For now, we borrow the list from Mozilla and tweaked it slightly.
+  // (e.g. Characters like U+00A0, U+3000, U+3002 are omitted because
+  //  they're gonna be canonicalized to U+0020 and full stop before
+  //  reaching here.)
+  // The original list is available at
+  // http://kb.mozillazine.org/Network.IDN.blacklist_chars and
+  // at http://mxr.mozilla.org/seamonkey/source/modules/libpref/src/init/all.js#703
+
+  UErrorCode status = U_ZERO_ERROR;
+#ifdef U_WCHAR_IS_UTF16
+  icu::UnicodeSet dangerous_characters(
+      icu::UnicodeString(
+          L"[[\\ \u00ad\u00bc\u00bd\u01c3\u0337\u0338"
+          L"\u05c3\u05f4\u06d4\u0702\u115f\u1160][\u2000-\u200b]"
+          L"[\u2024\u2027\u2028\u2029\u2039\u203a\u2044\u205f]"
+          L"[\u2154-\u2156][\u2159-\u215b][\u215f\u2215\u23ae"
+          L"\u29f6\u29f8\u2afb\u2afd][\u2ff0-\u2ffb][\u3014"
+          L"\u3015\u3033\u3164\u321d\u321e\u33ae\u33af\u33c6\u33df\ufe14"
+          L"\ufe15\ufe3f\ufe5d\ufe5e\ufeff\uff0e\uff06\uff61\uffa0\ufff9]"
+          L"[\ufffa-\ufffd]\U0001f50f\U0001f510\U0001f512\U0001f513]"),
+      status);
+  DCHECK(U_SUCCESS(status));
+  icu::RegexMatcher dangerous_patterns(icu::UnicodeString(
+      // Lone katakana no, so, or n
+      L"[^\\p{Katakana}][\u30ce\u30f3\u30bd][^\\p{Katakana}]"
+      // Repeating Japanese accent characters
+      L"|[\u3099\u309a\u309b\u309c][\u3099\u309a\u309b\u309c]"),
+      0, status);
+#else
+  icu::UnicodeSet dangerous_characters(icu::UnicodeString(
+      "[[\\u0020\\u00ad\\u00bc\\u00bd\\u01c3\\u0337\\u0338"
+      "\\u05c3\\u05f4\\u06d4\\u0702\\u115f\\u1160][\\u2000-\\u200b]"
+      "[\\u2024\\u2027\\u2028\\u2029\\u2039\\u203a\\u2044\\u205f]"
+      "[\\u2154-\\u2156][\\u2159-\\u215b][\\u215f\\u2215\\u23ae"
+      "\\u29f6\\u29f8\\u2afb\\u2afd][\\u2ff0-\\u2ffb][\\u3014"
+      "\\u3015\\u3033\\u3164\\u321d\\u321e\\u33ae\\u33af\\u33c6\\u33df\\ufe14"
+      "\\ufe15\\ufe3f\\ufe5d\\ufe5e\\ufeff\\uff0e\\uff06\\uff61\\uffa0\\ufff9]"
+      "[\\ufffa-\\ufffd]\\U0001f50f\\U0001f510\\U0001f512\\U0001f513]", -1,
+      US_INV), status);
+  DCHECK(U_SUCCESS(status));
+  icu::RegexMatcher dangerous_patterns(icu::UnicodeString(
+      // Lone katakana no, so, or n
+      "[^\\p{Katakana}][\\u30ce\\u30f3\\u30bd][^\\p{Katakana}]"
+      // Repeating Japanese accent characters
+      "|[\\u3099\\u309a\\u309b\\u309c][\\u3099\\u309a\\u309b\\u309c]"),
+      0, status);
+#endif
+  DCHECK(U_SUCCESS(status));
+  icu::UnicodeSet component_characters;
+  icu::UnicodeString component_string(str, str_len);
+  component_characters.addAll(component_string);
+  if (dangerous_characters.containsSome(component_characters))
+    return false;
+
+  DCHECK(U_SUCCESS(status));
+  dangerous_patterns.reset(component_string);
+  if (dangerous_patterns.find())
+    return false;
+
+  // If the language list is empty, the result is completely determined
+  // by whether a component is a single script or not. This will block
+  // even "safe" script mixing cases like <Chinese, Latin-ASCII> that are
+  // allowed with |languages| (while it blocks Chinese + Latin letters with
+  // an accent as should be the case), but we want to err on the safe side
+  // when |languages| is empty.
+  if (languages.empty())
+    return IsIDNComponentInSingleScript(str, str_len);
+
+  // |common_characters| is made up of  ASCII numbers, hyphen, plus and
+  // underscore that are used across scripts and allowed in domain names.
+  // (sync'd with characters allowed in url_canon_host with square
+  // brackets excluded.) See kHostCharLookup[] array in url_canon_host.cc.
+  icu::UnicodeSet common_characters(UNICODE_STRING_SIMPLE("[[0-9]\\-_+\\ ]"),
+                                    status);
+  DCHECK(U_SUCCESS(status));
+  // Subtract common characters because they're always allowed so that
+  // we just have to check if a language-specific set contains
+  // the remainder.
+  component_characters.removeAll(common_characters);
+
+  base::StringTokenizer t(languages, ",");
+  while (t.GetNext()) {
+    if (IsComponentCoveredByLang(component_characters, t.token()))
+      return true;
+  }
+  return false;
+}
+
+// A wrapper to use LazyInstance<>::Leaky with ICU's UIDNA, a C pointer to
+// a UTS46/IDNA 2008 handling object opened with uidna_openUTS46().
+//
+// We use UTS46 with BiDiCheck to migrate from IDNA 2003 to IDNA 2008 with
+// the backward compatibility in mind. What it does:
+//
+// 1. Use the up-to-date Unicode data.
+// 2. Define a case folding/mapping with the up-to-date Unicode data as
+//    in IDNA 2003.
+// 3. Use transitional mechanism for 4 deviation characters (sharp-s,
+//    final sigma, ZWJ and ZWNJ) for now.
+// 4. Continue to allow symbols and punctuations.
+// 5. Apply new BiDi check rules more permissive than the IDNA 2003 BiDI rules.
+// 6. Do not apply STD3 rules
+// 7. Do not allow unassigned code points.
+//
+// It also closely matches what IE 10 does except for the BiDi check (
+// http://goo.gl/3XBhqw ).
+// See http://http://unicode.org/reports/tr46/ and references therein
+// for more details.
+struct UIDNAWrapper {
+  UIDNAWrapper() {
+    UErrorCode err = U_ZERO_ERROR;
+    // TODO(jungshik): Change options as different parties (browsers,
+    // registrars, search engines) converge toward a consensus.
+    value = uidna_openUTS46(UIDNA_CHECK_BIDI, &err);
+    if (U_FAILURE(err))
+      value = NULL;
+  }
+
+  UIDNA* value;
+};
+
+static base::LazyInstance<UIDNAWrapper>::Leaky
+    g_uidna = LAZY_INSTANCE_INITIALIZER;
+
+// Converts one component of a host (between dots) to IDN if safe. The result
+// will be APPENDED to the given output string and will be the same as the input
+// if it is not IDN or the IDN is unsafe to display.  Returns whether any
+// conversion was performed.
+bool IDNToUnicodeOneComponent(const base::char16* comp,
+                              size_t comp_len,
+                              const std::string& languages,
+                              base::string16* out) {
+  DCHECK(out);
+  if (comp_len == 0)
+    return false;
+
+  // Only transform if the input can be an IDN component.
+  static const base::char16 kIdnPrefix[] = {'x', 'n', '-', '-'};
+  if ((comp_len > arraysize(kIdnPrefix)) &&
+      !memcmp(comp, kIdnPrefix, arraysize(kIdnPrefix) * sizeof(base::char16))) {
+    UIDNA* uidna = g_uidna.Get().value;
+    DCHECK(uidna != NULL);
+    size_t original_length = out->length();
+    int output_length = 64;
+    UIDNAInfo info = UIDNA_INFO_INITIALIZER;
+    UErrorCode status;
+    do {
+      out->resize(original_length + output_length);
+      status = U_ZERO_ERROR;
+      // This returns the actual length required. If this is more than 64
+      // code units, |status| will be U_BUFFER_OVERFLOW_ERROR and we'll try
+      // the conversion again, but with a sufficiently large buffer.
+      output_length = uidna_labelToUnicode(
+          uidna, comp, static_cast<int32_t>(comp_len), &(*out)[original_length],
+          output_length, &info, &status);
+    } while ((status == U_BUFFER_OVERFLOW_ERROR && info.errors == 0));
+
+    if (U_SUCCESS(status) && info.errors == 0) {
+      // Converted successfully. Ensure that the converted component
+      // can be safely displayed to the user.
+      out->resize(original_length + output_length);
+      if (IsIDNComponentSafe(out->data() + original_length, output_length,
+                             languages))
+        return true;
+    }
+
+    // Something went wrong. Revert to original string.
+    out->resize(original_length);
+  }
+
+  // We get here with no IDN or on error, in which case we just append the
+  // literal input.
+  out->append(comp, comp_len);
+  return false;
+}
+
+// TODO(brettw) bug 734373: check the scripts for each host component and
+// don't un-IDN-ize if there is more than one. Alternatively, only IDN for
+// scripts that the user has installed. For now, just put the entire
+// path through IDN. Maybe this feature can be implemented in ICU itself?
+//
+// We may want to skip this step in the case of file URLs to allow unicode
+// UNC hostnames regardless of encodings.
+base::string16 IDNToUnicodeWithAdjustments(
+    const std::string& host,
+    const std::string& languages,
+    base::OffsetAdjuster::Adjustments* adjustments) {
+  if (adjustments)
+    adjustments->clear();
+  // Convert the ASCII input to a base::string16 for ICU.
+  base::string16 input16;
+  input16.reserve(host.length());
+  input16.insert(input16.end(), host.begin(), host.end());
+
+  // Do each component of the host separately, since we enforce script matching
+  // on a per-component basis.
+  base::string16 out16;
+  {
+    for (size_t component_start = 0, component_end;
+         component_start < input16.length();
+         component_start = component_end + 1) {
+      // Find the end of the component.
+      component_end = input16.find('.', component_start);
+      if (component_end == base::string16::npos)
+        component_end = input16.length();  // For getting the last component.
+      size_t component_length = component_end - component_start;
+      size_t new_component_start = out16.length();
+      bool converted_idn = false;
+      if (component_end > component_start) {
+        // Add the substring that we just found.
+        converted_idn = IDNToUnicodeOneComponent(
+            input16.data() + component_start, component_length, languages,
+            &out16);
+      }
+      size_t new_component_length = out16.length() - new_component_start;
+
+      if (converted_idn && adjustments) {
+        adjustments->push_back(base::OffsetAdjuster::Adjustment(
+            component_start, component_length, new_component_length));
+      }
+
+      // Need to add the dot we just found (if we found one).
+      if (component_end < input16.length())
+        out16.push_back('.');
+    }
+  }
+  return out16;
+}
+
+// If |component| is valid, its begin is incremented by |delta|.
+void AdjustComponent(int delta, url::Component* component) {
+  if (!component->is_valid())
+    return;
+
+  DCHECK(delta >= 0 || component->begin >= -delta);
+  component->begin += delta;
+}
+
+// Adjusts all the components of |parsed| by |delta|, except for the scheme.
+void AdjustAllComponentsButScheme(int delta, url::Parsed* parsed) {
+  AdjustComponent(delta, &(parsed->username));
+  AdjustComponent(delta, &(parsed->password));
+  AdjustComponent(delta, &(parsed->host));
+  AdjustComponent(delta, &(parsed->port));
+  AdjustComponent(delta, &(parsed->path));
+  AdjustComponent(delta, &(parsed->query));
+  AdjustComponent(delta, &(parsed->ref));
+}
+
+// Helper for FormatUrlWithOffsets().
+base::string16 FormatViewSourceUrl(
+    const GURL& url,
+    const std::string& languages,
+    FormatUrlTypes format_types,
+    UnescapeRule::Type unescape_rules,
+    url::Parsed* new_parsed,
+    size_t* prefix_end,
+    base::OffsetAdjuster::Adjustments* adjustments) {
+  DCHECK(new_parsed);
+  const char kViewSource[] = "view-source:";
+  const size_t kViewSourceLength = arraysize(kViewSource) - 1;
+
+  // Format the underlying URL and record adjustments.
+  const std::string& url_str(url.possibly_invalid_spec());
+  adjustments->clear();
+  base::string16 result(base::ASCIIToUTF16(kViewSource) +
+      FormatUrlWithAdjustments(GURL(url_str.substr(kViewSourceLength)),
+                               languages, format_types, unescape_rules,
+                               new_parsed, prefix_end, adjustments));
+  // Revise |adjustments| by shifting to the offsets to prefix that the above
+  // call to FormatUrl didn't get to see.
+  for (base::OffsetAdjuster::Adjustments::iterator it = adjustments->begin();
+       it != adjustments->end(); ++it)
+    it->original_offset += kViewSourceLength;
+
+  // Adjust positions of the parsed components.
+  if (new_parsed->scheme.is_nonempty()) {
+    // Assume "view-source:real-scheme" as a scheme.
+    new_parsed->scheme.len += kViewSourceLength;
+  } else {
+    new_parsed->scheme.begin = 0;
+    new_parsed->scheme.len = kViewSourceLength - 1;
+  }
+  AdjustAllComponentsButScheme(kViewSourceLength, new_parsed);
+
+  if (prefix_end)
+    *prefix_end += kViewSourceLength;
+
+  return result;
+}
+
+class AppendComponentTransform {
+ public:
+  AppendComponentTransform() {}
+  virtual ~AppendComponentTransform() {}
+
+  virtual base::string16 Execute(
+      const std::string& component_text,
+      base::OffsetAdjuster::Adjustments* adjustments) const = 0;
+
+  // NOTE: No DISALLOW_COPY_AND_ASSIGN here, since gcc < 4.3.0 requires an
+  // accessible copy constructor in order to call AppendFormattedComponent()
+  // with an inline temporary (see http://gcc.gnu.org/bugs/#cxx%5Frvalbind ).
+};
+
+class HostComponentTransform : public AppendComponentTransform {
+ public:
+  explicit HostComponentTransform(const std::string& languages)
+      : languages_(languages) {
+  }
+
+ private:
+  base::string16 Execute(
+      const std::string& component_text,
+      base::OffsetAdjuster::Adjustments* adjustments) const override {
+    return IDNToUnicodeWithAdjustments(component_text, languages_,
+                                       adjustments);
+  }
+
+  const std::string& languages_;
+};
+
+class NonHostComponentTransform : public AppendComponentTransform {
+ public:
+  explicit NonHostComponentTransform(UnescapeRule::Type unescape_rules)
+      : unescape_rules_(unescape_rules) {
+  }
+
+ private:
+  base::string16 Execute(
+      const std::string& component_text,
+      base::OffsetAdjuster::Adjustments* adjustments) const override {
+    return (unescape_rules_ == UnescapeRule::NONE) ?
+        base::UTF8ToUTF16WithAdjustments(component_text, adjustments) :
+        UnescapeAndDecodeUTF8URLComponentWithAdjustments(component_text,
+            unescape_rules_, adjustments);
+  }
+
+  const UnescapeRule::Type unescape_rules_;
+};
+
+// Transforms the portion of |spec| covered by |original_component| according to
+// |transform|.  Appends the result to |output|.  If |output_component| is
+// non-NULL, its start and length are set to the transformed component's new
+// start and length.  If |adjustments| is non-NULL, appends adjustments (if
+// any) that reflect the transformation the original component underwent to
+// become the transformed value appended to |output|.
+void AppendFormattedComponent(const std::string& spec,
+                              const url::Component& original_component,
+                              const AppendComponentTransform& transform,
+                              base::string16* output,
+                              url::Component* output_component,
+                              base::OffsetAdjuster::Adjustments* adjustments) {
+  DCHECK(output);
+  if (original_component.is_nonempty()) {
+    size_t original_component_begin =
+        static_cast<size_t>(original_component.begin);
+    size_t output_component_begin = output->length();
+    std::string component_str(spec, original_component_begin,
+                              static_cast<size_t>(original_component.len));
+
+    // Transform |component_str| and modify |adjustments| appropriately.
+    base::OffsetAdjuster::Adjustments component_transform_adjustments;
+    output->append(
+        transform.Execute(component_str, &component_transform_adjustments));
+
+    // Shift all the adjustments made for this component so the offsets are
+    // valid for the original string and add them to |adjustments|.
+    for (base::OffsetAdjuster::Adjustments::iterator comp_iter =
+         component_transform_adjustments.begin();
+         comp_iter != component_transform_adjustments.end(); ++comp_iter)
+      comp_iter->original_offset += original_component_begin;
+    if (adjustments) {
+      adjustments->insert(adjustments->end(),
+                          component_transform_adjustments.begin(),
+                          component_transform_adjustments.end());
+    }
+
+    // Set positions of the parsed component.
+    if (output_component) {
+      output_component->begin = static_cast<int>(output_component_begin);
+      output_component->len =
+          static_cast<int>(output->length() - output_component_begin);
+    }
+  } else if (output_component) {
+    output_component->reset();
+  }
+}
+
+}  // namespace
+
+const FormatUrlType kFormatUrlOmitNothing                     = 0;
+const FormatUrlType kFormatUrlOmitUsernamePassword            = 1 << 0;
+const FormatUrlType kFormatUrlOmitHTTP                        = 1 << 1;
+const FormatUrlType kFormatUrlOmitTrailingSlashOnBareHostname = 1 << 2;
+const FormatUrlType kFormatUrlOmitAll = kFormatUrlOmitUsernamePassword |
+    kFormatUrlOmitHTTP | kFormatUrlOmitTrailingSlashOnBareHostname;
+
+base::string16 IDNToUnicode(const std::string& host,
+                            const std::string& languages) {
+  return IDNToUnicodeWithAdjustments(host, languages, NULL);
+}
+
 std::string GetDirectoryListingEntry(const base::string16& name,
                                      const std::string& raw_bytes,
                                      bool is_dir,
                                      int64_t size,
-                                     base::Time modified) {
+                                     Time modified) {
   std::string result;
   result.append("<script>addRow(");
   base::EscapeJSONString(name, true, &result);
   result.append(",");
   if (raw_bytes.empty()) {
     base::EscapeJSONString(EscapePath(base::UTF16ToUTF8(name)), true, &result);
   } else {
     base::EscapeJSONString(EscapePath(raw_bytes), true, &result);
   }
-
   if (is_dir) {
     result.append(",1,");
   } else {
     result.append(",0,");
   }
 
   // Negative size means unknown or not applicable (e.g. directory).
   base::string16 size_string;
   if (size >= 0)
     size_string = base::FormatBytesUnlocalized(size);
   base::EscapeJSONString(size_string, true, &result);
 
   result.append(",");
 
   base::string16 modified_str;
   // |modified| can be NULL in FTP listings.
-  if (!modified.is_null())
+  if (!modified.is_null()) {
     modified_str = base::TimeFormatShortDateAndTime(modified);
+  }
   base::EscapeJSONString(modified_str, true, &result);
 
   result.append(");</script>\n");
 
   return result;
 }
 
+void AppendFormattedHost(const GURL& url,
+                         const std::string& languages,
+                         base::string16* output) {
+  AppendFormattedComponent(url.possibly_invalid_spec(),
+      url.parsed_for_possibly_invalid_spec().host,
+      HostComponentTransform(languages), output, NULL, NULL);
+}
+
+base::string16 FormatUrlWithOffsets(
+    const GURL& url,
+    const std::string& languages,
+    FormatUrlTypes format_types,
+    UnescapeRule::Type unescape_rules,
+    url::Parsed* new_parsed,
+    size_t* prefix_end,
+    std::vector<size_t>* offsets_for_adjustment) {
+  base::OffsetAdjuster::Adjustments adjustments;
+  const base::string16& format_url_return_value =
+      FormatUrlWithAdjustments(url, languages, format_types, unescape_rules,
+                               new_parsed, prefix_end, &adjustments);
+  base::OffsetAdjuster::AdjustOffsets(adjustments, offsets_for_adjustment);
+  if (offsets_for_adjustment) {
+    std::for_each(
+        offsets_for_adjustment->begin(),
+        offsets_for_adjustment->end(),
+        base::LimitOffset<std::string>(format_url_return_value.length()));
+  }
+  return format_url_return_value;
+}
+
+base::string16 FormatUrlWithAdjustments(
+    const GURL& url,
+    const std::string& languages,
+    FormatUrlTypes format_types,
+    UnescapeRule::Type unescape_rules,
+    url::Parsed* new_parsed,
+    size_t* prefix_end,
+    base::OffsetAdjuster::Adjustments* adjustments) {
+  DCHECK(adjustments != NULL);
+  adjustments->clear();
+  url::Parsed parsed_temp;
+  if (!new_parsed)
+    new_parsed = &parsed_temp;
+  else
+    *new_parsed = url::Parsed();
+
+  // Special handling for view-source:.  Don't use content::kViewSourceScheme
+  // because this library shouldn't depend on chrome.
+  const char kViewSource[] = "view-source";
+  // Reject "view-source:view-source:..." to avoid deep recursion.
+  const char kViewSourceTwice[] = "view-source:view-source:";
+  if (url.SchemeIs(kViewSource) &&
+      !base::StartsWith(url.possibly_invalid_spec(), kViewSourceTwice,
+                        base::CompareCase::INSENSITIVE_ASCII)) {
+    return FormatViewSourceUrl(url, languages, format_types,
+                               unescape_rules, new_parsed, prefix_end,
+                               adjustments);
+  }
+
+  // We handle both valid and invalid URLs (this will give us the spec
+  // regardless of validity).
+  const std::string& spec = url.possibly_invalid_spec();
+  const url::Parsed& parsed = url.parsed_for_possibly_invalid_spec();
+
+  // Scheme & separators.  These are ASCII.
+  base::string16 url_string;
+  url_string.insert(
+      url_string.end(), spec.begin(),
+      spec.begin() + parsed.CountCharactersBefore(url::Parsed::USERNAME, true));
+  const char kHTTP[] = "http://";
+  const char kFTP[] = "ftp.";
+  // url_fixer::FixupURL() treats "ftp.foo.com" as ftp://ftp.foo.com.  This
+  // means that if we trim "http://" off a URL whose host starts with "ftp." and
+  // the user inputs this into any field subject to fixup (which is basically
+  // all input fields), the meaning would be changed.  (In fact, often the
+  // formatted URL is directly pre-filled into an input field.)  For this reason
+  // we avoid stripping "http://" in this case.
+  bool omit_http =
+      (format_types & kFormatUrlOmitHTTP) &&
+      base::EqualsASCII(url_string, kHTTP) &&
+      !base::StartsWith(url.host(), kFTP, base::CompareCase::SENSITIVE);
+  new_parsed->scheme = parsed.scheme;
+
+  // Username & password.
+  if ((format_types & kFormatUrlOmitUsernamePassword) != 0) {
+    // Remove the username and password fields. We don't want to display those
+    // to the user since they can be used for attacks,
+    // e.g. "http://google.com:search@evil.ru/"
+    new_parsed->username.reset();
+    new_parsed->password.reset();
+    // Update the adjustments based on removed username and/or password.
+    if (parsed.username.is_nonempty() || parsed.password.is_nonempty()) {
+      if (parsed.username.is_nonempty() && parsed.password.is_nonempty()) {
+        // The seeming off-by-two is to account for the ':' after the username
+        // and '@' after the password.
+        adjustments->push_back(base::OffsetAdjuster::Adjustment(
+            static_cast<size_t>(parsed.username.begin),
+            static_cast<size_t>(parsed.username.len + parsed.password.len + 2),
+            0));
+      } else {
+        const url::Component* nonempty_component =
+            parsed.username.is_nonempty() ? &parsed.username : &parsed.password;
+        // The seeming off-by-one is to account for the '@' after the
+        // username/password.
+        adjustments->push_back(base::OffsetAdjuster::Adjustment(
+            static_cast<size_t>(nonempty_component->begin),
+            static_cast<size_t>(nonempty_component->len + 1),
+            0));
+      }
+    }
+  } else {
+    AppendFormattedComponent(spec, parsed.username,
+                             NonHostComponentTransform(unescape_rules),
+                             &url_string, &new_parsed->username, adjustments);
+    if (parsed.password.is_valid())
+      url_string.push_back(':');
+    AppendFormattedComponent(spec, parsed.password,
+                             NonHostComponentTransform(unescape_rules),
+                             &url_string, &new_parsed->password, adjustments);
+    if (parsed.username.is_valid() || parsed.password.is_valid())
+      url_string.push_back('@');
+  }
+  if (prefix_end)
+    *prefix_end = static_cast<size_t>(url_string.length());
+
+  // Host.
+  AppendFormattedComponent(spec, parsed.host, HostComponentTransform(languages),
+                           &url_string, &new_parsed->host, adjustments);
+
+  // Port.
+  if (parsed.port.is_nonempty()) {
+    url_string.push_back(':');
+    new_parsed->port.begin = url_string.length();
+    url_string.insert(url_string.end(),
+                      spec.begin() + parsed.port.begin,
+                      spec.begin() + parsed.port.end());
+    new_parsed->port.len = url_string.length() - new_parsed->port.begin;
+  } else {
+    new_parsed->port.reset();
+  }
+
+  // Path & query.  Both get the same general unescape & convert treatment.
+  if (!(format_types & kFormatUrlOmitTrailingSlashOnBareHostname) ||
+      !CanStripTrailingSlash(url)) {
+    AppendFormattedComponent(spec, parsed.path,
+                             NonHostComponentTransform(unescape_rules),
+                             &url_string, &new_parsed->path, adjustments);
+  } else {
+    if (parsed.path.len > 0) {
+      adjustments->push_back(base::OffsetAdjuster::Adjustment(
+          parsed.path.begin, parsed.path.len, 0));
+    }
+  }
+  if (parsed.query.is_valid())
+    url_string.push_back('?');
+  AppendFormattedComponent(spec, parsed.query,
+                           NonHostComponentTransform(unescape_rules),
+                           &url_string, &new_parsed->query, adjustments);
+
+  // Ref.  This is valid, unescaped UTF-8, so we can just convert.
+  if (parsed.ref.is_valid())
+    url_string.push_back('#');
+  AppendFormattedComponent(spec, parsed.ref,
+                           NonHostComponentTransform(UnescapeRule::NONE),
+                           &url_string, &new_parsed->ref, adjustments);
+
+  // If we need to strip out http do it after the fact.
+  if (omit_http &&
+      base::StartsWith(url_string, base::ASCIIToUTF16(kHTTP),
+                       base::CompareCase::SENSITIVE)) {
+    const size_t kHTTPSize = arraysize(kHTTP) - 1;
+    url_string = url_string.substr(kHTTPSize);
+    // Because offsets in the |adjustments| are already calculated with respect
+    // to the string with the http:// prefix in it, those offsets remain correct
+    // after stripping the prefix.  The only thing necessary is to add an
+    // adjustment to reflect the stripped prefix.
+    adjustments->insert(adjustments->begin(),
+        base::OffsetAdjuster::Adjustment(0, kHTTPSize, 0));
+
+    if (prefix_end)
+      *prefix_end -= kHTTPSize;
+
+    // Adjust new_parsed.
+    DCHECK(new_parsed->scheme.is_valid());
+    int delta = -(new_parsed->scheme.len + 3);  // +3 for ://.
+    new_parsed->scheme.reset();
+    AdjustAllComponentsButScheme(delta, new_parsed);
+  }
+
+  return url_string;
+}
+
+base::string16 FormatUrl(const GURL& url,
+                         const std::string& languages,
+                         FormatUrlTypes format_types,
+                         UnescapeRule::Type unescape_rules,
+                         url::Parsed* new_parsed,
+                         size_t* prefix_end,
+                         size_t* offset_for_adjustment) {
+  Offsets offsets;
+  if (offset_for_adjustment)
+    offsets.push_back(*offset_for_adjustment);
+  base::string16 result = FormatUrlWithOffsets(url, languages, format_types,
+      unescape_rules, new_parsed, prefix_end, &offsets);
+  if (offset_for_adjustment)
+    *offset_for_adjustment = offsets[0];
+  return result;
+}
+
 }  // namespace net