| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright 2015 The Chromium Authors. All rights reserved. | |
| 2 // Use of this source code is governed by a BSD-style license that can be | |
| 3 // found in the LICENSE file. | |
| 4 | |
| 5 #include "components/url_formatter/url_formatter.h" | |
| 6 | |
| 7 #include <algorithm> | |
| 8 #include <map> | |
| 9 #include <utility> | |
| 10 | |
| 11 #include "base/lazy_instance.h" | |
| 12 #include "base/logging.h" | |
| 13 #include "base/macros.h" | |
| 14 #include "base/memory/singleton.h" | |
| 15 #include "base/stl_util.h" | |
| 16 #include "base/strings/string_tokenizer.h" | |
| 17 #include "base/strings/string_util.h" | |
| 18 #include "base/strings/utf_offset_string_conversions.h" | |
| 19 #include "base/strings/utf_string_conversions.h" | |
| 20 #include "base/synchronization/lock.h" | |
| 21 #include "third_party/icu/source/common/unicode/uidna.h" | |
| 22 #include "third_party/icu/source/common/unicode/uniset.h" | |
| 23 #include "third_party/icu/source/common/unicode/uscript.h" | |
| 24 #include "third_party/icu/source/i18n/unicode/regex.h" | |
| 25 #include "third_party/icu/source/i18n/unicode/ulocdata.h" | |
| 26 #include "url/gurl.h" | |
| 27 #include "url/third_party/mozilla/url_parse.h" | |
| 28 | |
| 29 namespace url_formatter { | |
| 30 | |
| 31 namespace { | |
| 32 | |
| 33 base::string16 IDNToUnicodeWithAdjustments( | |
| 34 const std::string& host, | |
| 35 const std::string& languages, | |
| 36 base::OffsetAdjuster::Adjustments* adjustments); | |
| 37 bool IDNToUnicodeOneComponent(const base::char16* comp, | |
| 38 size_t comp_len, | |
| 39 const std::string& languages, | |
| 40 base::string16* out); | |
| 41 | |
| 42 class AppendComponentTransform { | |
| 43 public: | |
| 44 AppendComponentTransform() {} | |
| 45 virtual ~AppendComponentTransform() {} | |
| 46 | |
| 47 virtual base::string16 Execute( | |
| 48 const std::string& component_text, | |
| 49 base::OffsetAdjuster::Adjustments* adjustments) const = 0; | |
| 50 | |
| 51 // NOTE: No DISALLOW_COPY_AND_ASSIGN here, since gcc < 4.3.0 requires an | |
| 52 // accessible copy constructor in order to call AppendFormattedComponent() | |
| 53 // with an inline temporary (see http://gcc.gnu.org/bugs/#cxx%5Frvalbind ). | |
| 54 }; | |
| 55 | |
| 56 class HostComponentTransform : public AppendComponentTransform { | |
| 57 public: | |
| 58 explicit HostComponentTransform(const std::string& languages) | |
| 59 : languages_(languages) {} | |
| 60 | |
| 61 private: | |
| 62 base::string16 Execute( | |
| 63 const std::string& component_text, | |
| 64 base::OffsetAdjuster::Adjustments* adjustments) const override { | |
| 65 return IDNToUnicodeWithAdjustments(component_text, languages_, adjustments); | |
| 66 } | |
| 67 | |
| 68 const std::string& languages_; | |
| 69 }; | |
| 70 | |
| 71 class NonHostComponentTransform : public AppendComponentTransform { | |
| 72 public: | |
| 73 explicit NonHostComponentTransform(net::UnescapeRule::Type unescape_rules) | |
| 74 : unescape_rules_(unescape_rules) {} | |
| 75 | |
| 76 private: | |
| 77 base::string16 Execute( | |
| 78 const std::string& component_text, | |
| 79 base::OffsetAdjuster::Adjustments* adjustments) const override { | |
| 80 return (unescape_rules_ == net::UnescapeRule::NONE) | |
| 81 ? base::UTF8ToUTF16WithAdjustments(component_text, adjustments) | |
| 82 : net::UnescapeAndDecodeUTF8URLComponentWithAdjustments( | |
| 83 component_text, unescape_rules_, adjustments); | |
| 84 } | |
| 85 | |
| 86 const net::UnescapeRule::Type unescape_rules_; | |
| 87 }; | |
| 88 | |
| 89 // Transforms the portion of |spec| covered by |original_component| according to | |
| 90 // |transform|. Appends the result to |output|. If |output_component| is | |
| 91 // non-NULL, its start and length are set to the transformed component's new | |
| 92 // start and length. If |adjustments| is non-NULL, appends adjustments (if | |
| 93 // any) that reflect the transformation the original component underwent to | |
| 94 // become the transformed value appended to |output|. | |
| 95 void AppendFormattedComponent(const std::string& spec, | |
| 96 const url::Component& original_component, | |
| 97 const AppendComponentTransform& transform, | |
| 98 base::string16* output, | |
| 99 url::Component* output_component, | |
| 100 base::OffsetAdjuster::Adjustments* adjustments) { | |
| 101 DCHECK(output); | |
| 102 if (original_component.is_nonempty()) { | |
| 103 size_t original_component_begin = | |
| 104 static_cast<size_t>(original_component.begin); | |
| 105 size_t output_component_begin = output->length(); | |
| 106 std::string component_str(spec, original_component_begin, | |
| 107 static_cast<size_t>(original_component.len)); | |
| 108 | |
| 109 // Transform |component_str| and modify |adjustments| appropriately. | |
| 110 base::OffsetAdjuster::Adjustments component_transform_adjustments; | |
| 111 output->append( | |
| 112 transform.Execute(component_str, &component_transform_adjustments)); | |
| 113 | |
| 114 // Shift all the adjustments made for this component so the offsets are | |
| 115 // valid for the original string and add them to |adjustments|. | |
| 116 for (base::OffsetAdjuster::Adjustments::iterator comp_iter = | |
| 117 component_transform_adjustments.begin(); | |
| 118 comp_iter != component_transform_adjustments.end(); ++comp_iter) | |
| 119 comp_iter->original_offset += original_component_begin; | |
| 120 if (adjustments) { | |
| 121 adjustments->insert(adjustments->end(), | |
| 122 component_transform_adjustments.begin(), | |
| 123 component_transform_adjustments.end()); | |
| 124 } | |
| 125 | |
| 126 // Set positions of the parsed component. | |
| 127 if (output_component) { | |
| 128 output_component->begin = static_cast<int>(output_component_begin); | |
| 129 output_component->len = | |
| 130 static_cast<int>(output->length() - output_component_begin); | |
| 131 } | |
| 132 } else if (output_component) { | |
| 133 output_component->reset(); | |
| 134 } | |
| 135 } | |
| 136 | |
| 137 // If |component| is valid, its begin is incremented by |delta|. | |
| 138 void AdjustComponent(int delta, url::Component* component) { | |
| 139 if (!component->is_valid()) | |
| 140 return; | |
| 141 | |
| 142 DCHECK(delta >= 0 || component->begin >= -delta); | |
| 143 component->begin += delta; | |
| 144 } | |
| 145 | |
| 146 // Adjusts all the components of |parsed| by |delta|, except for the scheme. | |
| 147 void AdjustAllComponentsButScheme(int delta, url::Parsed* parsed) { | |
| 148 AdjustComponent(delta, &(parsed->username)); | |
| 149 AdjustComponent(delta, &(parsed->password)); | |
| 150 AdjustComponent(delta, &(parsed->host)); | |
| 151 AdjustComponent(delta, &(parsed->port)); | |
| 152 AdjustComponent(delta, &(parsed->path)); | |
| 153 AdjustComponent(delta, &(parsed->query)); | |
| 154 AdjustComponent(delta, &(parsed->ref)); | |
| 155 } | |
| 156 | |
| 157 // Helper for FormatUrlWithOffsets(). | |
| 158 base::string16 FormatViewSourceUrl( | |
| 159 const GURL& url, | |
| 160 const std::string& languages, | |
| 161 FormatUrlTypes format_types, | |
| 162 net::UnescapeRule::Type unescape_rules, | |
| 163 url::Parsed* new_parsed, | |
| 164 size_t* prefix_end, | |
| 165 base::OffsetAdjuster::Adjustments* adjustments) { | |
| 166 DCHECK(new_parsed); | |
| 167 const char kViewSource[] = "view-source:"; | |
| 168 const size_t kViewSourceLength = arraysize(kViewSource) - 1; | |
| 169 | |
| 170 // Format the underlying URL and record adjustments. | |
| 171 const std::string& url_str(url.possibly_invalid_spec()); | |
| 172 adjustments->clear(); | |
| 173 base::string16 result( | |
| 174 base::ASCIIToUTF16(kViewSource) + | |
| 175 FormatUrlWithAdjustments(GURL(url_str.substr(kViewSourceLength)), | |
| 176 languages, format_types, unescape_rules, | |
| 177 new_parsed, prefix_end, adjustments)); | |
| 178 // Revise |adjustments| by shifting to the offsets to prefix that the above | |
| 179 // call to FormatUrl didn't get to see. | |
| 180 for (base::OffsetAdjuster::Adjustments::iterator it = adjustments->begin(); | |
| 181 it != adjustments->end(); ++it) | |
| 182 it->original_offset += kViewSourceLength; | |
| 183 | |
| 184 // Adjust positions of the parsed components. | |
| 185 if (new_parsed->scheme.is_nonempty()) { | |
| 186 // Assume "view-source:real-scheme" as a scheme. | |
| 187 new_parsed->scheme.len += kViewSourceLength; | |
| 188 } else { | |
| 189 new_parsed->scheme.begin = 0; | |
| 190 new_parsed->scheme.len = kViewSourceLength - 1; | |
| 191 } | |
| 192 AdjustAllComponentsButScheme(kViewSourceLength, new_parsed); | |
| 193 | |
| 194 if (prefix_end) | |
| 195 *prefix_end += kViewSourceLength; | |
| 196 | |
| 197 return result; | |
| 198 } | |
| 199 | |
| 200 // TODO(brettw) bug 734373: check the scripts for each host component and | |
| 201 // don't un-IDN-ize if there is more than one. Alternatively, only IDN for | |
| 202 // scripts that the user has installed. For now, just put the entire | |
| 203 // path through IDN. Maybe this feature can be implemented in ICU itself? | |
| 204 // | |
| 205 // We may want to skip this step in the case of file URLs to allow unicode | |
| 206 // UNC hostnames regardless of encodings. | |
| 207 base::string16 IDNToUnicodeWithAdjustments( | |
| 208 const std::string& host, | |
| 209 const std::string& languages, | |
| 210 base::OffsetAdjuster::Adjustments* adjustments) { | |
| 211 if (adjustments) | |
| 212 adjustments->clear(); | |
| 213 // Convert the ASCII input to a base::string16 for ICU. | |
| 214 base::string16 input16; | |
| 215 input16.reserve(host.length()); | |
| 216 input16.insert(input16.end(), host.begin(), host.end()); | |
| 217 | |
| 218 // Do each component of the host separately, since we enforce script matching | |
| 219 // on a per-component basis. | |
| 220 base::string16 out16; | |
| 221 for (size_t component_start = 0, component_end; | |
| 222 component_start < input16.length(); | |
| 223 component_start = component_end + 1) { | |
| 224 // Find the end of the component. | |
| 225 component_end = input16.find('.', component_start); | |
| 226 if (component_end == base::string16::npos) | |
| 227 component_end = input16.length(); // For getting the last component. | |
| 228 size_t component_length = component_end - component_start; | |
| 229 size_t new_component_start = out16.length(); | |
| 230 bool converted_idn = false; | |
| 231 if (component_end > component_start) { | |
| 232 // Add the substring that we just found. | |
| 233 converted_idn = | |
| 234 IDNToUnicodeOneComponent(input16.data() + component_start, | |
| 235 component_length, languages, &out16); | |
| 236 } | |
| 237 size_t new_component_length = out16.length() - new_component_start; | |
| 238 | |
| 239 if (converted_idn && adjustments) { | |
| 240 adjustments->push_back(base::OffsetAdjuster::Adjustment( | |
| 241 component_start, component_length, new_component_length)); | |
| 242 } | |
| 243 | |
| 244 // Need to add the dot we just found (if we found one). | |
| 245 if (component_end < input16.length()) | |
| 246 out16.push_back('.'); | |
| 247 } | |
| 248 return out16; | |
| 249 } | |
| 250 | |
| 251 // Does some simple normalization of scripts so we can allow certain scripts | |
| 252 // to exist together. | |
| 253 // TODO(brettw) bug 880223: we should allow some other languages to be | |
| 254 // oombined such as Chinese and Latin. We will probably need a more | |
| 255 // complicated system of language pairs to have more fine-grained control. | |
| 256 UScriptCode NormalizeScript(UScriptCode code) { | |
| 257 switch (code) { | |
| 258 case USCRIPT_KATAKANA: | |
| 259 case USCRIPT_HIRAGANA: | |
| 260 case USCRIPT_KATAKANA_OR_HIRAGANA: | |
| 261 case USCRIPT_HANGUL: // This one is arguable. | |
| 262 return USCRIPT_HAN; | |
| 263 default: | |
| 264 return code; | |
| 265 } | |
| 266 } | |
| 267 | |
| 268 bool IsIDNComponentInSingleScript(const base::char16* str, int str_len) { | |
| 269 UScriptCode first_script = USCRIPT_INVALID_CODE; | |
| 270 bool is_first = true; | |
| 271 | |
| 272 int i = 0; | |
| 273 while (i < str_len) { | |
| 274 unsigned code_point; | |
| 275 U16_NEXT(str, i, str_len, code_point); | |
| 276 | |
| 277 UErrorCode err = U_ZERO_ERROR; | |
| 278 UScriptCode cur_script = uscript_getScript(code_point, &err); | |
| 279 if (err != U_ZERO_ERROR) | |
| 280 return false; // Report mixed on error. | |
| 281 cur_script = NormalizeScript(cur_script); | |
| 282 | |
| 283 // TODO(brettw) We may have to check for USCRIPT_INHERENT as well. | |
| 284 if (is_first && cur_script != USCRIPT_COMMON) { | |
| 285 first_script = cur_script; | |
| 286 is_first = false; | |
| 287 } else { | |
| 288 if (cur_script != USCRIPT_COMMON && cur_script != first_script) | |
| 289 return false; | |
| 290 } | |
| 291 } | |
| 292 return true; | |
| 293 } | |
| 294 | |
| 295 // Check if the script of a language can be 'safely' mixed with | |
| 296 // Latin letters in the ASCII range. | |
| 297 bool IsCompatibleWithASCIILetters(const std::string& lang) { | |
| 298 // For now, just list Chinese, Japanese and Korean (positive list). | |
| 299 // An alternative is negative-listing (languages using Greek and | |
| 300 // Cyrillic letters), but it can be more dangerous. | |
| 301 return !lang.substr(0, 2).compare("zh") || !lang.substr(0, 2).compare("ja") || | |
| 302 !lang.substr(0, 2).compare("ko"); | |
| 303 } | |
| 304 | |
| 305 typedef std::map<std::string, icu::UnicodeSet*> LangToExemplarSetMap; | |
| 306 | |
| 307 class LangToExemplarSet { | |
| 308 public: | |
| 309 static LangToExemplarSet* GetInstance() { | |
| 310 return Singleton<LangToExemplarSet>::get(); | |
| 311 } | |
| 312 | |
| 313 private: | |
| 314 LangToExemplarSetMap map; | |
| 315 LangToExemplarSet() {} | |
| 316 ~LangToExemplarSet() { | |
| 317 STLDeleteContainerPairSecondPointers(map.begin(), map.end()); | |
| 318 } | |
| 319 | |
| 320 friend class Singleton<LangToExemplarSet>; | |
| 321 friend struct DefaultSingletonTraits<LangToExemplarSet>; | |
| 322 friend bool GetExemplarSetForLang(const std::string&, icu::UnicodeSet**); | |
| 323 friend void SetExemplarSetForLang(const std::string&, icu::UnicodeSet*); | |
| 324 | |
| 325 DISALLOW_COPY_AND_ASSIGN(LangToExemplarSet); | |
| 326 }; | |
| 327 | |
| 328 bool GetExemplarSetForLang(const std::string& lang, | |
| 329 icu::UnicodeSet** lang_set) { | |
| 330 const LangToExemplarSetMap& map = LangToExemplarSet::GetInstance()->map; | |
| 331 LangToExemplarSetMap::const_iterator pos = map.find(lang); | |
| 332 if (pos != map.end()) { | |
| 333 *lang_set = pos->second; | |
| 334 return true; | |
| 335 } | |
| 336 return false; | |
| 337 } | |
| 338 | |
| 339 void SetExemplarSetForLang(const std::string& lang, icu::UnicodeSet* lang_set) { | |
| 340 LangToExemplarSetMap& map = LangToExemplarSet::GetInstance()->map; | |
| 341 map.insert(std::make_pair(lang, lang_set)); | |
| 342 } | |
| 343 | |
| 344 static base::LazyInstance<base::Lock>::Leaky g_lang_set_lock = | |
| 345 LAZY_INSTANCE_INITIALIZER; | |
| 346 | |
| 347 // Returns true if all the characters in component_characters are used by | |
| 348 // the language |lang|. | |
| 349 bool IsComponentCoveredByLang(const icu::UnicodeSet& component_characters, | |
| 350 const std::string& lang) { | |
| 351 CR_DEFINE_STATIC_LOCAL(const icu::UnicodeSet, kASCIILetters, ('a', 'z')); | |
| 352 icu::UnicodeSet* lang_set = nullptr; | |
| 353 // We're called from both the UI thread and the history thread. | |
| 354 { | |
| 355 base::AutoLock lock(g_lang_set_lock.Get()); | |
| 356 if (!GetExemplarSetForLang(lang, &lang_set)) { | |
| 357 UErrorCode status = U_ZERO_ERROR; | |
| 358 ULocaleData* uld = ulocdata_open(lang.c_str(), &status); | |
| 359 // TODO(jungshik) Turn this check on when the ICU data file is | |
| 360 // rebuilt with the minimal subset of locale data for languages | |
| 361 // to which Chrome is not localized but which we offer in the list | |
| 362 // of languages selectable for Accept-Languages. With the rebuilt ICU | |
| 363 // data, ulocdata_open never should fall back to the default locale. | |
| 364 // (issue 2078) | |
| 365 // DCHECK(U_SUCCESS(status) && status != U_USING_DEFAULT_WARNING); | |
| 366 if (U_SUCCESS(status) && status != U_USING_DEFAULT_WARNING) { | |
| 367 lang_set = reinterpret_cast<icu::UnicodeSet*>(ulocdata_getExemplarSet( | |
| 368 uld, nullptr, 0, ULOCDATA_ES_STANDARD, &status)); | |
| 369 // On success, if |lang| is compatible with ASCII Latin letters, add | |
| 370 // them. | |
| 371 if (lang_set && IsCompatibleWithASCIILetters(lang)) | |
| 372 lang_set->addAll(kASCIILetters); | |
| 373 } | |
| 374 | |
| 375 if (!lang_set) | |
| 376 lang_set = new icu::UnicodeSet(1, 0); | |
| 377 | |
| 378 lang_set->freeze(); | |
| 379 SetExemplarSetForLang(lang, lang_set); | |
| 380 ulocdata_close(uld); | |
| 381 } | |
| 382 } | |
| 383 return !lang_set->isEmpty() && lang_set->containsAll(component_characters); | |
| 384 } | |
| 385 | |
| 386 // Returns true if the given Unicode host component is safe to display to the | |
| 387 // user. | |
| 388 bool IsIDNComponentSafe(const base::char16* str, | |
| 389 int str_len, | |
| 390 const std::string& languages) { | |
| 391 // Most common cases (non-IDN) do not reach here so that we don't | |
| 392 // need a fast return path. | |
| 393 // TODO(jungshik) : Check if there's any character inappropriate | |
| 394 // (although allowed) for domain names. | |
| 395 // See http://www.unicode.org/reports/tr39/#IDN_Security_Profiles and | |
| 396 // http://www.unicode.org/reports/tr39/data/xidmodifications.txt | |
| 397 // For now, we borrow the list from Mozilla and tweaked it slightly. | |
| 398 // (e.g. Characters like U+00A0, U+3000, U+3002 are omitted because | |
| 399 // they're gonna be canonicalized to U+0020 and full stop before | |
| 400 // reaching here.) | |
| 401 // The original list is available at | |
| 402 // http://kb.mozillazine.org/Network.IDN.blacklist_chars and | |
| 403 // at | |
| 404 // http://mxr.mozilla.org/seamonkey/source/modules/libpref/src/init/all.js#703 | |
| 405 | |
| 406 UErrorCode status = U_ZERO_ERROR; | |
| 407 #ifdef U_WCHAR_IS_UTF16 | |
| 408 icu::UnicodeSet dangerous_characters( | |
| 409 icu::UnicodeString( | |
| 410 L"[[\\ \u00ad\u00bc\u00bd\u01c3\u0337\u0338" | |
| 411 L"\u05c3\u05f4\u06d4\u0702\u115f\u1160][\u2000-\u200b]" | |
| 412 L"[\u2024\u2027\u2028\u2029\u2039\u203a\u2044\u205f]" | |
| 413 L"[\u2154-\u2156][\u2159-\u215b][\u215f\u2215\u23ae" | |
| 414 L"\u29f6\u29f8\u2afb\u2afd][\u2ff0-\u2ffb][\u3014" | |
| 415 L"\u3015\u3033\u3164\u321d\u321e\u33ae\u33af\u33c6\u33df\ufe14" | |
| 416 L"\ufe15\ufe3f\ufe5d\ufe5e\ufeff\uff0e\uff06\uff61\uffa0\ufff9]" | |
| 417 L"[\ufffa-\ufffd]\U0001f50f\U0001f510\U0001f512\U0001f513]"), | |
| 418 status); | |
| 419 DCHECK(U_SUCCESS(status)); | |
| 420 icu::RegexMatcher dangerous_patterns( | |
| 421 icu::UnicodeString( | |
| 422 // Lone katakana no, so, or n | |
| 423 L"[^\\p{Katakana}][\u30ce\u30f3\u30bd][^\\p{Katakana}]" | |
| 424 // Repeating Japanese accent characters | |
| 425 L"|[\u3099\u309a\u309b\u309c][\u3099\u309a\u309b\u309c]"), | |
| 426 0, status); | |
| 427 #else | |
| 428 icu::UnicodeSet dangerous_characters( | |
| 429 icu::UnicodeString( | |
| 430 "[[\\u0020\\u00ad\\u00bc\\u00bd\\u01c3\\u0337\\u0338" | |
| 431 "\\u05c3\\u05f4\\u06d4\\u0702\\u115f\\u1160][\\u2000-\\u200b]" | |
| 432 "[\\u2024\\u2027\\u2028\\u2029\\u2039\\u203a\\u2044\\u205f]" | |
| 433 "[\\u2154-\\u2156][\\u2159-\\u215b][\\u215f\\u2215\\u23ae" | |
| 434 "\\u29f6\\u29f8\\u2afb\\u2afd][\\u2ff0-\\u2ffb][\\u3014" | |
| 435 "\\u3015\\u3033\\u3164\\u321d\\u321e\\u33ae\\u33af\\u33c6\\u33df\\ufe" | |
| 436 "14" | |
| 437 "\\ufe15\\ufe3f\\ufe5d\\ufe5e\\ufeff\\uff0e\\uff06\\uff61\\uffa0\\uff" | |
| 438 "f9]" | |
| 439 "[\\ufffa-\\ufffd]\\U0001f50f\\U0001f510\\U0001f512\\U0001f513]", | |
| 440 -1, US_INV), | |
| 441 status); | |
| 442 DCHECK(U_SUCCESS(status)); | |
| 443 icu::RegexMatcher dangerous_patterns( | |
| 444 icu::UnicodeString( | |
| 445 // Lone katakana no, so, or n | |
| 446 "[^\\p{Katakana}][\\u30ce\\u30f3\\u30bd][^\\p{Katakana}]" | |
| 447 // Repeating Japanese accent characters | |
| 448 "|[\\u3099\\u309a\\u309b\\u309c][\\u3099\\u309a\\u309b\\u309c]"), | |
| 449 0, status); | |
| 450 #endif | |
| 451 DCHECK(U_SUCCESS(status)); | |
| 452 icu::UnicodeSet component_characters; | |
| 453 icu::UnicodeString component_string(str, str_len); | |
| 454 component_characters.addAll(component_string); | |
| 455 if (dangerous_characters.containsSome(component_characters)) | |
| 456 return false; | |
| 457 | |
| 458 DCHECK(U_SUCCESS(status)); | |
| 459 dangerous_patterns.reset(component_string); | |
| 460 if (dangerous_patterns.find()) | |
| 461 return false; | |
| 462 | |
| 463 // If the language list is empty, the result is completely determined | |
| 464 // by whether a component is a single script or not. This will block | |
| 465 // even "safe" script mixing cases like <Chinese, Latin-ASCII> that are | |
| 466 // allowed with |languages| (while it blocks Chinese + Latin letters with | |
| 467 // an accent as should be the case), but we want to err on the safe side | |
| 468 // when |languages| is empty. | |
| 469 if (languages.empty()) | |
| 470 return IsIDNComponentInSingleScript(str, str_len); | |
| 471 | |
| 472 // |common_characters| is made up of ASCII numbers, hyphen, plus and | |
| 473 // underscore that are used across scripts and allowed in domain names. | |
| 474 // (sync'd with characters allowed in url_canon_host with square | |
| 475 // brackets excluded.) See kHostCharLookup[] array in url_canon_host.cc. | |
| 476 icu::UnicodeSet common_characters(UNICODE_STRING_SIMPLE("[[0-9]\\-_+\\ ]"), | |
| 477 status); | |
| 478 DCHECK(U_SUCCESS(status)); | |
| 479 // Subtract common characters because they're always allowed so that | |
| 480 // we just have to check if a language-specific set contains | |
| 481 // the remainder. | |
| 482 component_characters.removeAll(common_characters); | |
| 483 | |
| 484 base::StringTokenizer t(languages, ","); | |
| 485 while (t.GetNext()) { | |
| 486 if (IsComponentCoveredByLang(component_characters, t.token())) | |
| 487 return true; | |
| 488 } | |
| 489 return false; | |
| 490 } | |
| 491 | |
| 492 // A wrapper to use LazyInstance<>::Leaky with ICU's UIDNA, a C pointer to | |
| 493 // a UTS46/IDNA 2008 handling object opened with uidna_openUTS46(). | |
| 494 // | |
| 495 // We use UTS46 with BiDiCheck to migrate from IDNA 2003 to IDNA 2008 with | |
| 496 // the backward compatibility in mind. What it does: | |
| 497 // | |
| 498 // 1. Use the up-to-date Unicode data. | |
| 499 // 2. Define a case folding/mapping with the up-to-date Unicode data as | |
| 500 // in IDNA 2003. | |
| 501 // 3. Use transitional mechanism for 4 deviation characters (sharp-s, | |
| 502 // final sigma, ZWJ and ZWNJ) for now. | |
| 503 // 4. Continue to allow symbols and punctuations. | |
| 504 // 5. Apply new BiDi check rules more permissive than the IDNA 2003 BiDI rules. | |
| 505 // 6. Do not apply STD3 rules | |
| 506 // 7. Do not allow unassigned code points. | |
| 507 // | |
| 508 // It also closely matches what IE 10 does except for the BiDi check ( | |
| 509 // http://goo.gl/3XBhqw ). | |
| 510 // See http://http://unicode.org/reports/tr46/ and references therein | |
| 511 // for more details. | |
| 512 struct UIDNAWrapper { | |
| 513 UIDNAWrapper() { | |
| 514 UErrorCode err = U_ZERO_ERROR; | |
| 515 // TODO(jungshik): Change options as different parties (browsers, | |
| 516 // registrars, search engines) converge toward a consensus. | |
| 517 value = uidna_openUTS46(UIDNA_CHECK_BIDI, &err); | |
| 518 if (U_FAILURE(err)) | |
| 519 value = NULL; | |
| 520 } | |
| 521 | |
| 522 UIDNA* value; | |
| 523 }; | |
| 524 | |
| 525 static base::LazyInstance<UIDNAWrapper>::Leaky g_uidna = | |
| 526 LAZY_INSTANCE_INITIALIZER; | |
| 527 | |
| 528 // Converts one component of a host (between dots) to IDN if safe. The result | |
| 529 // will be APPENDED to the given output string and will be the same as the input | |
| 530 // if it is not IDN or the IDN is unsafe to display. Returns whether any | |
| 531 // conversion was performed. | |
| 532 bool IDNToUnicodeOneComponent(const base::char16* comp, | |
| 533 size_t comp_len, | |
| 534 const std::string& languages, | |
| 535 base::string16* out) { | |
| 536 DCHECK(out); | |
| 537 if (comp_len == 0) | |
| 538 return false; | |
| 539 | |
| 540 // Only transform if the input can be an IDN component. | |
| 541 static const base::char16 kIdnPrefix[] = {'x', 'n', '-', '-'}; | |
| 542 if ((comp_len > arraysize(kIdnPrefix)) && | |
| 543 !memcmp(comp, kIdnPrefix, arraysize(kIdnPrefix) * sizeof(base::char16))) { | |
| 544 UIDNA* uidna = g_uidna.Get().value; | |
| 545 DCHECK(uidna != NULL); | |
| 546 size_t original_length = out->length(); | |
| 547 int output_length = 64; | |
| 548 UIDNAInfo info = UIDNA_INFO_INITIALIZER; | |
| 549 UErrorCode status; | |
| 550 do { | |
| 551 out->resize(original_length + output_length); | |
| 552 status = U_ZERO_ERROR; | |
| 553 // This returns the actual length required. If this is more than 64 | |
| 554 // code units, |status| will be U_BUFFER_OVERFLOW_ERROR and we'll try | |
| 555 // the conversion again, but with a sufficiently large buffer. | |
| 556 output_length = uidna_labelToUnicode( | |
| 557 uidna, comp, static_cast<int32_t>(comp_len), &(*out)[original_length], | |
| 558 output_length, &info, &status); | |
| 559 } while ((status == U_BUFFER_OVERFLOW_ERROR && info.errors == 0)); | |
| 560 | |
| 561 if (U_SUCCESS(status) && info.errors == 0) { | |
| 562 // Converted successfully. Ensure that the converted component | |
| 563 // can be safely displayed to the user. | |
| 564 out->resize(original_length + output_length); | |
| 565 if (IsIDNComponentSafe(out->data() + original_length, output_length, | |
| 566 languages)) | |
| 567 return true; | |
| 568 } | |
| 569 | |
| 570 // Something went wrong. Revert to original string. | |
| 571 out->resize(original_length); | |
| 572 } | |
| 573 | |
| 574 // We get here with no IDN or on error, in which case we just append the | |
| 575 // literal input. | |
| 576 out->append(comp, comp_len); | |
| 577 return false; | |
| 578 } | |
| 579 | |
| 580 } // namespace | |
| 581 | |
| 582 const FormatUrlType kFormatUrlOmitNothing = 0; | |
| 583 const FormatUrlType kFormatUrlOmitUsernamePassword = 1 << 0; | |
| 584 const FormatUrlType kFormatUrlOmitHTTP = 1 << 1; | |
| 585 const FormatUrlType kFormatUrlOmitTrailingSlashOnBareHostname = 1 << 2; | |
| 586 const FormatUrlType kFormatUrlOmitAll = | |
| 587 kFormatUrlOmitUsernamePassword | kFormatUrlOmitHTTP | | |
| 588 kFormatUrlOmitTrailingSlashOnBareHostname; | |
| 589 | |
| 590 base::string16 FormatUrl(const GURL& url, | |
| 591 const std::string& languages, | |
| 592 FormatUrlTypes format_types, | |
| 593 net::UnescapeRule::Type unescape_rules, | |
| 594 url::Parsed* new_parsed, | |
| 595 size_t* prefix_end, | |
| 596 size_t* offset_for_adjustment) { | |
| 597 std::vector<size_t> offsets; | |
| 598 if (offset_for_adjustment) | |
| 599 offsets.push_back(*offset_for_adjustment); | |
| 600 base::string16 result = | |
| 601 FormatUrlWithOffsets(url, languages, format_types, unescape_rules, | |
| 602 new_parsed, prefix_end, &offsets); | |
| 603 if (offset_for_adjustment) | |
| 604 *offset_for_adjustment = offsets[0]; | |
| 605 return result; | |
| 606 } | |
| 607 | |
| 608 base::string16 FormatUrlWithOffsets( | |
| 609 const GURL& url, | |
| 610 const std::string& languages, | |
| 611 FormatUrlTypes format_types, | |
| 612 net::UnescapeRule::Type unescape_rules, | |
| 613 url::Parsed* new_parsed, | |
| 614 size_t* prefix_end, | |
| 615 std::vector<size_t>* offsets_for_adjustment) { | |
| 616 base::OffsetAdjuster::Adjustments adjustments; | |
| 617 const base::string16& format_url_return_value = | |
| 618 FormatUrlWithAdjustments(url, languages, format_types, unescape_rules, | |
| 619 new_parsed, prefix_end, &adjustments); | |
| 620 base::OffsetAdjuster::AdjustOffsets(adjustments, offsets_for_adjustment); | |
| 621 if (offsets_for_adjustment) { | |
| 622 std::for_each( | |
| 623 offsets_for_adjustment->begin(), offsets_for_adjustment->end(), | |
| 624 base::LimitOffset<std::string>(format_url_return_value.length())); | |
| 625 } | |
| 626 return format_url_return_value; | |
| 627 } | |
| 628 | |
| 629 base::string16 FormatUrlWithAdjustments( | |
| 630 const GURL& url, | |
| 631 const std::string& languages, | |
| 632 FormatUrlTypes format_types, | |
| 633 net::UnescapeRule::Type unescape_rules, | |
| 634 url::Parsed* new_parsed, | |
| 635 size_t* prefix_end, | |
| 636 base::OffsetAdjuster::Adjustments* adjustments) { | |
| 637 DCHECK(adjustments != NULL); | |
| 638 adjustments->clear(); | |
| 639 url::Parsed parsed_temp; | |
| 640 if (!new_parsed) | |
| 641 new_parsed = &parsed_temp; | |
| 642 else | |
| 643 *new_parsed = url::Parsed(); | |
| 644 | |
| 645 // Special handling for view-source:. Don't use content::kViewSourceScheme | |
| 646 // because this library shouldn't depend on chrome. | |
| 647 const char kViewSource[] = "view-source"; | |
| 648 // Reject "view-source:view-source:..." to avoid deep recursion. | |
| 649 const char kViewSourceTwice[] = "view-source:view-source:"; | |
| 650 if (url.SchemeIs(kViewSource) && | |
| 651 !base::StartsWith(url.possibly_invalid_spec(), kViewSourceTwice, | |
| 652 base::CompareCase::INSENSITIVE_ASCII)) { | |
| 653 return FormatViewSourceUrl(url, languages, format_types, unescape_rules, | |
| 654 new_parsed, prefix_end, adjustments); | |
| 655 } | |
| 656 | |
| 657 // We handle both valid and invalid URLs (this will give us the spec | |
| 658 // regardless of validity). | |
| 659 const std::string& spec = url.possibly_invalid_spec(); | |
| 660 const url::Parsed& parsed = url.parsed_for_possibly_invalid_spec(); | |
| 661 | |
| 662 // Scheme & separators. These are ASCII. | |
| 663 base::string16 url_string; | |
| 664 url_string.insert( | |
| 665 url_string.end(), spec.begin(), | |
| 666 spec.begin() + parsed.CountCharactersBefore(url::Parsed::USERNAME, true)); | |
| 667 const char kHTTP[] = "http://"; | |
| 668 const char kFTP[] = "ftp."; | |
| 669 // url_formatter::FixupURL() treats "ftp.foo.com" as ftp://ftp.foo.com. This | |
| 670 // means that if we trim "http://" off a URL whose host starts with "ftp." and | |
| 671 // the user inputs this into any field subject to fixup (which is basically | |
| 672 // all input fields), the meaning would be changed. (In fact, often the | |
| 673 // formatted URL is directly pre-filled into an input field.) For this reason | |
| 674 // we avoid stripping "http://" in this case. | |
| 675 bool omit_http = | |
| 676 (format_types & kFormatUrlOmitHTTP) && | |
| 677 base::EqualsASCII(url_string, kHTTP) && | |
| 678 !base::StartsWith(url.host(), kFTP, base::CompareCase::SENSITIVE); | |
| 679 new_parsed->scheme = parsed.scheme; | |
| 680 | |
| 681 // Username & password. | |
| 682 if ((format_types & kFormatUrlOmitUsernamePassword) != 0) { | |
| 683 // Remove the username and password fields. We don't want to display those | |
| 684 // to the user since they can be used for attacks, | |
| 685 // e.g. "http://google.com:search@evil.ru/" | |
| 686 new_parsed->username.reset(); | |
| 687 new_parsed->password.reset(); | |
| 688 // Update the adjustments based on removed username and/or password. | |
| 689 if (parsed.username.is_nonempty() || parsed.password.is_nonempty()) { | |
| 690 if (parsed.username.is_nonempty() && parsed.password.is_nonempty()) { | |
| 691 // The seeming off-by-two is to account for the ':' after the username | |
| 692 // and '@' after the password. | |
| 693 adjustments->push_back(base::OffsetAdjuster::Adjustment( | |
| 694 static_cast<size_t>(parsed.username.begin), | |
| 695 static_cast<size_t>(parsed.username.len + parsed.password.len + 2), | |
| 696 0)); | |
| 697 } else { | |
| 698 const url::Component* nonempty_component = | |
| 699 parsed.username.is_nonempty() ? &parsed.username : &parsed.password; | |
| 700 // The seeming off-by-one is to account for the '@' after the | |
| 701 // username/password. | |
| 702 adjustments->push_back(base::OffsetAdjuster::Adjustment( | |
| 703 static_cast<size_t>(nonempty_component->begin), | |
| 704 static_cast<size_t>(nonempty_component->len + 1), 0)); | |
| 705 } | |
| 706 } | |
| 707 } else { | |
| 708 AppendFormattedComponent(spec, parsed.username, | |
| 709 NonHostComponentTransform(unescape_rules), | |
| 710 &url_string, &new_parsed->username, adjustments); | |
| 711 if (parsed.password.is_valid()) | |
| 712 url_string.push_back(':'); | |
| 713 AppendFormattedComponent(spec, parsed.password, | |
| 714 NonHostComponentTransform(unescape_rules), | |
| 715 &url_string, &new_parsed->password, adjustments); | |
| 716 if (parsed.username.is_valid() || parsed.password.is_valid()) | |
| 717 url_string.push_back('@'); | |
| 718 } | |
| 719 if (prefix_end) | |
| 720 *prefix_end = static_cast<size_t>(url_string.length()); | |
| 721 | |
| 722 // Host. | |
| 723 AppendFormattedComponent(spec, parsed.host, HostComponentTransform(languages), | |
| 724 &url_string, &new_parsed->host, adjustments); | |
| 725 | |
| 726 // Port. | |
| 727 if (parsed.port.is_nonempty()) { | |
| 728 url_string.push_back(':'); | |
| 729 new_parsed->port.begin = url_string.length(); | |
| 730 url_string.insert(url_string.end(), spec.begin() + parsed.port.begin, | |
| 731 spec.begin() + parsed.port.end()); | |
| 732 new_parsed->port.len = url_string.length() - new_parsed->port.begin; | |
| 733 } else { | |
| 734 new_parsed->port.reset(); | |
| 735 } | |
| 736 | |
| 737 // Path & query. Both get the same general unescape & convert treatment. | |
| 738 if (!(format_types & kFormatUrlOmitTrailingSlashOnBareHostname) || | |
| 739 !CanStripTrailingSlash(url)) { | |
| 740 AppendFormattedComponent(spec, parsed.path, | |
| 741 NonHostComponentTransform(unescape_rules), | |
| 742 &url_string, &new_parsed->path, adjustments); | |
| 743 } else { | |
| 744 if (parsed.path.len > 0) { | |
| 745 adjustments->push_back(base::OffsetAdjuster::Adjustment( | |
| 746 parsed.path.begin, parsed.path.len, 0)); | |
| 747 } | |
| 748 } | |
| 749 if (parsed.query.is_valid()) | |
| 750 url_string.push_back('?'); | |
| 751 AppendFormattedComponent(spec, parsed.query, | |
| 752 NonHostComponentTransform(unescape_rules), | |
| 753 &url_string, &new_parsed->query, adjustments); | |
| 754 | |
| 755 // Ref. This is valid, unescaped UTF-8, so we can just convert. | |
| 756 if (parsed.ref.is_valid()) | |
| 757 url_string.push_back('#'); | |
| 758 AppendFormattedComponent(spec, parsed.ref, | |
| 759 NonHostComponentTransform(net::UnescapeRule::NONE), | |
| 760 &url_string, &new_parsed->ref, adjustments); | |
| 761 | |
| 762 // If we need to strip out http do it after the fact. | |
| 763 if (omit_http && base::StartsWith(url_string, base::ASCIIToUTF16(kHTTP), | |
| 764 base::CompareCase::SENSITIVE)) { | |
| 765 const size_t kHTTPSize = arraysize(kHTTP) - 1; | |
| 766 url_string = url_string.substr(kHTTPSize); | |
| 767 // Because offsets in the |adjustments| are already calculated with respect | |
| 768 // to the string with the http:// prefix in it, those offsets remain correct | |
| 769 // after stripping the prefix. The only thing necessary is to add an | |
| 770 // adjustment to reflect the stripped prefix. | |
| 771 adjustments->insert(adjustments->begin(), | |
| 772 base::OffsetAdjuster::Adjustment(0, kHTTPSize, 0)); | |
| 773 | |
| 774 if (prefix_end) | |
| 775 *prefix_end -= kHTTPSize; | |
| 776 | |
| 777 // Adjust new_parsed. | |
| 778 DCHECK(new_parsed->scheme.is_valid()); | |
| 779 int delta = -(new_parsed->scheme.len + 3); // +3 for ://. | |
| 780 new_parsed->scheme.reset(); | |
| 781 AdjustAllComponentsButScheme(delta, new_parsed); | |
| 782 } | |
| 783 | |
| 784 return url_string; | |
| 785 } | |
| 786 | |
| 787 bool CanStripTrailingSlash(const GURL& url) { | |
| 788 // Omit the path only for standard, non-file URLs with nothing but "/" after | |
| 789 // the hostname. | |
| 790 return url.IsStandard() && !url.SchemeIsFile() && !url.SchemeIsFileSystem() && | |
| 791 !url.has_query() && !url.has_ref() && url.path() == "/"; | |
| 792 } | |
| 793 | |
| 794 void AppendFormattedHost(const GURL& url, | |
| 795 const std::string& languages, | |
| 796 base::string16* output) { | |
| 797 AppendFormattedComponent( | |
| 798 url.possibly_invalid_spec(), url.parsed_for_possibly_invalid_spec().host, | |
| 799 HostComponentTransform(languages), output, NULL, NULL); | |
| 800 } | |
| 801 | |
| 802 base::string16 IDNToUnicode(const std::string& host, | |
| 803 const std::string& languages) { | |
| 804 return IDNToUnicodeWithAdjustments(host, languages, NULL); | |
| 805 } | |
| 806 | |
| 807 } // url_formatter | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1260033005:
Revert of Move net::FormatUrl and friends outside of //net and into //components (Closed)
Base URL: https://chromium.googlesource.com/chromium/src.git@master