Support IDNA 2008

net/base/net_util.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "net/base/net_util.h"
 
 #include <algorithm>
 #include <iterator>
 #include <map>
 
@@ skipping 306 matching lines @@
   // (e.g. Characters like U+00A0, U+3000, U+3002 are omitted because
   //  they're gonna be canonicalized to U+0020 and full stop before
   //  reaching here.)
   // The original list is available at
   // http://kb.mozillazine.org/Network.IDN.blacklist_chars and
   // at http://mxr.mozilla.org/seamonkey/source/modules/libpref/src/init/all.js#703
 
   UErrorCode status = U_ZERO_ERROR;
 #ifdef U_WCHAR_IS_UTF16
   icu::UnicodeSet dangerous_characters(icu::UnicodeString(
-      L"[[\\ \u00bc\u00bd\u01c3\u0337\u0338"
+      L"[[\\ \u00ad\u00bc\u00bd\u01c3\u0337\u0338"
       L"\u05c3\u05f4\u06d4\u0702\u115f\u1160][\u2000-\u200b]"
       L"[\u2024\u2027\u2028\u2029\u2039\u203a\u2044\u205f]"
       L"[\u2154-\u2156][\u2159-\u215b][\u215f\u2215\u23ae"
       L"\u29f6\u29f8\u2afb\u2afd][\u2ff0-\u2ffb][\u3014"
       L"\u3015\u3033\u3164\u321d\u321e\u33ae\u33af\u33c6\u33df\ufe14"
       L"\ufe15\ufe3f\ufe5d\ufe5e\ufeff\uff0e\uff06\uff61\uffa0\ufff9]"
       L"[\ufffa-\ufffd]]"), status);
   DCHECK(U_SUCCESS(status));
   icu::RegexMatcher dangerous_patterns(icu::UnicodeString(
       // Lone katakana no, so, or n
       L"[^\\p{Katakana}][\u30ce\u30f3\u30bd][^\\p{Katakana}]"
       // Repeating Japanese accent characters
       L"|[\u3099\u309a\u309b\u309c][\u3099\u309a\u309b\u309c]"),
       0, status);
 #else
   icu::UnicodeSet dangerous_characters(icu::UnicodeString(
-      "[[\\u0020\\u00bc\\u00bd\\u01c3\\u0337\\u0338"
+      "[[\\u0020\\u00ad\\u00bc\\u00bd\\u01c3\\u0337\\u0338"
       "\\u05c3\\u05f4\\u06d4\\u0702\\u115f\\u1160][\\u2000-\\u200b]"
       "[\\u2024\\u2027\\u2028\\u2029\\u2039\\u203a\\u2044\\u205f]"
       "[\\u2154-\\u2156][\\u2159-\\u215b][\\u215f\\u2215\\u23ae"
       "\\u29f6\\u29f8\\u2afb\\u2afd][\\u2ff0-\\u2ffb][\\u3014"
       "\\u3015\\u3033\\u3164\\u321d\\u321e\\u33ae\\u33af\\u33c6\\u33df\\ufe14"
       "\\ufe15\\ufe3f\\ufe5d\\ufe5e\\ufeff\\uff0e\\uff06\\uff61\\uffa0\\ufff9]"
       "[\\ufffa-\\ufffd]]", -1, US_INV), status);
   DCHECK(U_SUCCESS(status));
   icu::RegexMatcher dangerous_patterns(icu::UnicodeString(
       // Lone katakana no, so, or n
@@ skipping 36 matching lines @@
   component_characters.removeAll(common_characters);
 
   base::StringTokenizer t(languages, ",");
   while (t.GetNext()) {
     if (IsComponentCoveredByLang(component_characters, t.token()))
       return true;
   }
   return false;
 }
 
+// A wrapper to use LazyInstance<>::Leaky with ICU's UIDNA, a C pointer to
+// UTS46/IDNA 2008 handling object opened with |uidna_openUTS46|.

[Peter Kasting, 2013/09/23 21:12:44]

Nit: Function names don't go inside pipes; instead say "uidna_openUTS46()".

You need an "a" before "UTS46/IDNA 2008".

+//
+// We use UTS46 with BiDiCheck to migrate to IDNA 2008 with
+// the backward compatibility in mind. What it does :

[Peter Kasting, 2013/09/23 21:12:44]

Migrate from what?  Backward compatibility with what?

We either need to not reference any old versions or behavior at all, and only
speak in reference to the current behavior, or we need to say clearly what
behavior we're trying to reference.

Nit: Don't put spaces before colons (2 places)

+//
+// 1. Use the up-to-date Unicode data.
+// 2. Define a case folding/mapping with the up-to-date Unicode data as
+//    in IDNA 2003.
+// 3. Use transitional mechanism for 4 deviation characters (sharp-s,
+//    final sigma, ZWJ and ZWNJ) for now.
+// 4. Continue to allow symbols and punctuations.
+// 5. Apply new BiDi check rules more permissive than the IDNA 2003 BiDI rules.
+// 6. Do not apply STD3 rules
+// 7. Do not allow unassigned code points.
+//
+// It also closely matches what IE 10 does except for the BiDi check (
+// http://goo.gl/3XBhqw ).
+// See http://http://unicode.org/reports/tr46/ and references therein
+// for more details.
+struct UIDNAWrapper {
+  UIDNAWrapper() {
+    UErrorCode err = U_ZERO_ERROR;
+    // TODO(jungshik) : Change options as different parties (browsers,
+    // registrars, search engines) converge toward a consensus.
+    value = uidna_openUTS46(UIDNA_CHECK_BIDI, &err);
+    if (U_FAILURE(err))
+      value = NULL;
+  }
+
+  UIDNA* value;
+};
+
+static base::LazyInstance<UIDNAWrapper>::Leaky
+    g_uidna = LAZY_INSTANCE_INITIALIZER;
+
 // Converts one component of a host (between dots) to IDN if safe. The result
 // will be APPENDED to the given output string and will be the same as the input
 // if it is not IDN or the IDN is unsafe to display.  Returns whether any
 // conversion was performed.
 bool IDNToUnicodeOneComponent(const base::char16* comp,
                               size_t comp_len,
                               const std::string& languages,
                               base::string16* out) {
   DCHECK(out);
   if (comp_len == 0)
     return false;
 
   // Only transform if the input can be an IDN component.
   static const base::char16 kIdnPrefix[] = {'x', 'n', '-', '-'};
   if ((comp_len > arraysize(kIdnPrefix)) &&
       !memcmp(comp, kIdnPrefix, arraysize(kIdnPrefix) * sizeof(base::char16))) {
-    // Repeatedly expand the output string until it's big enough.  It looks like
-    // ICU will return the required size of the buffer, but that's not
-    // documented, so we'll just grow by 2x. This should be rare and is not on a
-    // critical path.
+    UIDNA* uidna = g_uidna.Get().value;
+    DCHECK(uidna != NULL);
     size_t original_length = out->length();
-    for (int extra_space = 64; ; extra_space *= 2) {
-      UErrorCode status = U_ZERO_ERROR;
-      out->resize(out->length() + extra_space);
-      int output_chars = uidna_IDNToUnicode(comp,
-          static_cast<int32_t>(comp_len), &(*out)[original_length], extra_space,
-          UIDNA_DEFAULT, NULL, &status);
-      if (status == U_ZERO_ERROR) {
-        // Converted successfully.
-        out->resize(original_length + output_chars);
-        if (IsIDNComponentSafe(out->data() + original_length, output_chars,
-                               languages))
-          return true;
-      }
+    int output_length = 64;
+    UIDNAInfo info = UIDNA_INFO_INITIALIZER;
+    UErrorCode status;
+    do {
+      out->resize(original_length + output_length);
+      status = U_ZERO_ERROR;
+      // This returns the actual length required. If this is more than 64
+      // code units, |status| will be U_BUFFER_OVERFLOW_ERROR and we'll try
+      // the conversion again, but with a sufficiently large buffer.
+      output_length = uidna_labelToUnicode(
+          uidna, comp, static_cast<int32_t>(comp_len), &(*out)[original_length],
+          output_length, &info, &status);
+    } while ((status == U_BUFFER_OVERFLOW_ERROR && info.errors == 0));
 
-      if (status != U_BUFFER_OVERFLOW_ERROR)
-        break;
+    if (U_SUCCESS(status) && info.errors == 0) {
+      // Converted successfully. Ensure that the converted component
+      // can be safely displayed to the user.
+      out->resize(original_length + output_length);
+      if (IsIDNComponentSafe(out->data() + original_length, output_length,
+                             languages))
+        return true;
     }
-    // Failed, revert back to original string.
+
+    // Something went wrong. Revert to original string.
     out->resize(original_length);
   }
 
   // We get here with no IDN or on error, in which case we just append the
   // literal input.
   out->append(comp, comp_len);
   return false;
 }
 
 // Clamps the offsets in |offsets_for_adjustment| to the length of |str|.
@@ skipping 1697 matching lines @@
 
 NetworkInterface::NetworkInterface(const std::string& name,
                                    const IPAddressNumber& address)
     : name(name), address(address) {
 }
 
 NetworkInterface::~NetworkInterface() {
 }
 
 }  // namespace net