Support IDNA 2008

net/base/net_util.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "net/base/net_util.h"
 
 #include <algorithm>
 #include <iterator>
 #include <map>
 
@@ skipping 306 matching lines @@
   // (e.g. Characters like U+00A0, U+3000, U+3002 are omitted because
   //  they're gonna be canonicalized to U+0020 and full stop before
   //  reaching here.)
   // The original list is available at
   // http://kb.mozillazine.org/Network.IDN.blacklist_chars and
   // at http://mxr.mozilla.org/seamonkey/source/modules/libpref/src/init/all.js#703
 
   UErrorCode status = U_ZERO_ERROR;
 #ifdef U_WCHAR_IS_UTF16
   icu::UnicodeSet dangerous_characters(icu::UnicodeString(
-      L"[[\\ \u00bc\u00bd\u01c3\u0337\u0338"
+      L"[[\\ \u00ad\u00bc\u00bd\u01c3\u0337\u0338"
       L"\u05c3\u05f4\u06d4\u0702\u115f\u1160][\u2000-\u200b]"
       L"[\u2024\u2027\u2028\u2029\u2039\u203a\u2044\u205f]"
       L"[\u2154-\u2156][\u2159-\u215b][\u215f\u2215\u23ae"
       L"\u29f6\u29f8\u2afb\u2afd][\u2ff0-\u2ffb][\u3014"
       L"\u3015\u3033\u3164\u321d\u321e\u33ae\u33af\u33c6\u33df\ufe14"
       L"\ufe15\ufe3f\ufe5d\ufe5e\ufeff\uff0e\uff06\uff61\uffa0\ufff9]"
       L"[\ufffa-\ufffd]]"), status);
   DCHECK(U_SUCCESS(status));
   icu::RegexMatcher dangerous_patterns(icu::UnicodeString(
       // Lone katakana no, so, or n
       L"[^\\p{Katakana}][\u30ce\u30f3\u30bd][^\\p{Katakana}]"
       // Repeating Japanese accent characters
       L"|[\u3099\u309a\u309b\u309c][\u3099\u309a\u309b\u309c]"),
       0, status);
 #else
   icu::UnicodeSet dangerous_characters(icu::UnicodeString(
-      "[[\\u0020\\u00bc\\u00bd\\u01c3\\u0337\\u0338"
+      "[[\\u0020\\u00ad\\u00bc\\u00bd\\u01c3\\u0337\\u0338"
       "\\u05c3\\u05f4\\u06d4\\u0702\\u115f\\u1160][\\u2000-\\u200b]"
       "[\\u2024\\u2027\\u2028\\u2029\\u2039\\u203a\\u2044\\u205f]"
       "[\\u2154-\\u2156][\\u2159-\\u215b][\\u215f\\u2215\\u23ae"
       "\\u29f6\\u29f8\\u2afb\\u2afd][\\u2ff0-\\u2ffb][\\u3014"
       "\\u3015\\u3033\\u3164\\u321d\\u321e\\u33ae\\u33af\\u33c6\\u33df\\ufe14"
       "\\ufe15\\ufe3f\\ufe5d\\ufe5e\\ufeff\\uff0e\\uff06\\uff61\\uffa0\\ufff9]"
       "[\\ufffa-\\ufffd]]", -1, US_INV), status);
   DCHECK(U_SUCCESS(status));
   icu::RegexMatcher dangerous_patterns(icu::UnicodeString(
       // Lone katakana no, so, or n
@@ skipping 36 matching lines @@
   component_characters.removeAll(common_characters);
 
   base::StringTokenizer t(languages, ",");
   while (t.GetNext()) {
     if (IsComponentCoveredByLang(component_characters, t.token()))
       return true;
   }
   return false;
 }
 
+struct uidna_wrapper {

[Peter Kasting, 2013/09/19 20:55:36]

Struct names should be CamelCase.

[jungshik at Google, 2013/09/20 21:33:41]

Done.

+  uidna_wrapper() {
+    UErrorCode err = U_ZERO_ERROR;
+    // This is the option closest to what we had in the past with IDNA 2003

[Peter Kasting, 2013/09/19 20:55:36]

Never write in a comment about what "used to" happen.  In the long run, no one
knows or cares about what "used to" happen or even knows when you're referring
to.

Instead, write a comment that merely explains why we're using this value now and
what it does for us.  I would also add a comment above the struct saying what
the point of the struct is; the broad picture of "what's going on" can go there,
and specific implementation details can go in the comment here.

[jungshik at Google, 2013/09/20 21:33:41]

Done. I just kept TODO comment here and explained other things before struct. 

+    // API and matches what IE 10 does except for BiDi check.
+    // IDNA 2003 always checks BiDi. We used to allow unassigned code
+    // points. However, with our Unicode DB pretty up to date, we'd not
+    // need to turn this on.
+    // We didn't use STD3 rules and we continue not to.
+    // TODO(jungshik) : Change options as different parties (browsers,
+    // registrars, search engines) converge toward a consensus.
+    int32_t options = UIDNA_CHECK_BIDI;

[Peter Kasting, 2013/09/19 20:55:36]

Nit: Don't make a temp for this, just inline it.

[jungshik at Google, 2013/09/20 21:33:41]

Done.

+    value = uidna_openUTS46(options, &err);
+    if (U_FAILURE(err))
+      value = NULL;
+  }
+
+  UIDNA* value;
+};
+
+static base::LazyInstance<uidna_wrapper>::Leaky
+    g_uidna = LAZY_INSTANCE_INITIALIZER;
+
 // Converts one component of a host (between dots) to IDN if safe. The result
 // will be APPENDED to the given output string and will be the same as the input
 // if it is not IDN or the IDN is unsafe to display.  Returns whether any
 // conversion was performed.
 bool IDNToUnicodeOneComponent(const base::char16* comp,
                               size_t comp_len,
                               const std::string& languages,
                               base::string16* out) {
   DCHECK(out);
   if (comp_len == 0)
     return false;
 
   // Only transform if the input can be an IDN component.
   static const base::char16 kIdnPrefix[] = {'x', 'n', '-', '-'};
   if ((comp_len > arraysize(kIdnPrefix)) &&
       !memcmp(comp, kIdnPrefix, arraysize(kIdnPrefix) * sizeof(base::char16))) {
-    // Repeatedly expand the output string until it's big enough.  It looks like
-    // ICU will return the required size of the buffer, but that's not
-    // documented, so we'll just grow by 2x. This should be rare and is not on a
-    // critical path.
+
+    UIDNA* uidna = g_uidna.Get().value;
+    DCHECK(uidna != NULL);
     size_t original_length = out->length();
-    for (int extra_space = 64; ; extra_space *= 2) {
+    for (int output_length = 64; ; ) {

[Peter Kasting, 2013/09/19 20:55:36]

How about writing the loop this way:

    // Try to convert the IDN to Unicode.  Arbitrarily start with 64 characters
    // of extra space for the conversion.
    int output_length = 64;
    UErrorCode status = U_ZERO_ERROR;
    UIDNAInfo info = UIDNA_INFO_INITIALIZER;
    do {
      out->resize(original_length + output_length);
      // This returns the actual length required.  If this is more than 64
      // characters, |status| will be U_BUFFER_OVERFLOW_ERROR and we'll try the
      // conversion again, but with a buffer that's sufficiently large.
      output_length = uidna_labelToUnicode(
          uidna, comp, static_cast<int32_t>(comp_len), &(*out)[original_length],
          output_length, &info, &status);
    } while ((status == U_BUFFER_OVERFLOW_ERROR) && (info.errors == 0));
    if (U_SUCCESS(status) && (info.errors == 0)) {
      // Converted successfully.  Ensure the converted component can be safely
      // displayed to the user.
      out->resize(original_length + output_length);
      if (IsIDNComponentSafe(out->data() + original_length, output_length,
                             languages))
        return true;
    }
    // Something went wrong.  Revert to the original string.
    out->resize(original_length);

This seems slightly clearer than the for() version, with more comments, and a
small line-wrapping change for style guide compliance.

[jungshik at Google, 2013/09/20 21:33:41]

Done with a slight change because UErrorCode has to be reset every time.

       UErrorCode status = U_ZERO_ERROR;
-      out->resize(out->length() + extra_space);
-      int output_chars = uidna_IDNToUnicode(comp,
-          static_cast<int32_t>(comp_len), &(*out)[original_length], extra_space,
-          UIDNA_DEFAULT, NULL, &status);
-      if (status == U_ZERO_ERROR) {
+      UIDNAInfo info = UIDNA_INFO_INITIALIZER;
+      out->resize(original_length + output_length);
+      output_length = uidna_labelToUnicode(uidna, comp,
+          static_cast<int32_t>(comp_len), &(*out)[original_length],
+          output_length, &info, &status);
+      if (U_SUCCESS(status) && info.errors == 0) {
         // Converted successfully.
-        out->resize(original_length + output_chars);
-        if (IsIDNComponentSafe(out->data() + original_length, output_chars,
+        out->resize(original_length + output_length);
+        if (IsIDNComponentSafe(out->data() + original_length, output_length,
                                languages))
           return true;
       }
 
-      if (status != U_BUFFER_OVERFLOW_ERROR)
+      if (status != U_BUFFER_OVERFLOW_ERROR || info.errors != 0)
         break;
     }
     // Failed, revert back to original string.
     out->resize(original_length);
   }
 
   // We get here with no IDN or on error, in which case we just append the
   // literal input.
   out->append(comp, comp_len);
   return false;
@@ skipping 1700 matching lines @@
 
 NetworkInterface::NetworkInterface(const std::string& name,
                                    const IPAddressNumber& address)
     : name(name), address(address) {
 }
 
 NetworkInterface::~NetworkInterface() {
 }
 
 }  // namespace net