All host names with nonascii characters (cyrillic) + escapable characters (,)...

googleurl/src/url_canon_host.cc

 // Copyright 2007, Google Inc.
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 // notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
@@ skipping 116 matching lines @@
 //    canonicalized host since we know nothing weird can happen (escaped
 //    characters could be unescaped to non-7-bit, so they have to be treated
 //    with suspicion at this point). It does not use the |has_non_ascii| flag.
 //
 //  * When the caller has an 8-bit string that may need unescaping.
 //    DoComplexHost calls us this situation to do unescaping and validation.
 //    After this, it may do other IDN operations depending on the value of the
 //    |*has_non_ascii| flag.
 //
 // The return value indicates if the output is a potentially valid host name.
-template<typename CHAR>
-bool DoSimpleHost(const CHAR* host, int host_len, CanonOutput* output,
+template<typename INCHAR, typename OUTCHAR>
+bool DoSimpleHost(const INCHAR* host,
+                  int host_len,
+                  CanonOutputT<OUTCHAR>* output,
                   bool* has_non_ascii) {
   *has_non_ascii = false;
 
   bool success = true;
-  for (int i = 0; i < host_len; i++) {
-    unsigned char source = static_cast<unsigned char>(host[i]);
+  for (int i = 0; i < host_len; ++i) {
+    unsigned int source = host[i];
     if (source == '%') {
-      // Handle unescaping. This will replace |source| with the unescaped char.
-      if (!DecodeEscaped(host, &i, host_len, &source)) {
+      // Unescape first, if possible.
+      // Source will be used only if decode operation was successful.
+      if (!DecodeEscaped(host, &i, host_len,
+                         reinterpret_cast<unsigned char*>(&source))) {
         // Invalid escaped character. There is nothing that can make this
         // host valid. We append an escaped percent so the URL looks reasonable
         // and mark as failed.
         AppendEscapedChar('%', output);
         success = false;
         continue;
       }
     }
 
-    if (source >= 0x80) {
-      // Handle non-ASCII.
-      *has_non_ascii = true;
-      output->push_back(source);
-    } else {
+    if (source <= 0x80) {
       // We have ASCII input, we can use our lookup table.
       unsigned char replacement = kHostCharLookup[source];
       if (!replacement) {
         // Invalid character, add it as percent-escaped and mark as failed.
         AppendEscapedChar(source, output);
         success = false;
       } else if (replacement == kEsc) {
         // This character is valid but should be escaped.
         AppendEscapedChar(source, output);
       } else {
         // Common case, the given character is valid in a hostname, the lookup
         // table tells us the canonical representation of that character (lower
         // cased).
         output->push_back(replacement);
       }
+    } else {
+      // It's a non-ascii char. Just push it to the output.
+      // In case where we have char16 input, and char output it's safe to
+      // cast char16->char only if input string was converted to ASCII.
+      output->push_back(static_cast<OUTCHAR>(source));
+      *has_non_ascii = true;
     }
   }
+
   return success;
 }
 
-// Canonicalizes a host that requires IDN conversion. Returns true on success.
+// Canonicalizes a host that requires IDN conversion. Returns true on success
 bool DoIDNHost(const char16* src, int src_len, CanonOutput* output) {
+  // We need to escape URL before doing IDN conversion, since punicode strings
+  // cannot be escaped after they are created.
+  RawCanonOutputW<kTempHostBufferLen> url_escaped_host;
+  bool has_non_ascii;
+  DoSimpleHost(src, src_len, &url_escaped_host, &has_non_ascii);
+
   StackBufferW wide_output;
-  if (!IDNToASCII(src, src_len, &wide_output)) {
+  if (!IDNToASCII(url_escaped_host.data(),
+                  url_escaped_host.length(),
+                  &wide_output)) {
     // Some error, give up. This will write some reasonable looking
     // representation of the string to the output.
     AppendInvalidNarrowString(src, 0, src_len, output);
     return false;
   }
 
   // Now we check the ASCII output like a normal host. It will also handle
   // unescaping. Although we unescaped everything before this function call, if
   // somebody does %00 as fullwidth, ICU will convert this to ASCII.
-  bool has_non_ascii;
-  bool success = DoSimpleHost<char16>(wide_output.data(),
-                                      wide_output.length(),
-                                      output, &has_non_ascii);
+  bool success = DoSimpleHost(wide_output.data(),
+                              wide_output.length(),
+                              output, &has_non_ascii);
   DCHECK(!has_non_ascii);
   return success;
 }
 
 // 8-bit convert host to its ASCII version: this converts the UTF-8 input to
 // UTF-16. The has_escaped flag should be set if the input string requires
 // unescaping.
 bool DoComplexHost(const char* host, int host_len,
                    bool has_non_ascii, bool has_escaped, CanonOutput* output) {
   // Save the current position in the output. We may write stuff and rewind it
@@ skipping 169 matching lines @@
 }
 
 void CanonicalizeHostVerbose(const char16* spec,
                              const url_parse::Component& host,
                              CanonOutput* output,
                              CanonHostInfo *host_info) {
   DoHost<char16, char16>(spec, host, output, host_info);
 }
 
 }  // namespace url_canon