Fixed ExtractSearchTermsFromURL for search engines with encoding != "UTF-8".

components/search_engines/template_url_unittest.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/base_paths.h"
 #include "base/command_line.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/string_util.h"
 #include "base/strings/utf_string_conversions.h"
 #include "components/metrics/proto/omnibox_event.pb.h"
@@ skipping 1124 matching lines @@
 
   EXPECT_FALSE(url.ExtractSearchTermsFromURL(
       GURL("http://term-in-path.com/begin"), search_terms_data_, &result));
   EXPECT_EQ(base::string16(), result);
 
   EXPECT_FALSE(url.ExtractSearchTermsFromURL(
       GURL("http://term-in-path.com/end"), search_terms_data_, &result));
   EXPECT_EQ(base::string16(), result);
 }
 
+TEST_F(TemplateURLTest, ExtractSearchTermsFromURL_CyrillicInWindows1251) {

[Peter Kasting, 2015/04/13 18:59:48]

Nit: I would name this "ExtraceSearchTermsFromNonUTF8URL" and the next one
"ExtraceSearchTermsFromUTF8URL".

Add comments on these describing what they're trying to test.

[Vitaly Baranov, 2015/04/14 13:26:15]

Done.

+  TemplateURLData data;
+  data.SetURL("http://windows-1251.ru/?q={searchTerms}");
+  data.alternate_urls.push_back("http://windows-1251.ru/#q={searchTerms}");
+  data.alternate_urls.push_back("http://windows-1251.ru/path/{searchTerms}");
+  data.input_encodings.push_back("windows-1251");
+  TemplateURL url(data);
+  base::string16 result;
+
+  EXPECT_TRUE(url.ExtractSearchTermsFromURL(
+      GURL("http://windows-1251.ru/?q="
+           "%C7%E4%F0%E0%E2%F1%F2%E2%F3%E9%2C+%EC%E8%F0!"),
+      search_terms_data_, &result));
+  EXPECT_EQ(base::UTF8ToUTF16("Здравствуй, мир!"), result);

[Peter Kasting, 2015/04/13 18:59:48]

Don't put extended characters directly into source code -- use C++ escaped
values.

That unfortunately makes this less readable, but it's safer and less likely to
break on systems with different default encodings.

[Vitaly Baranov, 2015/04/14 13:26:15]

Done.

+
+  EXPECT_TRUE(url.ExtractSearchTermsFromURL(
+      GURL("http://windows-1251.ru/#q=%E4%E2%E0+%F1%EB%EE%E2%E0"),
+      search_terms_data_, &result));
+  EXPECT_EQ(base::UTF8ToUTF16("два слова"), result);
+
+  EXPECT_TRUE(url.ExtractSearchTermsFromURL(
+      GURL("http://windows-1251.ru/path/%E1%F3%EA%E2%FB%20%C0%20%E8%20A"),
+      search_terms_data_, &result));
+  EXPECT_EQ(base::UTF8ToUTF16("буквы А и A"), result);
+}
+
+TEST_F(TemplateURLTest, ExtractSearchTermsFromURL_CyrillicInUTF8) {
+  TemplateURLData data;
+  data.SetURL("http://utf-8.ru/?q={searchTerms}");
+  data.alternate_urls.push_back("http://utf-8.ru/#q={searchTerms}");
+  data.alternate_urls.push_back("http://utf-8.ru/path/{searchTerms}");
+  TemplateURL url(data);
+  base::string16 result;
+
+  EXPECT_TRUE(url.ExtractSearchTermsFromURL(
+      GURL("http://utf-8.ru/?q=Здравствуй%2C+мир!"),
+      search_terms_data_, &result));
+  EXPECT_EQ(base::UTF8ToUTF16("Здравствуй, мир!"), result);
+
+  EXPECT_TRUE(url.ExtractSearchTermsFromURL(
+      GURL("http://utf-8.ru/#q=два+слова"),
+      search_terms_data_, &result));
+  EXPECT_EQ(base::UTF8ToUTF16("два слова"), result);
+
+  EXPECT_TRUE(url.ExtractSearchTermsFromURL(
+      GURL("http://utf-8.ru/path/буквы%20А%20и%20A"),
+      search_terms_data_, &result));
+  EXPECT_EQ(base::UTF8ToUTF16("буквы А и A"), result);
+}
+
 TEST_F(TemplateURLTest, HasSearchTermsReplacementKey) {
   TemplateURLData data;
   data.SetURL("http://google.com/?q={searchTerms}");
   data.instant_url = "http://google.com/instant#q={searchTerms}";
   data.alternate_urls.push_back("http://google.com/alt/#q={searchTerms}");
   data.alternate_urls.push_back(
       "http://google.com/alt/?ext=foo&q={searchTerms}#ref=bar");
   data.search_terms_replacement_key = "espv";
   TemplateURL url(data);
 
@@ skipping 445 matching lines @@
   EXPECT_EQ("http://bar/search?q=foo&pfq=full%20query%20text&qha=2338&xssi=t",
             result);
 
   TemplateURL url2(data);
   search_terms_args.prefetch_query.clear();
   search_terms_args.prefetch_query_type.clear();
   result =
       url2.url_ref().ReplaceSearchTerms(search_terms_args, search_terms_data_);
   EXPECT_EQ("http://bar/search?q=foo&xssi=t", result);
 }