Remove diacritics when normalizing autofill profile strings for comparison.

components/autofill/core/browser/autofill_profile.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/autofill/core/browser/autofill_profile.h"
 
 #include <algorithm>
 #include <functional>
 #include <map>
 #include <memory>
@@ skipping 18 matching lines @@
 #include "components/autofill/core/browser/autofill_type.h"
 #include "components/autofill/core/browser/contact_info.h"
 #include "components/autofill/core/browser/phone_number.h"
 #include "components/autofill/core/browser/phone_number_i18n.h"
 #include "components/autofill/core/browser/state_names.h"
 #include "components/autofill/core/browser/validation.h"
 #include "components/autofill/core/common/autofill_l10n_util.h"
 #include "components/autofill/core/common/form_field_data.h"
 #include "grit/components_strings.h"
 #include "third_party/icu/source/common/unicode/uchar.h"
+#include "third_party/icu/source/common/unicode/utypes.h"
+#include "third_party/icu/source/i18n/unicode/translit.h"
 #include "third_party/libaddressinput/chromium/addressinput_util.h"
 #include "third_party/libaddressinput/src/cpp/include/libaddressinput/address_data.h"
 #include "third_party/libaddressinput/src/cpp/include/libaddressinput/address_formatter.h"
 #include "third_party/libaddressinput/src/cpp/include/libaddressinput/address_metadata.h"
 #include "ui/base/l10n/l10n_util.h"
 
 using base::ASCIIToUTF16;
 using base::UTF16ToUTF8;
 using i18n::addressinput::AddressData;
 using i18n::addressinput::AddressField;
@@ skipping 179 matching lines @@
   bool operator()(const base::string16& phone) {
     return i18n::PhoneNumbersMatch(phone, phone_, country_code_, app_locale_);
   }
 
  private:
   base::string16 phone_;
   std::string country_code_;
   std::string app_locale_;
 };
 
+base::string16 NormalizeForComparison(const base::string16& text) {
+  using icu::UnicodeString;
+  using icu::Transliterator;
+
+   // Remove diacritics and fold case.

[Mathieu, 2016/05/27 17:47:13]

extra space before comment

[Roger McFarlane (Chromium), 2016/05/27 20:20:42]

Done.

+  UErrorCode status = U_ZERO_ERROR;
+  std::unique_ptr<Transliterator> transliterator(Transliterator::createInstance(
+      "NFD; [:Nonspacing Mark:] Remove; Lower; NFC", UTRANS_FORWARD, status));

[Mathieu, 2016/05/27 17:47:13]

Can we put an explanation (or a link to the explanation) for the magic string?
The more details the better, so that the reader (which could be us in 6 months)
know what's happening.

[Roger McFarlane (Chromium), 2016/05/27 20:20:43]

Done.

+  if (U_FAILURE(status)) {
+    // This should not happen. Log the error and fall back.
+    LOG(ERROR) << "normalization failed: " << u_errorName(status);

[Mathieu, 2016/05/27 17:47:13]

Let's capitalize the comment and make it more explicit so that it's clear to
someone reading the logs.

[Roger McFarlane (Chromium), 2016/05/27 20:20:43]

Done.

+    return text;
+  }
+
+  UnicodeString value = UnicodeString(text.data(), text.length());
+  transliterator->transliterate(value);

[Mathieu, 2016/05/27 17:47:13]

can we check "if (transliterator)" here? ICU has a bad habit of returning null
instances sometimes.

[Roger McFarlane (Chromium), 2016/05/27 20:20:43]

Done.

+
+  return base::string16(value.getBuffer(), value.length());
+}
+
 }  // namespace
 
 AutofillProfile::AutofillProfile(const std::string& guid,
                                  const std::string& origin)
     : AutofillDataModel(guid, origin),
       record_type_(LOCAL_PROFILE),
       phone_number_(this) {
 }
 
 AutofillProfile::AutofillProfile(RecordType type, const std::string& server_id)
@@ skipping 574 matching lines @@
       case U_LINE_SEPARATOR:
       case U_PARAGRAPH_SEPARATOR:
         // Convert sequences of spaces to single spaces.
         if (!previous_was_whitespace) {
           ret.push_back(' ');
           previous_was_whitespace = true;
         }
         break;
 
       case U_UPPERCASE_LETTER:
       case U_TITLECASE_LETTER:

[Mathieu, 2016/05/27 17:47:13]

Will this now be done by the normalizer?

[Roger McFarlane (Chromium), 2016/05/27 20:20:42]

Yes! Done.

         previous_was_whitespace = false;
         base::WriteUnicodeCharacter(u_tolower(iter.get()), &ret);
         break;
 
       default:
         previous_was_whitespace = false;
         base::WriteUnicodeCharacter(iter.get(), &ret);
         break;
     }
     iter.Advance();
   }
 
   // Trim off trailing whitespace if we left one.
   if (previous_was_whitespace)
     ret.resize(ret.size() - 1);
 
-  return ret;
+  return NormalizeForComparison(ret);
 }
 
 // static
 bool AutofillProfile::AreProfileStringsSimilar(const base::string16& a,
                                                const base::string16& b) {
   return CanonicalizeProfileString(a) == CanonicalizeProfileString(b);
 }
 
 void AutofillProfile::GetSupportedTypes(
     ServerFieldTypeSet* supported_types) const {
@@ skipping 224 matching lines @@
             << " " << UTF16ToUTF8(profile.GetRawInfo(ADDRESS_HOME_CITY)) << " "
             << UTF16ToUTF8(profile.GetRawInfo(ADDRESS_HOME_STATE)) << " "
             << UTF16ToUTF8(profile.GetRawInfo(ADDRESS_HOME_ZIP)) << " "
             << UTF16ToUTF8(profile.GetRawInfo(ADDRESS_HOME_SORTING_CODE)) << " "
             << UTF16ToUTF8(profile.GetRawInfo(ADDRESS_HOME_COUNTRY)) << " "
             << profile.language_code() << " "
             << UTF16ToUTF8(profile.GetRawInfo(PHONE_HOME_WHOLE_NUMBER));
 }
 
 }  // namespace autofill