Remove diacritics when normalizing autofill profile strings for comparison.

components/autofill/core/browser/autofill_profile.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/autofill/core/browser/autofill_profile.h"
 
 #include <algorithm>
 #include <functional>
 #include <map>
 #include <memory>
@@ skipping 18 matching lines @@
 #include "components/autofill/core/browser/autofill_type.h"
 #include "components/autofill/core/browser/contact_info.h"
 #include "components/autofill/core/browser/phone_number.h"
 #include "components/autofill/core/browser/phone_number_i18n.h"
 #include "components/autofill/core/browser/state_names.h"
 #include "components/autofill/core/browser/validation.h"
 #include "components/autofill/core/common/autofill_l10n_util.h"
 #include "components/autofill/core/common/form_field_data.h"
 #include "grit/components_strings.h"
 #include "third_party/icu/source/common/unicode/uchar.h"
+#include "third_party/icu/source/common/unicode/utypes.h"
+#include "third_party/icu/source/i18n/unicode/translit.h"
 #include "third_party/libaddressinput/chromium/addressinput_util.h"
 #include "third_party/libaddressinput/src/cpp/include/libaddressinput/address_data.h"
 #include "third_party/libaddressinput/src/cpp/include/libaddressinput/address_formatter.h"
 #include "third_party/libaddressinput/src/cpp/include/libaddressinput/address_metadata.h"
 #include "ui/base/l10n/l10n_util.h"
 
 using base::ASCIIToUTF16;
 using base::UTF16ToUTF8;
 using i18n::addressinput::AddressData;
 using i18n::addressinput::AddressField;
@@ skipping 179 matching lines @@
   bool operator()(const base::string16& phone) {
     return i18n::PhoneNumbersMatch(phone, phone_, country_code_, app_locale_);
   }
 
  private:
   base::string16 phone_;
   std::string country_code_;
   std::string app_locale_;
 };
 
+base::string16 NormalizeForComparison(const base::string16& text) {
+  using icu::UnicodeString;
+  using icu::Transliterator;
+
+  // Use ICU transliteration to remove diacritics and fold case.
+  // See http://userguide.icu-project.org/transforms/general
+  UErrorCode status = U_ZERO_ERROR;
+  std::unique_ptr<Transliterator> transliterator(Transliterator::createInstance(
+      "NFD; [:Nonspacing Mark:] Remove; Lower; NFC", UTRANS_FORWARD, status));
+  if (U_FAILURE(status) || transliterator == nullptr) {
+    LOG(ERROR) << "Failed to create ICU Transliterator: "
+               << u_errorName(status);
+    return text;
+  }
+
+  UnicodeString value = UnicodeString(text.data(), text.length());
+  transliterator->transliterate(value);
+
+  return base::string16(value.getBuffer(), value.length());
+}
+
 }  // namespace
 
 AutofillProfile::AutofillProfile(const std::string& guid,
                                  const std::string& origin)
     : AutofillDataModel(guid, origin),
       record_type_(LOCAL_PROFILE),
       phone_number_(this) {
 }
 
 AutofillProfile::AutofillProfile(RecordType type, const std::string& server_id)
@@ skipping 539 matching lines @@
                             (base::Time::Now() - use_date()).InDays());
   RecordUse();
 }
 
 // static
 base::string16 AutofillProfile::CanonicalizeProfileString(
     const base::string16& str) {
   base::string16 ret;
   ret.reserve(str.size());
 
-  bool previous_was_whitespace = false;
+  // This algorithm is not designed to be perfect, we could get arbitrarily
+  // fancy here trying to canonicalize address lines. Instead, this is designed
+  // to handle common cases for all types of data (addresses and names) without
+  // the need of domain-specific logic.
+  //
+  // 1. Convert punctuation to spaces and normalize all whitespace to spaces.
+  //    This will convert "Mid-Island Plz." -> "Mid Island Plz " (the trailing
+  //    space will be trimmed off outside of the end of the loop).
+  //
+  // 2. Collapse consecutive punctuation/whitespace characters to a single
+  //    space. We pretend the string has already started with whitespace in
+  //    order to trim leading spaces.
+  //
+  // 3. Remove diacritics (accents and other non-spacing marks) and perform
+  //    case folding to lower-case.
 
-  // This algorithm isn't designed to be perfect, we could get arbitrarily
-  // fancy here trying to canonicalize address lines. Instead, this is designed
-  // to handle common cases for all types of data (addresses and names)
-  // without the need of domain-specific logic.
-  base::i18n::UTF16CharIterator iter(&str);
-  while (!iter.end()) {
+  bool previous_was_whitespace = true;
+  for (base::i18n::UTF16CharIterator iter(&str); !iter.end(); iter.Advance()) {
     switch (u_charType(iter.get())) {
+      // Punctuation
       case U_DASH_PUNCTUATION:
       case U_START_PUNCTUATION:
       case U_END_PUNCTUATION:
       case U_CONNECTOR_PUNCTUATION:
       case U_OTHER_PUNCTUATION:
-        // Convert punctuation to spaces. This will convert "Mid-Island Plz."
-        // -> "Mid Island Plz" (the trailing space will be trimmed off at the
-        // end of the loop).
+      // Whitespace
+      case U_CONTROL_CHAR:  // To escape the '\n' character.
+      case U_SPACE_SEPARATOR:
+      case U_LINE_SEPARATOR:
+      case U_PARAGRAPH_SEPARATOR:
         if (!previous_was_whitespace) {
           ret.push_back(' ');
           previous_was_whitespace = true;
         }
         break;
 
-      case U_CONTROL_CHAR:  // To escape the '\n' character.
-      case U_SPACE_SEPARATOR:
-      case U_LINE_SEPARATOR:
-      case U_PARAGRAPH_SEPARATOR:
-        // Convert sequences of spaces to single spaces.
-        if (!previous_was_whitespace) {
-          ret.push_back(' ');
-          previous_was_whitespace = true;
-        }
-        break;
-
-      case U_UPPERCASE_LETTER:
-      case U_TITLECASE_LETTER:
-        previous_was_whitespace = false;
-        base::WriteUnicodeCharacter(u_tolower(iter.get()), &ret);
-        break;
-
       default:
         previous_was_whitespace = false;
         base::WriteUnicodeCharacter(iter.get(), &ret);
         break;
     }
-    iter.Advance();
   }
 
   // Trim off trailing whitespace if we left one.
-  if (previous_was_whitespace)
+  if (previous_was_whitespace && !ret.empty())
     ret.resize(ret.size() - 1);
 
-  return ret;
+  // Remove diacritics and perform case folding.
+  return NormalizeForComparison(ret);
 }
 
 // static
 bool AutofillProfile::AreProfileStringsSimilar(const base::string16& a,
                                                const base::string16& b) {
   return CanonicalizeProfileString(a) == CanonicalizeProfileString(b);
 }
 
 void AutofillProfile::GetSupportedTypes(
     ServerFieldTypeSet* supported_types) const {
@@ skipping 224 matching lines @@
             << " " << UTF16ToUTF8(profile.GetRawInfo(ADDRESS_HOME_CITY)) << " "
             << UTF16ToUTF8(profile.GetRawInfo(ADDRESS_HOME_STATE)) << " "
             << UTF16ToUTF8(profile.GetRawInfo(ADDRESS_HOME_ZIP)) << " "
             << UTF16ToUTF8(profile.GetRawInfo(ADDRESS_HOME_SORTING_CODE)) << " "
             << UTF16ToUTF8(profile.GetRawInfo(ADDRESS_HOME_COUNTRY)) << " "
             << profile.language_code() << " "
             << UTF16ToUTF8(profile.GetRawInfo(PHONE_HOME_WHOLE_NUMBER));
 }
 
 }  // namespace autofill