Log Autofill.Quality.*.ByFieldType by predicted type for unknown data.

components/autofill/core/browser/autofill_metrics.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/autofill/core/browser/autofill_metrics.h"
 
 #include <algorithm>
 #include <utility>
 #include <vector>
 
 #include "base/logging.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/metrics/sparse_histogram.h"
 #include "base/metrics/user_metrics.h"
+#include "base/strings/string_piece.h"
+#include "base/strings/string_util.h"
 #include "base/time/time.h"
 #include "components/autofill/core/browser/autofill_experiments.h"
 #include "components/autofill/core/browser/autofill_field.h"
 #include "components/autofill/core/browser/autofill_type.h"
 #include "components/autofill/core/browser/form_structure.h"
 #include "components/autofill/core/common/form_data.h"
 #include "components/ukm/public/ukm_entry_builder.h"
 
 namespace internal {
 const char kUKMCardUploadDecisionEntryName[] = "Autofill.CardUploadDecision";
@@ skipping 48 matching lines @@
   GROUP_CREDIT_CARD_DATE,
   GROUP_CREDIT_CARD_TYPE,
   GROUP_PASSWORD,
   GROUP_ADDRESS_LINE_3,
   GROUP_USERNAME,
   GROUP_STREET_ADDRESS,
   GROUP_CREDIT_CARD_VERIFICATION,
   NUM_FIELD_TYPE_GROUPS_FOR_METRICS
 };
 
+const int KMaxFieldTypeGroupMetric =
+    (NUM_FIELD_TYPE_GROUPS_FOR_METRICS << 8) |
+    AutofillMetrics::NUM_FIELD_TYPE_QUALITY_METRICS;
+
 std::string PreviousSaveCreditCardPromptUserDecisionToString(
     int previous_save_credit_card_prompt_user_decision) {
   DCHECK_LT(previous_save_credit_card_prompt_user_decision,
             prefs::NUM_PREVIOUS_SAVE_CREDIT_CARD_PROMPT_USER_DECISIONS);
   std::string previous_response;
   if (previous_save_credit_card_prompt_user_decision ==
       prefs::PREVIOUS_SAVE_CREDIT_CARD_PROMPT_USER_DECISION_ACCEPTED)
     previous_response = ".PreviouslyAccepted";
   else if (previous_save_credit_card_prompt_user_decision ==
            prefs::PREVIOUS_SAVE_CREDIT_CARD_PROMPT_USER_DECISION_DENIED)
@@ skipping 128 matching lines @@
 
     case USERNAME_FIELD:
       group = GROUP_USERNAME;
       break;
 
     case TRANSACTION:
       NOTREACHED();
       break;
   }
 
-  // Interpolate the |metric| with the |group|, so that all metrics for a given
-  // |group| are adjacent.
-  return (group * AutofillMetrics::NUM_FIELD_TYPE_QUALITY_METRICS) + metric;
+  // Use bits 8-15 for the group and bits 0-7 for the metric.
+  static_assert(AutofillMetrics::NUM_FIELD_TYPE_QUALITY_METRICS <= UINT8_MAX,
+                "maximum field type quality metric must fit into 8 bits");
+  static_assert(NUM_FIELD_TYPE_GROUPS_FOR_METRICS <= UINT8_MAX,
+                "number of field type groups must fit into 8 bits");
+  return (group << 8) | metric;
 }
 
 namespace {
 
 // A version of the UMA_HISTOGRAM_ENUMERATION macro that allows the |name|
 // to vary over the program's runtime.
 void LogUMAHistogramEnumeration(const std::string& name,
                                 int sample,
                                 int boundary_value) {
   DCHECK_LT(sample, boundary_value);
@@ skipping 17 matching lines @@
   base::HistogramBase* histogram =
       base::Histogram::FactoryTimeGet(
           name,
           base::TimeDelta::FromMilliseconds(1),
           base::TimeDelta::FromHours(1),
           50,
           base::HistogramBase::kUmaTargetedHistogramFlag);
   histogram->AddTime(duration);
 }
 
-// Logs a type quality metric.  The primary histogram name is constructed based
-// on |base_name|.  The field-specific histogram name also factors in the
-// |field_type|.  Logs a sample of |metric|, which should be in the range
-// [0, |num_possible_metrics|). May log a suffixed version of the metric
-// depending on |metric_type|.
-void LogTypeQualityMetric(const std::string& base_name,
-                          AutofillMetrics::FieldTypeQualityMetric metric,
-                          ServerFieldType field_type,
-                          AutofillMetrics::QualityMetricType metric_type) {
-  DCHECK_LT(metric, AutofillMetrics::NUM_FIELD_TYPE_QUALITY_METRICS);
-
-  std::string suffix;
+const char* GetQualityMetricTypeSuffix(
+    AutofillMetrics::QualityMetricType metric_type) {
   switch (metric_type) {
-    case AutofillMetrics::TYPE_SUBMISSION:
-      break;
-    case AutofillMetrics::TYPE_NO_SUBMISSION:
-      suffix = ".NoSubmission";
-      break;
-    case AutofillMetrics::TYPE_AUTOCOMPLETE_BASED:
-      suffix = ".BasedOnAutocomplete";
-      break;
     default:
       NOTREACHED();
+    // Fall through...
+
+    case AutofillMetrics::TYPE_SUBMISSION:
+      return "";
+    case AutofillMetrics::TYPE_NO_SUBMISSION:
+      return ".NoSubmission";
+    case AutofillMetrics::TYPE_AUTOCOMPLETE_BASED:
+      return ".BasedOnAutocomplete";
   }
-  LogUMAHistogramEnumeration(base_name + suffix, metric,
+}
+
+// Given a set of |possible_types| for a field, select the best type to use as
+// the "actual" field type when calculating metrics. If the |predicted_type| is
+// among the |possible_types] then use that as the best type (i.e., the
+// prediction is deemed to have been correct).
+ServerFieldType GetActualFieldType(const ServerFieldTypeSet& possible_types,
+                                   ServerFieldType predicted_type) {
+  DCHECK_NE(possible_types.size(), 0u);
+
+  if (possible_types.count(EMPTY_TYPE)) {
+    DCHECK_EQ(possible_types.size(), 1u);
+    return EMPTY_TYPE;
+  }
+
+  if (possible_types.count(UNKNOWN_TYPE)) {
+    DCHECK_EQ(possible_types.size(), 1u);
+    return UNKNOWN_TYPE;
+  }
+
+  if (possible_types.count(predicted_type))
+    return predicted_type;
+
+  // Collapse field types that Chrome treats as identical, e.g. home and
+  // billing address fields.
+  ServerFieldTypeSet collapsed_field_types;
+  for (const auto& type : possible_types) {
+    DCHECK_NE(type, EMPTY_TYPE);
+    DCHECK_NE(type, UNKNOWN_TYPE);
+
+    // A phone number that's only missing its country code is (for metrics
+    // purposes) the same as the whole phone number.
+    if (type == PHONE_HOME_CITY_AND_NUMBER)
+      collapsed_field_types.insert(PHONE_HOME_WHOLE_NUMBER);
+    else
+      collapsed_field_types.insert(AutofillType(type).GetStorableType());
+  }
+
+  // Capture the field's type, if it is unambiguous.
+  ServerFieldType actual_type = AMBIGUOUS_TYPE;
+  if (collapsed_field_types.size() == 1)
+    actual_type = *collapsed_field_types.begin();
+
+  DVLOG(2) << "Inferred Type: " << AutofillType(actual_type).ToString();
+  return actual_type;
+}
+
+// Logs field type prediction quality metrics.  The primary histogram name is
+// constructed based on |source| The field-specific histogram name also factors
+// possible and predicted field types (|possible_types| and |predicted_type|,
+// respectively). May log a suffixed version of the metric depending on
+// |metric_type|.
+void LogPredictionQualityMetrics(
+    const base::StringPiece& source,
+    const ServerFieldTypeSet& possible_types,
+    ServerFieldType predicted_type,
+    AutofillMetrics::QualityMetricType metric_type) {
+  // Generate histogram names.
+  const char* const suffix = GetQualityMetricTypeSuffix(metric_type);
+  std::string raw_data_histogram =
+      base::JoinString({"Autofill.FieldPrediction.", source, suffix}, "");
+  std::string aggregate_histogram = base::JoinString(
+      {"Autofill.FieldPredictionQuality.Aggregate.", source, suffix}, "");
+  std::string type_specific_histogram = base::JoinString(
+      {"Autofill.FieldPredictionQuality.ByFieldType.", source, suffix}, "");
+
+  // Get the best type classification we can for the field.
+  ServerFieldType actual_type =
+      GetActualFieldType(possible_types, predicted_type);
+
+  DVLOG(2) << "Predicted: " << AutofillType(predicted_type).ToString() << "; "
+           << "Actual: " << AutofillType(actual_type).ToString();
+
+  DCHECK_LE(predicted_type, UINT16_MAX);
+  DCHECK_LE(actual_type, UINT16_MAX);
+  UMA_HISTOGRAM_SPARSE_SLOWLY(raw_data_histogram,
+                              (predicted_type << 16) | actual_type);
+
+  // NO_SERVER_DATA is the equivalent of predicting UNKNOWN.
+  if (predicted_type == NO_SERVER_DATA)
+    predicted_type = UNKNOWN_TYPE;
+
+  // The actual type being EMPTY_TYPE is the same as UNKNOWN_TYPE for comparison
+  // purposes, but remember whether or not it was empty for more precise logging
+  // later.
+  bool is_empty = (actual_type == EMPTY_TYPE);
+  bool is_ambiguous = (actual_type == AMBIGUOUS_TYPE);
+  if (is_empty || is_ambiguous)
+    actual_type = UNKNOWN_TYPE;
+
+  // If the predicted and actual types match then it's either a true positive
+  // or a true negative (if they are both unknown). Do not log type specific
+  // true negatives (instead log a true positive for the "Ambiguous" type).
+  if (predicted_type == actual_type) {
+    if (actual_type == UNKNOWN_TYPE) {
+      // Only log aggregate true negative; do not log type specific metrics
+      // for UNKNOWN/EMPTY.
+      DVLOG(2) << "TRUE NEGATIVE";
+      LogUMAHistogramEnumeration(
+          aggregate_histogram,
+          (is_empty ? AutofillMetrics::TRUE_NEGATIVE_EMPTY
+                    : (is_ambiguous ? AutofillMetrics::TRUE_NEGATIVE_AMBIGUOUS
+                                    : AutofillMetrics::TRUE_NEGATIVE_UNKNOWN)),
+          AutofillMetrics::NUM_FIELD_TYPE_QUALITY_METRICS);
+      return;
+    }
+
+    DVLOG(2) << "TRUE POSITIVE";
+    // Log both aggregate and type specific true positive if we correctly
+    // predict that type with which the field was filled.
+    LogUMAHistogramEnumeration(aggregate_histogram,
+                               AutofillMetrics::TRUE_POSITIVE,
+                               AutofillMetrics::NUM_FIELD_TYPE_QUALITY_METRICS);
+    LogUMAHistogramEnumeration(
+        type_specific_histogram,
+        GetFieldTypeGroupMetric(actual_type, AutofillMetrics::TRUE_POSITIVE),
+        KMaxFieldTypeGroupMetric);
+    return;
+  }
+
+  // Note: At this point predicted_type != actual type
+  // If actual type is UNKNOWN_TYPE then the prediction is a false positive.
+  // Further specialize the type of false positive by whether the field was
+  // empty or contained an unknown value.
+  if (actual_type == UNKNOWN_TYPE) {
+    DVLOG(2) << "FALSE POSITIVE";
+    auto metric =
+        (is_empty ? AutofillMetrics::FALSE_POSITIVE_EMPTY
+                  : (is_ambiguous ? AutofillMetrics::FALSE_POSITIVE_AMBIGUOUS
+                                  : AutofillMetrics::FALSE_POSITIVE_UNKNOWN));
+    LogUMAHistogramEnumeration(aggregate_histogram, metric,
+                               AutofillMetrics::NUM_FIELD_TYPE_QUALITY_METRICS);
+    LogUMAHistogramEnumeration(type_specific_histogram,
+                               GetFieldTypeGroupMetric(predicted_type, metric),
+                               KMaxFieldTypeGroupMetric);
+    return;
+  }
+
+  // Note: At this point predicted_type != actual type, actual_type != UNKNOWN.
+  // If predicted type is UNKNOWN_TYPE then the prediction is a false negative
+  // unknown.
+  if (predicted_type == UNKNOWN_TYPE) {
+    DVLOG(2) << "FALSE NEGATIVE";
+    LogUMAHistogramEnumeration(aggregate_histogram,
+                               AutofillMetrics::FALSE_NEGATIVE_UNKNOWN,
+                               AutofillMetrics::NUM_FIELD_TYPE_QUALITY_METRICS);
+    LogUMAHistogramEnumeration(
+        type_specific_histogram,
+        GetFieldTypeGroupMetric(actual_type,
+                                AutofillMetrics::FALSE_NEGATIVE_UNKNOWN),
+        KMaxFieldTypeGroupMetric);
+    return;
+  }
+
+  DVLOG(2) << "MISMATCH";
+
+  // Note: At this point predicted_type != actual type, actual_type != UNKNOWN,
+  //       predicted_type != UNKNOWN.
+  // This is a mismatch. From the reference of the actual type, this is a false
+  // negative (it was T, but predicted U). From the reference of the prediction,
+  // this is a false positive (predicted it was T, but it was U).
+  LogUMAHistogramEnumeration(aggregate_histogram,
+                             AutofillMetrics::FALSE_NEGATIVE_MISMATCH,
                              AutofillMetrics::NUM_FIELD_TYPE_QUALITY_METRICS);
-
-  int field_type_group_metric = GetFieldTypeGroupMetric(field_type, metric);
-  int num_field_type_group_metrics =
-      AutofillMetrics::NUM_FIELD_TYPE_QUALITY_METRICS *
-      NUM_FIELD_TYPE_GROUPS_FOR_METRICS;
-  LogUMAHistogramEnumeration(base_name + ".ByFieldType" + suffix,
-                             field_type_group_metric,
-                             num_field_type_group_metrics);
+  LogUMAHistogramEnumeration(
+      type_specific_histogram,
+      GetFieldTypeGroupMetric(actual_type,
+                              AutofillMetrics::FALSE_NEGATIVE_MISMATCH),
+      KMaxFieldTypeGroupMetric);
+  LogUMAHistogramEnumeration(
+      type_specific_histogram,
+      GetFieldTypeGroupMetric(predicted_type,
+                              AutofillMetrics::FALSE_POSITIVE_MISMATCH),
+      KMaxFieldTypeGroupMetric);
 }
 
 }  // namespace
 
 // static
 void AutofillMetrics::LogCardUploadDecisionMetrics(
     int upload_decision_metrics) {
   DCHECK(upload_decision_metrics);
   DCHECK_LT(upload_decision_metrics, 1 << kNumCardUploadDecisionMetrics);
 
@@ skipping 178 matching lines @@
 }
 
 // static
 void AutofillMetrics::LogDeveloperEngagementMetric(
     DeveloperEngagementMetric metric) {
   DCHECK_LT(metric, NUM_DEVELOPER_ENGAGEMENT_METRICS);
   UMA_HISTOGRAM_ENUMERATION("Autofill.DeveloperEngagement", metric,
                             NUM_DEVELOPER_ENGAGEMENT_METRICS);
 }
 
-// static
-void AutofillMetrics::LogHeuristicTypePrediction(
-    FieldTypeQualityMetric metric,
-    ServerFieldType field_type,
-    QualityMetricType metric_type) {
-  LogTypeQualityMetric("Autofill.Quality.HeuristicType", metric, field_type,
-                       metric_type);
+void AutofillMetrics::LogHeuristicPredictionQualityMetrics(
+    const ServerFieldTypeSet& possible_types,
+    ServerFieldType predicted_type,
+    AutofillMetrics::QualityMetricType metric_type) {
+  LogPredictionQualityMetrics("Heuristic", possible_types, predicted_type,
+                              metric_type);
+}
+
+void AutofillMetrics::LogServerPredictionQualityMetrics(
+    const ServerFieldTypeSet& possible_types,
+    ServerFieldType predicted_type,
+    AutofillMetrics::QualityMetricType metric_type) {
+  LogPredictionQualityMetrics("Server", possible_types, predicted_type,
+                              metric_type);
+}
+
+void AutofillMetrics::LogOverallPredictionQualityMetrics(
+    const ServerFieldTypeSet& possible_types,
+    ServerFieldType predicted_type,
+    AutofillMetrics::QualityMetricType metric_type) {
+  LogPredictionQualityMetrics("Overall", possible_types, predicted_type,
+                              metric_type);
 }
 
 // static
-void AutofillMetrics::LogOverallTypePrediction(FieldTypeQualityMetric metric,
-                                               ServerFieldType field_type,
-                                               QualityMetricType metric_type) {
-  LogTypeQualityMetric("Autofill.Quality.PredictedType", metric, field_type,
-                       metric_type);
-}
-
-// static
-void AutofillMetrics::LogServerTypePrediction(FieldTypeQualityMetric metric,
-                                              ServerFieldType field_type,
-                                              QualityMetricType metric_type) {
-  LogTypeQualityMetric("Autofill.Quality.ServerType", metric, field_type,
-                       metric_type);
-}
-
-// static
 void AutofillMetrics::LogServerQueryMetric(ServerQueryMetric metric) {
   DCHECK_LT(metric, NUM_SERVER_QUERY_METRICS);
   UMA_HISTOGRAM_ENUMERATION("Autofill.ServerQueryResponse", metric,
                             NUM_SERVER_QUERY_METRICS);
 }
 
 // static
 void AutofillMetrics::LogUserHappinessMetric(UserHappinessMetric metric) {
   DCHECK_LT(metric, NUM_USER_HAPPINESS_METRICS);
   UMA_HISTOGRAM_ENUMERATION("Autofill.UserHappiness", metric,
@@ skipping 642 matching lines @@
   DCHECK(!form_parsed_timestamp_.is_null());
   return (base::TimeTicks::Now() - form_parsed_timestamp_).InMilliseconds();
 }
 
 void AutofillMetrics::FormInteractionsUkmLogger::GetNewSourceID() {
   source_id_ = ukm_recorder_->GetNewSourceID();
   ukm_recorder_->UpdateSourceURL(source_id_, url_);
 }
 
 }  // namespace autofill