Log Autofill.Quality.*.ByFieldType by predicted type for unknown data.

components/autofill/core/browser/autofill_metrics.cc

Index: components/autofill/core/browser/autofill_metrics.cc
diff --git a/components/autofill/core/browser/autofill_metrics.cc b/components/autofill/core/browser/autofill_metrics.cc
index e631f1dfd66e84505ebb6d5659c2ea3614b73846..ea0467d2d5720fdcec3c9ecc76535aea4f798717 100644
--- a/components/autofill/core/browser/autofill_metrics.cc
+++ b/components/autofill/core/browser/autofill_metrics.cc
@@ -80,6 +80,10 @@ enum FieldTypeGroupForMetrics {
   NUM_FIELD_TYPE_GROUPS_FOR_METRICS
 };
 
+const int KMaxFieldTypeGroupMetric =
+    NUM_FIELD_TYPE_GROUPS_FOR_METRICS *
+    AutofillMetrics::NUM_FIELD_TYPE_QUALITY_METRICS;
+
 std::string PreviousSaveCreditCardPromptUserDecisionToString(
     int previous_save_credit_card_prompt_user_decision) {
   DCHECK_LT(previous_save_credit_card_prompt_user_decision,
@@ -268,40 +272,171 @@ void LogUMAHistogramLongTimes(const std::string& name,
   histogram->AddTime(duration);
 }
 
+const char* GetQualityMetricTypeSuffix(
+    AutofillMetrics::QualityMetricType metric_type) {
+  switch (metric_type) {
+    default:
+      NOTREACHED();
+    // Fall through...
+
+    case AutofillMetrics::TYPE_SUBMISSION:
+      return "";
+    case AutofillMetrics::TYPE_NO_SUBMISSION:
+      return ".NoSubmission";
+    case AutofillMetrics::TYPE_AUTOCOMPLETE_BASED:
+      return ".BasedOnAutocomplete";
+  }
+}
+
+ServerFieldType GetFieldType(const ServerFieldTypeSet field_types,

[Mathieu, 2017/05/29 14:00:19]

GetFieldType is a little vague, can we find a more specific name?

[Roger McFarlane (Chromium), 2017/05/30 21:43:22]

Done.

+                             ServerFieldType predicted_type) {
+  if (field_types.count(predicted_type))
+    return predicted_type;
+
+  if (field_types.count(EMPTY_TYPE))
+    return EMPTY_TYPE;
+
+  if (field_types.count(UNKNOWN_TYPE))
+    return UNKNOWN_TYPE;
+
+  // Collapse field types that Chrome treats as identical, e.g. home and
+  // billing address fields.
+  ServerFieldTypeSet collapsed_field_types;
+  for (const auto& type : field_types) {
+    // Since we currently only support US phone numbers, the (city code + main
+    // digits) number is almost always identical to the whole phone number.
+    // TODO(isherman): Improve this logic once we add support for

[Mathieu, 2017/05/29 14:00:19]

hmm...

[Roger McFarlane (Chromium), 2017/05/30 21:43:22]

Indeed. It's not clear to me that we need to do this collapsing thing at all. I
wasn't intending to change the counting behaviour in this CL, so I retained the
original logic; but, this function could probably be reduced to returning the
first item if there's only one, else predicted_type if it's in the set, else
UNKNOWN_TYPE.

+    // international numbers.
+    if (type == PHONE_HOME_CITY_AND_NUMBER)
+      collapsed_field_types.insert(PHONE_HOME_WHOLE_NUMBER);
+    else
+      collapsed_field_types.insert(AutofillType(type).GetStorableType());
+  }
+
+  // Capture the field's type, if it is unambiguous.
+  ServerFieldType actual_type = UNKNOWN_TYPE;
+  if (collapsed_field_types.size() == 1)
+    actual_type = *collapsed_field_types.begin();
+
+  DVLOG(2) << "Inferred Type: " << AutofillType(actual_type).ToString();
+  return actual_type;
+}
+
 // Logs a type quality metric.  The primary histogram name is constructed based
 // on |base_name|.  The field-specific histogram name also factors in the
 // |field_type|.  Logs a sample of |metric|, which should be in the range
 // [0, |num_possible_metrics|). May log a suffixed version of the metric
 // depending on |metric_type|.
-void LogTypeQualityMetric(const std::string& base_name,
-                          AutofillMetrics::FieldTypeQualityMetric metric,
-                          ServerFieldType field_type,
-                          AutofillMetrics::QualityMetricType metric_type) {
-  DCHECK_LT(metric, AutofillMetrics::NUM_FIELD_TYPE_QUALITY_METRICS);
+AutofillMetrics::FieldTypeQualityMetric LogTypeQualityMetric(
+    const std::string& base_name,
+    ServerFieldTypeSet possible_types,
+    ServerFieldType predicted_type,
+    AutofillMetrics::QualityMetricType metric_type) {
+  // NO_SERVER_DATA is the equivalent of predicting UNKNOWN.
+  if (predicted_type == NO_SERVER_DATA)
+    predicted_type = UNKNOWN_TYPE;
+
+  // Get the best type classification we can for the field. EMPTY is the same
+  // as UNKNOWN for our purposes, but remember whether or not it was empty for
+  // later.
+  ServerFieldType actual_type = GetFieldType(possible_types, predicted_type);

[Mathieu, 2017/05/29 14:00:19]

GetActualFieldType perhaps?

[Roger McFarlane (Chromium), 2017/05/30 21:43:22]

Done.

+  bool is_empty = (actual_type == EMPTY_TYPE);
+  if (is_empty)
+    actual_type = UNKNOWN_TYPE;
+
+  // Capture the aggregate quality metric.
+  const char* const suffix = GetQualityMetricTypeSuffix(metric_type);
+  std::string aggregate_histogram = base_name + suffix;
+  std::string type_specific_histogram = base_name + ".ByFieldType" + suffix;
+
+  DVLOG(2) << "Predicted: " << AutofillType(predicted_type).ToString() << "; "
+           << "Actual: " << AutofillType(actual_type).ToString();
+
+  // If the predicted and actual types match then it's either a true positive
+  // or a true negative (if they are both unknown). Do not log type specific
+  // true negatives (instead log a true positive for the "Ambiguous" type).
+  if (predicted_type == actual_type) {
+    if (actual_type == UNKNOWN_TYPE) {
+      // Only log aggregate true negative; do not log type specific metrics
+      // for UNKNOWN/EMPTY.
+      DVLOG(2) << "TRUE NEGATIVE";
+      auto empty_or_unknown = is_empty ? AutofillMetrics::TRUE_NEGATIVE_EMPTY
+                                       : AutofillMetrics::TRUE_NEGATIVE_UNKNOWN;
+      LogUMAHistogramEnumeration(
+          aggregate_histogram, empty_or_unknown,
+          AutofillMetrics::NUM_FIELD_TYPE_QUALITY_METRICS);
+      return empty_or_unknown;
+    }
 
-  std::string suffix;
-  switch (metric_type) {
-    case AutofillMetrics::TYPE_SUBMISSION:
-      break;
-    case AutofillMetrics::TYPE_NO_SUBMISSION:
-      suffix = ".NoSubmission";
-      break;
-    case AutofillMetrics::TYPE_AUTOCOMPLETE_BASED:
-      suffix = ".BasedOnAutocomplete";
-      break;
-    default:
-      NOTREACHED();
+    DVLOG(2) << "TRUE POSITIVE";
+    // Log both aggregate and type specific true positive if we correctly
+    // predict that type with which the field was filled.
+    LogUMAHistogramEnumeration(aggregate_histogram,
+                               AutofillMetrics::TRUE_POSITIVE,
+                               AutofillMetrics::NUM_FIELD_TYPE_QUALITY_METRICS);
+    LogUMAHistogramEnumeration(
+        type_specific_histogram,
+        GetFieldTypeGroupMetric(actual_type, AutofillMetrics::TRUE_POSITIVE),
+        KMaxFieldTypeGroupMetric);
+    return AutofillMetrics::TRUE_POSITIVE;
   }
-  LogUMAHistogramEnumeration(base_name + suffix, metric,
+
+  // Note: At this point predicted_type != actual type
+  // If actual type is UNKNOWN_TYPE then the prediction is a false positive.
+  // Further specialize the type of false positive by whether the field was
+  // empty or contained an unknown value.
+  if (actual_type == UNKNOWN_TYPE) {
+    DVLOG(2) << "FALSE POSITIVE";
+    auto empty_or_unknown = is_empty ? AutofillMetrics::FALSE_POSITIVE_EMPTY
+                                     : AutofillMetrics::FALSE_POSITIVE_UNKNOWN;
+    LogUMAHistogramEnumeration(aggregate_histogram, empty_or_unknown,
+                               AutofillMetrics::NUM_FIELD_TYPE_QUALITY_METRICS);
+    LogUMAHistogramEnumeration(
+        type_specific_histogram,
+        GetFieldTypeGroupMetric(predicted_type, empty_or_unknown),
+        KMaxFieldTypeGroupMetric);
+    return empty_or_unknown;
+  }
+
+  // Note: At this point predicted_type != actual type, actual_type != UNKNOWN.
+  // If predicted type is UNKNOWN_TYPE then the prediction is a false negative
+  // unknown.
+  if (predicted_type == UNKNOWN_TYPE) {
+    DVLOG(2) << "FALSE NEGATIVE";
+    LogUMAHistogramEnumeration(aggregate_histogram,
+                               AutofillMetrics::FALSE_NEGATIVE_UNKNOWN,
+                               AutofillMetrics::NUM_FIELD_TYPE_QUALITY_METRICS);
+    LogUMAHistogramEnumeration(
+        type_specific_histogram,
+        GetFieldTypeGroupMetric(actual_type,
+                                AutofillMetrics::FALSE_NEGATIVE_UNKNOWN),
+        KMaxFieldTypeGroupMetric);
+    return AutofillMetrics::FALSE_NEGATIVE_UNKNOWN;
+  }
+
+  DVLOG(2) << "MISMATCH";
+
+  // Note: At this point predicted_type != actual type, actual_type != UNKNOWN,
+  //       predicted_type != UNKNOWN.
+  // This is a mismatch. From the reference of the actual type, this is a false
+  // negative (it was T, but predicted U). From the reference of the prediction,
+  // this is a false positive (predicted it was T, but it was U).
+  LogUMAHistogramEnumeration(aggregate_histogram,
+                             AutofillMetrics::FALSE_NEGATIVE_MISMATCH,
                              AutofillMetrics::NUM_FIELD_TYPE_QUALITY_METRICS);
+  LogUMAHistogramEnumeration(

[Mathieu, 2017/05/29 14:00:19]

Is it weird two log the same histograms for both reference points (actual and
predicted)? Will it be harder to analyze?

[Roger McFarlane (Chromium), 2017/05/30 21:43:22]

It's impossible to do some analyses if you don't (because these histograms lose
information, retaining either actual or predicted, but not both).

+      type_specific_histogram,
+      GetFieldTypeGroupMetric(actual_type,
+                              AutofillMetrics::FALSE_NEGATIVE_MISMATCH),
+      KMaxFieldTypeGroupMetric);
+  LogUMAHistogramEnumeration(
+      type_specific_histogram,
+      GetFieldTypeGroupMetric(predicted_type,
+                              AutofillMetrics::FALSE_POSITIVE_MISMATCH),
+      KMaxFieldTypeGroupMetric);
 
-  int field_type_group_metric = GetFieldTypeGroupMetric(field_type, metric);
-  int num_field_type_group_metrics =
-      AutofillMetrics::NUM_FIELD_TYPE_QUALITY_METRICS *
-      NUM_FIELD_TYPE_GROUPS_FOR_METRICS;
-  LogUMAHistogramEnumeration(base_name + ".ByFieldType" + suffix,
-                             field_type_group_metric,
-                             num_field_type_group_metrics);
+  // Return the aggregate finding.
+  return AutofillMetrics::FALSE_NEGATIVE_MISMATCH;
 }
 
 }  // namespace
@@ -502,27 +637,28 @@ void AutofillMetrics::LogDeveloperEngagementMetric(
 
 // static
 void AutofillMetrics::LogHeuristicTypePrediction(
-    FieldTypeQualityMetric metric,
-    ServerFieldType field_type,
+    ServerFieldTypeSet possible_types,
+    ServerFieldType predicted_type,
     QualityMetricType metric_type) {
-  LogTypeQualityMetric("Autofill.Quality.HeuristicType", metric, field_type,
-                       metric_type);
+  LogTypeQualityMetric("Autofill.Quality.HeuristicType", possible_types,
+                       predicted_type, metric_type);
 }
 
 // static
-void AutofillMetrics::LogOverallTypePrediction(FieldTypeQualityMetric metric,
-                                               ServerFieldType field_type,
-                                               QualityMetricType metric_type) {
-  LogTypeQualityMetric("Autofill.Quality.PredictedType", metric, field_type,
-                       metric_type);
+void AutofillMetrics::LogOverallTypePrediction(
+    ServerFieldTypeSet possible_types,
+    ServerFieldType predicted_type,
+    QualityMetricType metric_type) {
+  LogTypeQualityMetric("Autofill.Quality.PredictedType", possible_types,
+                       predicted_type, metric_type);
 }
 
 // static
-void AutofillMetrics::LogServerTypePrediction(FieldTypeQualityMetric metric,
-                                              ServerFieldType field_type,
+void AutofillMetrics::LogServerTypePrediction(ServerFieldTypeSet possible_types,
+                                              ServerFieldType predicted_type,
                                               QualityMetricType metric_type) {
-  LogTypeQualityMetric("Autofill.Quality.ServerType", metric, field_type,
-                       metric_type);
+  LogTypeQualityMetric("Autofill.Quality.ServerType", possible_types,
+                       predicted_type, metric_type);
 }
 
 // static