Obtain confidence score based off last known good and first known bad revision results.

tools/auto_bisect/bisect_results.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import math
 import os
 
 import bisect_utils
 import math_utils
 import source_control
@@ skipping 212 matching lines @@
 
           # Only report potential regressions with high confidence.
           if is_same_direction and confidence > 50:
             other_regressions.append([revision_state, prev_state, confidence])
         previous_values.append(current_values)
         prev_state = revision_state
     return other_regressions
 
   @staticmethod
   def FindBreakingRevRange(revision_states):
+    """Finds the last known good and first known bad revisions.
+
+    Note that since revision_states is expected to be in reverse chronological
+    order, the last known good revision is the first revision in the list that
+    has the passed property set to 1, therefore the name
+    `first_working_revision`. The inverse applies to `last_broken_revision`.
+
+    Args:
+      revision_states: A list of RevisionState instances.
+
+    Returns:
+      A tuple containing the two revision states at the border. (Last
+      known good and first known bad.)
+    """
     first_working_revision = None
     last_broken_revision = None
 
     for revision_state in revision_states:
       if revision_state.passed == 1 and not first_working_revision:
         first_working_revision = revision_state
 
       if not revision_state.passed:
         last_broken_revision = revision_state
 
@@ skipping 37 matching lines @@
 
     regression_size = 100 * math_utils.RelativeChange(mean_of_good_runs,
                                                       mean_of_bad_runs)
     if math.isnan(regression_size):
       regression_size = 'zero-to-nonzero'
 
     regression_std_err = math.fabs(math_utils.PooledStandardError(
         [working_mean, broken_mean]) /
         max(0.0001, min(mean_of_good_runs, mean_of_bad_runs))) * 100.0
 
-    # Give a "confidence" in the bisect. At the moment we use how distinct the
-    # values are before and after the last broken revision, and how noisy the
-    # overall graph is.
-    confidence_params = (sum(working_means, []), sum(broken_means, []))
+    # Give a "confidence" in the bisect. Currently, we consider the values of
+    # only the revisions at the breaking range (last known good and first known
+    # bad) see the note in the docstring for FindBreakingRange.
+    confidence_params = (
+        sum([first_working_rev.value['values']], []),
+        sum([last_broken_rev.value['values']], [])
+    )
     confidence = cls.ConfidenceScore(*confidence_params)
 
     bad_greater_than_good = mean_of_bad_runs > mean_of_good_runs
 
     return {'regression_size': regression_size,
             'regression_std_err': regression_std_err,
             'confidence': confidence,
             'bad_greater_than_good': bad_greater_than_good}