Obtain confidence score based off last known good and first known bad revision results.

tools/auto_bisect/bisect_results.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import math
 import os
 
 import bisect_utils
 import math_utils
 import source_control
@@ skipping 211 matching lines @@
               bad_greater_than_good else not prev_greater_than_current)
 
           # Only report potential regressions with high confidence.
           if is_same_direction and confidence > 50:
             other_regressions.append([revision_state, prev_state, confidence])
         previous_values.append(current_values)
         prev_state = revision_state
     return other_regressions
 
   @staticmethod
   def FindBreakingRevRange(revision_states):

[qyearsley, 2015/01/16 18:37:03]

I think it's probably worth it to add a test for this function, even though it's
probably at least covered by the other tests methods.

[RobertoCN, 2015/01/30 21:23:32]

Done.

+    """Finds the last known good and first known bad revisions.
+
+    Note that since revision_states is expected to be in reverse cronological

[RobertoCN, 2015/01/14 18:50:47]

chronological not cronological.

[RobertoCN, 2015/01/30 21:23:32]

Done.

+    order, the last known good revision is the first revision in the list that
+    has the passed property set to 1, therefore the name
+    `first_working_revision`. The inverse applies to `last_broken_revision`.\

[RobertoCN, 2015/01/14 18:50:47]

Unnecessary backlash at the end of the line.(typo)

[RobertoCN, 2015/01/30 21:23:32]

Done.

+    """

[qyearsley, 2015/01/16 18:37:03]

[Optional] You could add Args and Returns sections to this docstring which
explicitly says that the input is a list of RevisionState and the output is a
pair of RevisionState.

[RobertoCN, 2015/01/30 21:23:32]

Done.

     first_working_revision = None
     last_broken_revision = None

[qyearsley, 2015/01/16 18:37:03]

[Optional] It might be less confusing to reverse the list so that it's in
chronological order, and call these last_good_rev and first_bad_rev. This could
be done in a separate CL, changing the other usages as well, and making the
revision states list chronological order.

[RobertoCN, 2015/01/30 21:23:32]

Acknowledged.

 
     for revision_state in revision_states:
       if revision_state.passed == 1 and not first_working_revision:
         first_working_revision = revision_state
 
       if not revision_state.passed:
         last_broken_revision = revision_state
 
     return first_working_revision, last_broken_revision
 
@@ skipping 35 matching lines @@
 
     regression_size = 100 * math_utils.RelativeChange(mean_of_good_runs,
                                                       mean_of_bad_runs)
     if math.isnan(regression_size):
       regression_size = 'zero-to-nonzero'
 
     regression_std_err = math.fabs(math_utils.PooledStandardError(
         [working_mean, broken_mean]) /
         max(0.0001, min(mean_of_good_runs, mean_of_bad_runs))) * 100.0
 
-    # Give a "confidence" in the bisect. At the moment we use how distinct the
-    # values are before and after the last broken revision, and how noisy the
-    # overall graph is.
-    confidence_params = (sum(working_means, []), sum(broken_means, []))
+    # Give a "confidence" in the bisect. Currently, we consider the values of
+    # only the revisions at the breaking range (last known good and first known
+    # bad) see the note in the docstring for FindBreakingRange.
+    confidence_params = (
+        sum([first_working_rev.value['values']], []),
+        sum([last_broken_rev.value['values']], [])
+    )
     confidence = cls.ConfidenceScore(*confidence_params)
 
     bad_greater_than_good = mean_of_bad_runs > mean_of_good_runs
 
     return {'regression_size': regression_size,
             'regression_std_err': regression_std_err,
             'confidence': confidence,
             'bad_greater_than_good': bad_greater_than_good}