Return 0 for confidence when there's only results for one "good" or one "bad" rev.

tools/bisect-perf-regression.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Performance Test Bisect Tool
 
 This script bisects a series of changelists using binary search. It starts at
 a bad revision where a performance metric has regressed, and asks for a last
 known-good revision. It will then binary search across this revision range by
@@ skipping 153 matching lines @@
 BUILD_RESULT_FAIL = 1
 BUILD_RESULT_SKIPPED = 2
 
 # Maximum time in seconds to wait after posting build request to tryserver.
 # TODO: Change these values based on the actual time taken by buildbots on
 # the tryserver.
 MAX_MAC_BUILD_TIME = 14400
 MAX_WIN_BUILD_TIME = 14400
 MAX_LINUX_BUILD_TIME = 14400
 
+# The confidence percentage at which confidence can be consider "high".
+HIGH_CONFIDENCE = 95
+
 # Patch template to add a new file, DEPS.sha under src folder.
 # This file contains SHA1 value of the DEPS changes made while bisecting
 # dependency repositories. This patch send along with DEPS patch to tryserver.
 # When a build requested is posted with a patch, bisect builders on tryserver,
 # once build is produced, it reads SHA value from this file and appends it
 # to build archive filename.
 DEPS_SHA_PATCH = """diff --git src/DEPS.sha src/DEPS.sha
 new file mode 100644
 --- /dev/null
 +++ src/DEPS.sha
 @@ -0,0 +1 @@
 +%(deps_sha)s
 """
 
 # The possible values of the --bisect_mode flag, which determines what to
 # use when classifying a revision as "good" or "bad".
 BISECT_MODE_MEAN = 'mean'
 BISECT_MODE_STD_DEV = 'std_dev'
 BISECT_MODE_RETURN_CODE = 'return_code'
 
-# The perf dashboard specifically looks for the string
-# "Estimated Confidence: 95%" to decide whether or not to cc the author(s).
-# If you change this, please update the perf dashboard as well.
+# The perf dashboard looks for a string like "Estimated Confidence: 95%"
+# to decide whether or not to cc the author(s). If you change this, please
+# update the perf dashboard as well.
 RESULTS_BANNER = """
 ===== BISECT JOB RESULTS =====
 Status: %(status)s
 
 Test Command: %(command)s
 Test Metric: %(metrics)s
 Relative Change: %(change)s
 Estimated Confidence: %(confidence)d%%"""
 
 # The perf dashboard specifically looks for the string
@@ skipping 66 matching lines @@
   their differences aren't due to chance alone.
 
 
   Args:
     good_results_lists: A list of lists of "good" result numbers.
     bad_results_lists: A list of lists of "bad" result numbers.
 
   Returns:
     A number in the range [0, 100].
   """
-  if not good_results_lists or not bad_results_lists:
+  # If there's only one item in either list, this means only one revision was
+  # classified good or bad; this isn't good enough evidence to make a decision.
+  # If an empty list was passed, that also implies zero confidence.
+  if len(good_results_lists) <= 1 or len(bad_results_lists) <= 1:
     return 0.0
 
   # Flatten the lists of results lists.
   sample1 = sum(good_results_lists, [])
   sample2 = sum(bad_results_lists, [])
+
+  # If there were only empty lists in either of the lists (this is unexpected
+  # and normally shouldn't happen), then we also want to return 0.
   if not sample1 or not sample2:
     return 0.0
 
   # The p-value is approximately the probability of obtaining the given set
   # of good and bad values just by chance.
   _, _, p_value = ttest.WelchsTTest(sample1, sample2)
   return 100.0 * (1.0 - p_value)
 
 
 def GetSHA1HexDigest(contents):
@@ skipping 2583 matching lines @@
 
     self._PrintTestedCommitsTable(revision_data_sorted,
                                   results_dict['first_working_revision'],
                                   results_dict['last_broken_revision'],
                                   100, final_step=False)
 
   def _ConfidenceLevelStatus(self, results_dict):
     if not results_dict['confidence']:
       return None
     confidence_status = 'Successful with %(level)s confidence%(warning)s.'
-    if results_dict['confidence'] >= 95:
+    if results_dict['confidence'] >= HIGH_CONFIDENCE:
       level = 'high'
     else:
       level = 'low'
     warning = ' and warnings'
     if not self.warnings:
       warning = ''
     return confidence_status % {'level': level, 'warning': warning}
 
   def _GetViewVCLinkFromDepotAndHash(self, cl, depot):
     info = self.source_control.QueryRevisionInfo(cl,
@@ skipping 263 matching lines @@
         'confidence': confidence,
     }
 
   def _CheckForWarnings(self, results_dict):
     if len(results_dict['culprit_revisions']) > 1:
       self.warnings.append('Due to build errors, regression range could '
                            'not be narrowed down to a single commit.')
     if self.opts.repeat_test_count == 1:
       self.warnings.append('Tests were only set to run once. This may '
                            'be insufficient to get meaningful results.')
-    if results_dict['confidence'] < 100:
-      if results_dict['confidence']:
-        self.warnings.append(
-            'Confidence is less than 100%. There could be other candidates '
-            'for this regression. Try bisecting again with increased '
-            'repeat_count or on a sub-metric that shows the regression more '
-            'clearly.')
-      else:
-        self.warnings.append(
-          'Confidence is 0%. Try bisecting again on another platform, with '
-          'increased repeat_count or on a sub-metric that shows the '
-          'regression more clearly.')
+    if 0 < results_dict['confidence'] < HIGH_CONFIDENCE:
+      self.warnings.append('Confidence is not high. Try bisecting again '
+                           'with increased repeat_count, larger range, or '
+                           'on another metric.')
+    if not results_dict['confidence']:
+      self.warnings.append('Confidence score is 0%. Try bisecting again on '
+                           'another platform or another metric.')
 
   def FormatAndPrintResults(self, bisect_results):
     """Prints the results from a bisection run in a readable format.
 
     Args:
       bisect_results: The results from a bisection test run.
     """
     revision_data = bisect_results['revision_data']
     revision_data_sorted = sorted(revision_data.iteritems(),
                                   key = lambda x: x[1]['sort'])
@@ skipping 460 matching lines @@
       # bugs. If you change this, please update the perf dashboard as well.
       bisect_utils.OutputAnnotationStepStart('Results')
     print 'Error: %s' % e.message
     if opts.output_buildbot_annotations:
       bisect_utils.OutputAnnotationStepClosed()
   return 1
 
 
 if __name__ == '__main__':
   sys.exit(main())