Requiring confidence in initial regression range before bisecting.

tools/auto_bisect/bisect_perf_regression.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Performance Test Bisect Tool
 
 This script bisects a series of changelists using binary search. It starts at
 a bad revision where a performance metric has regressed, and asks for a last
 known-good revision. It will then binary search across this revision range by
@@ skipping 32 matching lines @@
 import shutil
 import StringIO
 import sys
 import time
 import zipfile
 
 sys.path.append(os.path.join(
     os.path.dirname(__file__), os.path.pardir, 'telemetry'))
 
 from bisect_results import BisectResults
+from bisect_results import ConfidenceScore
 import bisect_utils
 import builder
 import math_utils
 import request_build
 import source_control
 from telemetry.util import cloud_storage
 
 # Below is the map of "depot" names to information about each depot. Each depot
 # is a repository, and in the process of bisecting, revision ranges in these
 # repositories may also be bisected.
@@ skipping 99 matching lines @@
 
 # Maximum time in seconds to wait after posting build request to the try server.
 # TODO: Change these values based on the actual time taken by buildbots on
 # the try server.
 MAX_MAC_BUILD_TIME = 14400
 MAX_WIN_BUILD_TIME = 14400
 MAX_LINUX_BUILD_TIME = 14400
 
 # The percentage at which confidence is considered high.
 HIGH_CONFIDENCE = 95
+# The confidence percentage we require to consider the initial range a
+# regression based on the test results of the inital good and bad revisions.
+REGRESSION_CONFIDENCE = 95
 
 # Patch template to add a new file, DEPS.sha under src folder.
 # This file contains SHA1 value of the DEPS changes made while bisecting
 # dependency repositories. This patch send along with DEPS patch to try server.
 # When a build requested is posted with a patch, bisect builders on try server,
 # once build is produced, it reads SHA value from this file and appends it
 # to build archive filename.
 DEPS_SHA_PATCH = """diff --git DEPS.sha DEPS.sha
 new file mode 100644
 --- /dev/null
@@ skipping 2282 matching lines @@
         else:
           message += "and the metric appears to have decreased. "
         if ((higher_is_better and metric_increased) or
             (not higher_is_better and not metric_increased)):
           results.error = (message + 'Then, the test results for the ends of '
                            'the given \'good\' - \'bad\' range of revisions '
                            'represent an improvement (and not a regression).')
           return results
         print message, "Therefore we continue to bisect."
 
+      # Check how likely it is that the good and bad results are different
+      # beyond chance-induced variation.
+      if not self.opts.debug_ignore_regression_confidence:
+        regression_confidence = ConfidenceScore(known_bad_value['values'],
+                                                known_good_value['values'])
+        if regression_confidence < REGRESSION_CONFIDENCE:
+          results.error = ('We could not reproduce the regression with this '
+                           'test/metric/platform combination with enough '
+                           'confidence. There\'s still a chance that this is '
+                           'actually a regression, but you may need to bisect '
+                           'a different platform.')
+          return results
+
       # Can just mark the good and bad revisions explicitly here since we
       # already know the results.
       bad_revision_data = revision_data[revision_list[0]]
       bad_revision_data['external'] = bad_results[2]
       bad_revision_data['perf_time'] = bad_results[3]
       bad_revision_data['build_time'] = bad_results[4]
       bad_revision_data['passed'] = False
       bad_revision_data['value'] = known_bad_value
 
       good_revision_data = revision_data[revision_list[max_revision]]
@@ skipping 477 matching lines @@
     self.max_time_minutes = 20
     self.metric = None
     self.command = None
     self.output_buildbot_annotations = None
     self.no_custom_deps = False
     self.working_directory = None
     self.extra_src = None
     self.debug_ignore_build = None
     self.debug_ignore_sync = None
     self.debug_ignore_perf_test = None
+    self.debug_ignore_regression_confidence = None
     self.debug_fake_first_test_mean = 0
     self.gs_bucket = None
     self.target_arch = 'ia32'
     self.target_build_type = 'Release'
     self.builder_host = None
     self.builder_port = None
     self.bisect_mode = BISECT_MODE_MEAN
     self.improvement_direction = 0
 
   @staticmethod
@@ skipping 147 matching lines @@
     group = optparse.OptionGroup(parser, 'Debug options')
     group.add_option('--debug_ignore_build',
                      action='store_true',
                      help='DEBUG: Don\'t perform builds.')
     group.add_option('--debug_ignore_sync',
                      action='store_true',
                      help='DEBUG: Don\'t perform syncs.')
     group.add_option('--debug_ignore_perf_test',
                      action='store_true',
                      help='DEBUG: Don\'t perform performance tests.')
+    group.add_option('--debug_ignore_regression_confidence',
+                     action='store_true',
+                     help='DEBUG: Don\'t score the confidence of the initial '
+                          'good and bad revisions\' test results.')
     group.add_option('--debug_fake_first_test_mean',
                      type='int',
                      default='0',
                      help=('DEBUG: When faking performance tests, return this '
                            'value as the mean of the first performance test, '
                            'and return a mean of 0.0 for further tests.'))
     parser.add_option_group(group)
     return parser
 
   def ParseCommandLine(self):
@@ skipping 141 matching lines @@
       # bugs. If you change this, please update the perf dashboard as well.
       bisect_utils.OutputAnnotationStepStart('Results')
     print 'Error: %s' % e.message
     if opts.output_buildbot_annotations:
       bisect_utils.OutputAnnotationStepClosed()
   return 1
 
 
 if __name__ == '__main__':
   sys.exit(main())