Re-applying reverted changes for regression confidence check + fix: ConfidenceScoretakes flat lists

tools/auto_bisect/bisect_perf_regression.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Performance Test Bisect Tool
 
 This script bisects a series of changelists using binary search. It starts at
 a bad revision where a performance metric has regressed, and asks for a last
 known-good revision. It will then binary search across this revision range by
@@ skipping 57 matching lines @@
 BUILD_RESULT_FAIL = 1
 BUILD_RESULT_SKIPPED = 2
 
 # Maximum time in seconds to wait after posting build request to the try server.
 # TODO: Change these values based on the actual time taken by buildbots on
 # the try server.
 MAX_MAC_BUILD_TIME = 14400
 MAX_WIN_BUILD_TIME = 14400
 MAX_LINUX_BUILD_TIME = 14400
 
+# The confidence percentage we require to consider the initial range a
+# regression based on the test results of the inital good and bad revisions.
+REGRESSION_CONFIDENCE = 95
+
 # Patch template to add a new file, DEPS.sha under src folder.
 # This file contains SHA1 value of the DEPS changes made while bisecting
 # dependency repositories. This patch send along with DEPS patch to try server.
 # When a build requested is posted with a patch, bisect builders on try server,
 # once build is produced, it reads SHA value from this file and appends it
 # to build archive filename.
 DEPS_SHA_PATCH = """diff --git DEPS.sha DEPS.sha
 new file mode 100644
 --- /dev/null
 +++ DEPS.sha
 @@ -0,0 +1 @@
 +%(deps_sha)s
 """
 
+REGRESSION_CONFIDENCE_ERROR_TEMPLATE = """
+We could not reproduce the regression with this test/metric/platform combination
+with enough confidence.
+
+Here are the results for the initial revision range:
+\'Good\' revision: {}

[qyearsley, 2014/10/23 00:38:39]

Giving names to each of these template fields would make it more readable when
the template is filled out.

For example, if template = """
Good revision: {good_rev}
Mean: {mean}
Std. Err.: {std_err}
...
"""
then you can call template.format(good_rev=123, mean=12, std_err=1)

[RobertoCN, 2014/10/23 19:51:45]

Done.

+\tmean: {}
+\tstd.err.:{}
+\tsample size:{}
+\'Bad\' revision: {}
+\tmean: {}
+\tstd.err.:{}
+\tsample size:{}
+
+NOTE: There\'s still a chance that this is actually a regression, but you may

[qyearsley, 2014/10/23 00:38:39]

Escaping single quotes isn't necessary insider a """-quoted string.

[RobertoCN, 2014/10/23 19:51:45]

Done.

+      need to bisect a different platform."""
+
 # Git branch name used to run bisect try jobs.
 BISECT_TRYJOB_BRANCH = 'bisect-tryjob'
 # Git master branch name.
 BISECT_MASTER_BRANCH = 'master'
 # File to store 'git diff' content.
 BISECT_PATCH_FILE = 'deps_patch.txt'
 # SVN repo where the bisect try jobs are submitted.
 SVN_REPO_URL = 'svn://svn.chromium.org/chrome-try/try-perf'
 
 class RunGitError(Exception):
@@ skipping 2108 matching lines @@
                    '\'good\' - \'bad\' range of revisions represent an '
                    'improvement (and not a regression).')
           return BisectResults(error=error)
         print message, "Therefore we continue to bisect."
 
       bisect_state = BisectState(target_depot, revision_list)
       revision_states = bisect_state.GetRevisionStates()
 
       min_revision = 0
       max_revision = len(revision_states) - 1
+      # Check how likely it is that the good and bad results are different
+      # beyond chance-induced variation.
+      if not self.opts.debug_ignore_regression_confidence:
+        # Adding good and bad values to a parameter list.
+        confidenceParams = []
+        for l in [known_bad_value['values'], known_good_value['values']]:
+          # Flatten if needed
+          if isinstance(l, list) and all([isinstance(x, list) for x in l]):
+            confidenceParams.append(sum(l, []))
+          else:
+            confidenceParams.append(l)
+        regression_confidence = BisectResults.ConfidenceScore(*confidenceParams)
+        if regression_confidence < REGRESSION_CONFIDENCE:
+          error = REGRESSION_CONFIDENCE_ERROR_TEMPLATE.format(
+              good_revision, known_good_value['mean'],
+              known_good_value['std_err'], len(known_good_value['values']),
+              bad_revision, known_bad_value['mean'],
+              known_bad_value['std_err'], len(known_bad_value['values']))
+          return BisectResults(error=error)

[qyearsley, 2014/10/23 00:38:38]

Might be a good idea to extract everything under the "if not
self.opts.debug_ignore_regression_confidence" into a separate function.

[RobertoCN, 2014/10/23 19:51:45]

Done.

 
       # Can just mark the good and bad revisions explicitly here since we
       # already know the results.
       bad_revision_state = revision_states[min_revision]
       bad_revision_state.external = bad_results[2]
       bad_revision_state.perf_time = bad_results[3]
       bad_revision_state.build_time = bad_results[4]
       bad_revision_state.passed = False
       bad_revision_state.value = known_bad_value
 
@@ skipping 188 matching lines @@
     self.max_time_minutes = 20
     self.metric = None
     self.command = None
     self.output_buildbot_annotations = None
     self.no_custom_deps = False
     self.working_directory = None
     self.extra_src = None
     self.debug_ignore_build = None
     self.debug_ignore_sync = None
     self.debug_ignore_perf_test = None
+    self.debug_ignore_regression_confidence = None
     self.debug_fake_first_test_mean = 0
     self.gs_bucket = None
     self.target_arch = 'ia32'
     self.target_build_type = 'Release'
     self.builder_host = None
     self.builder_port = None
     self.bisect_mode = bisect_utils.BISECT_MODE_MEAN
     self.improvement_direction = 0
 
   @staticmethod
@@ skipping 148 matching lines @@
     group = optparse.OptionGroup(parser, 'Debug options')
     group.add_option('--debug_ignore_build',
                      action='store_true',
                      help='DEBUG: Don\'t perform builds.')
     group.add_option('--debug_ignore_sync',
                      action='store_true',
                      help='DEBUG: Don\'t perform syncs.')
     group.add_option('--debug_ignore_perf_test',
                      action='store_true',
                      help='DEBUG: Don\'t perform performance tests.')
+    group.add_option('--debug_ignore_regression_confidence',
+                     action='store_true',
+                     help='DEBUG: Don\'t score the confidence of the initial '
+                          'good and bad revisions\' test results.')
     group.add_option('--debug_fake_first_test_mean',
                      type='int',
                      default='0',
                      help=('DEBUG: When faking performance tests, return this '
                            'value as the mean of the first performance test, '
                            'and return a mean of 0.0 for further tests.'))
     parser.add_option_group(group)
     return parser
 
   def ParseCommandLine(self):
@@ skipping 141 matching lines @@
       # bugs. If you change this, please update the perf dashboard as well.
       bisect_utils.OutputAnnotationStepStart('Results')
     print 'Error: %s' % e.message
     if opts.output_buildbot_annotations:
       bisect_utils.OutputAnnotationStepClosed()
   return 1
 
 
 if __name__ == '__main__':
   sys.exit(main())