Implementing direction_of_improvement

tools/auto_bisect/bisect_perf_regression.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Performance Test Bisect Tool
 
 This script bisects a series of changelists using binary search. It starts at
 a bad revision where a performance metric has regressed, and asks for a last
 known-good revision. It will then binary search across this revision range by
@@ skipping 1500 matching lines @@
       updated_deps_content = self.UpdateDepsContents(
           deps_contents, depot, revision, deps_var)
       # Write changes to DEPS file
       if updated_deps_content:
         WriteStringToFile(updated_deps_content, deps_file)
         return True
     except IOError, e:
       print 'Something went wrong while updating DEPS file. [%s]' % e
     return False
 
-
   def CreateDEPSPatch(self, depot, revision):
     """Modifies DEPS and returns diff as text.
 
     Args:
       depot: Current depot being bisected.
       revision: A git hash revision of the dependency repository.
 
     Returns:
       A tuple with git hash of chromium revision and DEPS patch text.
     """
@@ skipping 125 matching lines @@
     """
     success_code, failure_code = 0, -1
 
     if self.opts.debug_ignore_perf_test:
       fake_results = {
           'mean': 0.0,
           'std_err': 0.0,
           'std_dev': 0.0,
           'values': [0.0]
       }
+
+      # When debug_fake_test_mean is set, its value is returned as the mean
+      # and the flag is cleared so that further calls behave as if it wasn't
+      # set (returning the fake_results dict as defined above).
+      if self.opts.debug_fake_first_test_mean:
+        fake_results['mean'] = float(self.opts.debug_fake_first_test_mean)
+        self.opts.debug_fake_first_test_mean = 0
+
       return (fake_results, success_code)
 
     # For Windows platform set posix=False, to parse windows paths correctly.
     # On Windows, path separators '\' or '\\' are replace by '' when posix=True,
     # refer to http://bugs.python.org/issue1724822. By default posix=True.
     args = shlex.split(command_to_run, posix=not bisect_utils.IsWindowsHost())
 
     if not _GenerateProfileIfNecessary(args):
       err_text = 'Failed to generate profile for performance test.'
       return (err_text, failure_code)
@@ skipping 730 matching lines @@
       self.PrintRevisionsToBisectMessage(revision_list, target_depot)
 
       if self.opts.output_buildbot_annotations:
         bisect_utils.OutputAnnotationStepStart('Gathering Reference Values')
 
       print 'Gathering reference values for bisection.'
 
       # Perform the performance tests on the good and bad revisions, to get
       # reference values.
       bad_results, good_results = self.GatherReferenceValues(good_revision,
-                                                               bad_revision,
-                                                               command_to_run,
-                                                               metric,
-                                                               target_depot)
+                                                             bad_revision,
+                                                             command_to_run,
+                                                             metric,
+                                                             target_depot)
 
       if self.opts.output_buildbot_annotations:
         bisect_utils.OutputAnnotationStepClosed()
 
       if bad_results[1]:
         results.error = ('An error occurred while building and running '
             'the \'bad\' reference value. The bisect cannot continue without '
             'a working \'bad\' revision to start from.\n\nError: %s' %
             bad_results[0])
         return results
 
       if good_results[1]:
         results.error = ('An error occurred while building and running '
             'the \'good\' reference value. The bisect cannot continue without '
             'a working \'good\' revision to start from.\n\nError: %s' %
             good_results[0])
         return results
 
-
       # We need these reference values to determine if later runs should be
       # classified as pass or fail.
       known_bad_value = bad_results[0]
       known_good_value = good_results[0]
 
+      # Check the direction of improvement only if the improvement_direction
+      # option is set to a specific direction (1 for higher is better or -1 for
+      # lower is better).
+      improvement_dir = self.opts.improvement_direction
+      if improvement_dir:
+        higher_is_better  = improvement_dir > 0
+        if higher_is_better:
+          message = "Expecting higher values to be better for this metric, "
+        else:
+          message = "Expecting lower values to be better for this metric, "
+        metric_increased = known_bad_value['mean'] > known_good_value['mean']
+        if metric_increased:
+          message += "and the metric appears to have increased. "
+        else:
+          message += "and the metric appears to have decreased. "
+        if ((higher_is_better and metric_increased) or
+            (not higher_is_better and not metric_increased)):
+          results.error = (message + 'Then, the test results for the ends of '
+                           'the given \'good\' - \'bad\' range of revisions '
+                           'represent an improvement (and not a regression).')
+          return results
+        print message, "Therefore we continue to bisect."
+
       # Can just mark the good and bad revisions explicitly here since we
       # already know the results.
       bad_revision_data = revision_data[revision_list[0]]
       bad_revision_data['external'] = bad_results[2]
       bad_revision_data['perf_time'] = bad_results[3]
       bad_revision_data['build_time'] = bad_results[4]
       bad_revision_data['passed'] = False
       bad_revision_data['value'] = known_bad_value
 
       good_revision_data = revision_data[revision_list[max_revision]]
@@ skipping 477 matching lines @@
     self.max_time_minutes = 20
     self.metric = None
     self.command = None
     self.output_buildbot_annotations = None
     self.no_custom_deps = False
     self.working_directory = None
     self.extra_src = None
     self.debug_ignore_build = None
     self.debug_ignore_sync = None
     self.debug_ignore_perf_test = None
+    self.debug_fake_first_test_mean = 0
     self.gs_bucket = None
     self.target_arch = 'ia32'
     self.target_build_type = 'Release'
     self.builder_host = None
     self.builder_port = None
     self.bisect_mode = BISECT_MODE_MEAN
+    self.improvement_direction = 0
 
   @staticmethod
   def _CreateCommandLineParser():
     """Creates a parser with bisect options.
 
     Returns:
       An instance of optparse.OptionParser.
     """
     usage = ('%prog [options] [-- chromium-options]\n'
              'Perform binary search on revision history to find a minimal '
@@ skipping 13 matching lines @@
                      ' or svn revision.')
     group.add_option('-g', '--good_revision',
                      type='str',
                      help='A revision to start bisection where performance' +
                      ' test is known to pass. Must be earlier than the ' +
                      'bad revision. May be either a git or svn revision.')
     group.add_option('-m', '--metric',
                      type='str',
                      help='The desired metric to bisect on. For example ' +
                      '"vm_rss_final_b/vm_rss_f_b"')
+    group.add_option('-d', '--improvement_direction',
+                     type='int',
+                     default=0,
+                     help='An integer number representing the direction of ' +
+                     'improvement. 1 for higher is better, -1 for lower is ' +
+                     'better, 0 for ignore (default).')
     group.add_option('-r', '--repeat_test_count',
                      type='int',
                      default=20,
                      help='The number of times to repeat the performance '
                      'test. Values will be clamped to range [1, 100]. '
                      'Default value is 20.')
     group.add_option('--max_time_minutes',
                      type='int',
                      default=20,
                      help='The maximum time (in minutes) to take running the '
@@ skipping 100 matching lines @@
     group = optparse.OptionGroup(parser, 'Debug options')
     group.add_option('--debug_ignore_build',
                      action='store_true',
                      help='DEBUG: Don\'t perform builds.')
     group.add_option('--debug_ignore_sync',
                      action='store_true',
                      help='DEBUG: Don\'t perform syncs.')
     group.add_option('--debug_ignore_perf_test',
                      action='store_true',
                      help='DEBUG: Don\'t perform performance tests.')
+    group.add_option('--debug_fake_first_test_mean',
+                     type='int',
+                     default='0',
+                     help=('DEBUG: When faking performance tests, return this '
+                           'value as the mean of the first performance test, '
+                           'and return a mean of 0.0 for further tests.'))
     parser.add_option_group(group)
     return parser
 
   def ParseCommandLine(self):
     """Parses the command line for bisect options."""
     parser = self._CreateCommandLineParser()
     opts, _ = parser.parse_args()
 
     try:
       if not opts.command:
@@ skipping 135 matching lines @@
       # bugs. If you change this, please update the perf dashboard as well.
       bisect_utils.OutputAnnotationStepStart('Results')
     print 'Error: %s' % e.message
     if opts.output_buildbot_annotations:
       bisect_utils.OutputAnnotationStepClosed()
   return 1
 
 
 if __name__ == '__main__':
   sys.exit(main())