[bisect] - Let perf try jobs specify revisions to test.

tools/run-bisect-perf-regression.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Run Performance Test Bisect Tool
 
 This script is used by a try bot to run the bisect script with the parameters
 specified in the bisect config file. It checks out a copy of the depot in
 a subdirectory 'bisect' of the working directory provided, annd runs the
 bisect scrip there.
 """
 
 import optparse
 import os
 import platform
 import re
 import subprocess
 import sys
 import traceback
 
 from auto_bisect import bisect_perf_regression
 from auto_bisect import bisect_utils
 from auto_bisect import math_utils
+from auto_bisect import source_control
 
 CROS_BOARD_ENV = 'BISECT_CROS_BOARD'
 CROS_IP_ENV = 'BISECT_CROS_IP'
 
 SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
 SRC_DIR = os.path.join(SCRIPT_DIR, os.path.pardir)
 BISECT_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'auto_bisect', 'bisect.cfg')
 RUN_TEST_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'run-perf-test.cfg')
 WEBKIT_RUN_TEST_CONFIG_PATH = os.path.join(
     SRC_DIR, 'third_party', 'WebKit', 'Tools', 'run-perf-test.cfg')
@@ skipping 213 matching lines @@
       re.MULTILINE)
 
   results = {
       'html-results': html_results_pattern.findall(output),
       'profiler': profiler_pattern.findall(output),
   }
 
   return results
 
 
-def _RunPerformanceTest(config):
-  """Runs a performance test with and without the current patch.
-
-  Args:
-    config: Contents of the config file, a dictionary.
-
-  Attempts to build and run the current revision with and without the
-  current patch, with the parameters passed in.
-  """
-  # Bisect script expects to be run from the src directory
-  os.chdir(SRC_DIR)
-
-  bisect_utils.OutputAnnotationStepStart('Building With Patch')
-
-  opts = _CreateBisectOptionsFromConfig(config)
-  b = bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd())
-
-  if bisect_utils.RunGClient(['runhooks']):
-    raise RuntimeError('Failed to run gclient runhooks')
-
-  if not b.ObtainBuild('chromium'):
-    raise RuntimeError('Patched version failed to build.')
-
-  bisect_utils.OutputAnnotationStepClosed()
-  bisect_utils.OutputAnnotationStepStart('Running With Patch')
-
-  results_with_patch = b.RunPerformanceTestAndParseResults(
-      opts.command,
-      opts.metric,
-      reset_on_first_run=True,
-      upload_on_last_run=True,
-      results_label='Patch')
-
-  if results_with_patch[1]:
-    raise RuntimeError('Patched version failed to run performance test.')
-
-  bisect_utils.OutputAnnotationStepClosed()
-
-  bisect_utils.OutputAnnotationStepStart('Reverting Patch')
-  # TODO: When this is re-written to recipes, this should use bot_update's
-  # revert mechanism to fully revert the client. But for now, since we know that
-  # the perf try bot currently only supports src/ and src/third_party/WebKit, we
-  # simply reset those two directories.
-  bisect_utils.CheckRunGit(['reset', '--hard'])
-  bisect_utils.CheckRunGit(['reset', '--hard'],
-                           os.path.join('third_party', 'WebKit'))
-  bisect_utils.OutputAnnotationStepClosed()
-
-  bisect_utils.OutputAnnotationStepStart('Building Without Patch')
-
-  if bisect_utils.RunGClient(['runhooks']):
-    raise RuntimeError('Failed to run gclient runhooks')
-
-  if not b.ObtainBuild('chromium'):
-    raise RuntimeError('Unpatched version failed to build.')
-
-  bisect_utils.OutputAnnotationStepClosed()
-  bisect_utils.OutputAnnotationStepStart('Running Without Patch')
-
-  results_without_patch = b.RunPerformanceTestAndParseResults(
-      opts.command, opts.metric, upload_on_last_run=True, results_label='ToT')
-
-  if results_without_patch[1]:
-    raise RuntimeError('Unpatched version failed to run performance test.')
-
-  # Find the link to the cloud stored results file.
+def _ParseAndOutputCloudLinks(results_without_patch, results_with_patch):
   cloud_links_without_patch = _ParseCloudLinksFromOutput(
       results_without_patch[2])
   cloud_links_with_patch = _ParseCloudLinksFromOutput(
       results_with_patch[2])
 
   cloud_file_link = (cloud_links_without_patch['html-results'][0]
       if cloud_links_without_patch['html-results'] else '')
 
   profiler_file_links_with_patch = cloud_links_with_patch['profiler']
   profiler_file_links_without_patch = cloud_links_without_patch['profiler']
 
   # Calculate the % difference in the means of the 2 runs.
   percent_diff_in_means = None
   std_err = None
   if (results_with_patch[0].has_key('mean') and
       results_with_patch[0].has_key('values')):
     percent_diff_in_means = (results_with_patch[0]['mean'] /
         max(0.0001, results_without_patch[0]['mean'])) * 100.0 - 100.0
     std_err = math_utils.PooledStandardError(
         [results_with_patch[0]['values'], results_without_patch[0]['values']])
 
-  bisect_utils.OutputAnnotationStepClosed()
   if percent_diff_in_means is not None and std_err is not None:
     bisect_utils.OutputAnnotationStepStart('Results - %.02f +- %0.02f delta' %
         (percent_diff_in_means, std_err))
     print ' %s %s %s' % (''.center(10, ' '), 'Mean'.center(20, ' '),
         'Std. Error'.center(20, ' '))
     print ' %s %s %s' % ('Patch'.center(10, ' '),
         ('%.02f' % results_with_patch[0]['mean']).center(20, ' '),
         ('%.02f' % results_with_patch[0]['std_err']).center(20, ' '))
     print ' %s %s %s' % ('No Patch'.center(10, ' '),
         ('%.02f' % results_without_patch[0]['mean']).center(20, ' '),
         ('%.02f' % results_without_patch[0]['std_err']).center(20, ' '))
     if cloud_file_link:
       bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
     bisect_utils.OutputAnnotationStepClosed()
   elif cloud_file_link:
     bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
 
   if profiler_file_links_with_patch and profiler_file_links_without_patch:
     for i in xrange(len(profiler_file_links_with_patch)):
       bisect_utils.OutputAnnotationStepLink(
           'With Patch - Profiler Data[%d]' % i,
           profiler_file_links_with_patch[i])
     for i in xrange(len(profiler_file_links_without_patch)):
       bisect_utils.OutputAnnotationStepLink(
           'Without Patch - Profiler Data[%d]' % i,
           profiler_file_links_without_patch[i])
 
 
+def _ResolveRevisionsFromConfig(config):
+  if not 'good_revision' in config and not 'bad_revision' in config:
+    return (None, None)
+
+  bad_revision = source_control.ResolveToRevision(
+      config['bad_revision'], 'chromium', bisect_utils.DEPOT_DEPS_NAME, 100)
+  if not bad_revision:
+    raise RuntimeError('Failed to resolve [%s] to git hash.',
+        config['bad_revision'])
+  good_revision = source_control.ResolveToRevision(
+      config['good_revision'], 'chromium', bisect_utils.DEPOT_DEPS_NAME, -100)
+  if not good_revision:
+    raise RuntimeError('Failed to resolve [%s] to git hash.',
+        config['good_revision'])
+
+  return (good_revision, bad_revision)
+
+
+def _GetStepAnnotationStringsDict(config):
+  if 'good_revision' in config and 'bad_revision' in config:
+    return {
+        'build1': 'Building [%s]' % config['good_revision'],
+        'build2': 'Building [%s]' % config['bad_revision'],
+        'run1': 'Running [%s]' % config['good_revision'],
+        'run2': 'Running [%s]' % config['bad_revision'],
+        'results_label1': config['good_revision'],
+        'results_label2': config['bad_revision'],
+    }
+  else:
+    return {
+        'build1': 'Building With Patch',
+        'build2': 'Building Without Patch',
+        'run1': 'Running With Patch',
+        'run2': 'Running Without Patch',
+        'results_label1': 'Patch',
+        'results_label2': 'ToT',
+    }
+
+
+def _RunBuildStepForPerformanceTest(bisect_instance, build_string, revision):
+  if revision:
+    bisect_utils.OutputAnnotationStepStart('Syncing [%s]' % revision)
+    if not source_control.SyncToRevision(revision, 'gclient'):
+      raise RuntimeError('Failed to sync to [%s].' % revision)
+    bisect_utils.OutputAnnotationStepClosed()
+
+  bisect_utils.OutputAnnotationStepStart(build_string)
+
+  if bisect_utils.RunGClient(['runhooks']):
+    raise RuntimeError('Failed to run gclient runhooks')
+
+  if not bisect_instance.ObtainBuild('chromium'):
+    raise RuntimeError('Patched version failed to build.')
+
+  bisect_utils.OutputAnnotationStepClosed()
+
+
+def _RunCommandStepForPerformanceTest(bisect_instance,
+                                      opts,
+                                      reset_on_first_run,
+                                      upload_on_last_run,
+                                      results_label,
+                                      run_string):
+  bisect_utils.OutputAnnotationStepStart(run_string)
+
+  results = bisect_instance.RunPerformanceTestAndParseResults(
+      opts.command,
+      opts.metric,
+      reset_on_first_run=reset_on_first_run,
+      upload_on_last_run=upload_on_last_run,
+      results_label=results_label)
+
+  if results[1]:
+    raise RuntimeError('Patched version failed to run performance test.')
+
+  bisect_utils.OutputAnnotationStepClosed()
+
+  return results
+
+
+def _RunPerformanceTest(config):
+  """Runs a performance test with and without the current patch.
+
+  Args:
+    config: Contents of the config file, a dictionary.
+
+  Attempts to build and run the current revision with and without the
+  current patch, with the parameters passed in.
+  """
+  # Bisect script expects to be run from the src directory
+  os.chdir(SRC_DIR)
+
+  opts = _CreateBisectOptionsFromConfig(config)
+  revisions = _ResolveRevisionsFromConfig(config)
+  annotations_dict = _GetStepAnnotationStringsDict(config)
+  b = bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd())
+
+  _RunBuildStepForPerformanceTest(b, annotations_dict['build1'], revisions[0])
+
+  results_with_patch = _RunCommandStepForPerformanceTest(
+      b, opts, True, True, annotations_dict['results_label1'],
+      annotations_dict['run1'])
+
+  bisect_utils.OutputAnnotationStepStart('Reverting Patch')
+  # TODO: When this is re-written to recipes, this should use bot_update's
+  # revert mechanism to fully revert the client. But for now, since we know that
+  # the perf try bot currently only supports src/ and src/third_party/WebKit, we
+  # simply reset those two directories.
+  bisect_utils.CheckRunGit(['reset', '--hard'])
+  bisect_utils.CheckRunGit(['reset', '--hard'],
+                           os.path.join('third_party', 'WebKit'))
+  bisect_utils.OutputAnnotationStepClosed()
+
+  _RunBuildStepForPerformanceTest(b, annotations_dict['build2'], revisions[1])
+
+  results_without_patch = _RunCommandStepForPerformanceTest(
+      b, opts, False, True, annotations_dict['results_label2'],
+      annotations_dict['run1'])
+
+  # Find the link to the cloud stored results file.
+  _ParseAndOutputCloudLinks(results_without_patch, results_with_patch)
+
+
 def _SetupAndRunPerformanceTest(config, path_to_goma):
   """Attempts to build and run the current revision with and without the
   current patch, with the parameters passed in.
 
   Args:
     config: The config read from run-perf-test.cfg.
     path_to_goma: Path to goma directory.
 
   Returns:
     An exit code: 0 on success, otherwise 1.
@@ skipping 193 matching lines @@
     if config and config_is_valid:
       return _SetupAndRunPerformanceTest(config, opts.path_to_goma)
 
   print ('Error: Could not load config file. Double check your changes to '
          'auto_bisect/bisect.cfg or run-perf-test.cfg for syntax errors.\n')
   return 1
 
 
 if __name__ == '__main__':
   sys.exit(main())