First pass performance try bot.

tools/bisect-perf-regression.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Performance Test Bisect Tool
 
 This script bisects a series of changelists using binary search. It starts at
 a bad revision where a performance metric has regressed, and asks for a last
 known-good revision. It will then binary search across this revision range by
@@ skipping 17 matching lines @@
 An example usage (using git hashes):
 
 ./tools/bisect-perf-regression.py -c\
 "out/Release/performance_ui_tests --gtest_filter=ShutdownTest.SimpleUserQuit"\
 -g 1f6e67861535121c5c819c16a666f2436c207e7b\
 -b b732f23b4f81c382db0b23b9035f3dadc7d925bb\
 -m shutdown/simple-user-quit
 
 """
 
+import copy
 import datetime
 import errno
 import imp
 import math
 import optparse
 import os
 import re
 import shlex
 import shutil
 import subprocess
@@ skipping 1179 matching lines @@
       path_to_generate = os.path.join('tools', 'perf', 'generate_profile')
 
       if arg_dict.has_key('--profile-dir') and arg_dict.has_key('--browser'):
         profile_path, profile_type = os.path.split(arg_dict['--profile-dir'])
         return not RunProcess(['python', path_to_generate,
             '--profile-type-to-generate', profile_type,
             '--browser', arg_dict['--browser'], '--output-dir', profile_path])
       return False
     return True
 
-  def RunPerformanceTestAndParseResults(self, command_to_run, metric):
+  def RunPerformanceTestAndParseResults(self, command_to_run, metric,
+      reset_on_first_run=False, upload_on_last_run=False):
     """Runs a performance test on the current revision by executing the
     'command_to_run' and parses the results.
 
     Args:
       command_to_run: The command to be run to execute the performance test.
       metric: The metric to parse out from the results of the performance test.
 
     Returns:
       On success, it will return a tuple of the average value of the metric,
       and a success code of 0.
     """
 
     if self.opts.debug_ignore_perf_test:
       return ({'mean': 0.0, 'std_err': 0.0, 'std_dev': 0.0, 'values': [0.0]}, 0)
 
     if IsWindows():
       command_to_run = command_to_run.replace('/', r'\\')
 
     args = shlex.split(command_to_run)
 
     if not self._GenerateProfileIfNecessary(args):
       return ('Failed to generate profile for performance test.', -1)
 
     # If running a telemetry test for cros, insert the remote ip, and
     # identity parameters.
-    if self.opts.target_platform == 'cros':
-      if 'tools/perf/run_' in args[0]:
-        args.append('--remote=%s' % self.opts.cros_remote_ip)
-        args.append('--identity=%s' % CROS_TEST_KEY_PATH)
+    is_telemetry = ('tools/perf/run_' in command_to_run or
+        r'tools\\perf\\run_' in command_to_run)

[tonyg, 2013/10/17 01:01:32]

I believe you want the r'' OR the \\s, but not both.

[shatch, 2013/10/17 21:59:08]

Done.

+    if self.opts.target_platform == 'cros' and is_telemetry:
+      args.append('--remote=%s' % self.opts.cros_remote_ip)
+      args.append('--identity=%s' % CROS_TEST_KEY_PATH)
+
+    # These arguments only work with telemetry.
+    reset_on_first_run = reset_on_first_run and is_telemetry
+    upload_on_last_run = upload_on_last_run and is_telemetry
 
     cwd = os.getcwd()
     os.chdir(self.src_cwd)
 
     start_time = time.time()
 
     metric_values = []
+    output_of_all_runs = ''
     for i in xrange(self.opts.repeat_test_count):
       # Can ignore the return code since if the tests fail, it won't return 0.
       try:
-        (output, return_code) = RunProcessAndRetrieveOutput(args)
+        current_args = copy.copy(args)
+        if i == 0 and reset_on_first_run:
+          current_args.append('--reset-results')
+        elif i == self.opts.repeat_test_count - 1 and upload_on_last_run:
+          current_args.append('--upload-results')
+        (output, return_code) = RunProcessAndRetrieveOutput(current_args)
       except OSError, e:
         if e.errno == errno.ENOENT:
           err_text  = ("Something went wrong running the performance test. "
               "Please review the command line:\n\n")
           if 'src/' in ' '.join(args):
             err_text += ("Check that you haven't accidentally specified a path "
                 "with src/ in the command.\n\n")
           err_text += ' '.join(args)
           err_text += '\n'
 
           return (err_text, -1)
         raise
 
+      output_of_all_runs += output
       if self.opts.output_buildbot_annotations:
         print output
 
       metric_values += self.ParseMetricValuesFromOutput(metric, output)
 
       elapsed_minutes = (time.time() - start_time) / 60.0
 
       if elapsed_minutes >= self.opts.max_time_minutes or not metric_values:
         break
 
     os.chdir(cwd)
 
     # Need to get the average value if there were multiple values.
     if metric_values:
       truncated_mean = CalculateTruncatedMean(metric_values,
           self.opts.truncate_percent)
       standard_err = CalculateStandardError(metric_values)
       standard_dev = CalculateStandardDeviation(metric_values)
 
       values = {
         'mean': truncated_mean,
         'std_err': standard_err,
         'std_dev': standard_dev,
         'values': metric_values,
       }
 
       print 'Results of performance test: %12f %12f' % (
           truncated_mean, standard_err)
       print
-      return (values, 0)
+      return (values, 0, output_of_all_runs)
     else:
       return ('Invalid metric specified, or no values returned from '
-          'performance test.', -1)
+          'performance test.', -1, output_of_all_runs)
 
   def FindAllRevisionsToSync(self, revision, depot):
     """Finds all dependant revisions and depots that need to be synced for a
     given revision. This is only useful in the git workflow, as an svn depot
     may be split into multiple mirrors.
 
     ie. skia is broken up into 3 git mirrors over skia/src, skia/gyp, and
     skia/include. To sync skia/src properly, one has to find the proper
     revisions in skia/gyp and skia/include.
 
@@ skipping 1403 matching lines @@
 
     Args:
       values: a dict containing options to set.
 
     Returns:
       An instance of BisectOptions.
     """
     opts = BisectOptions()
 
     for k, v in values.iteritems():
-      assert hasattr(opts, name_to_attr[k]), 'Invalid %s attribute in '\
-          'BisectOptions.' % name_to_attr[k]
-      setattr(opts, name_to_attr[k], v)
+      assert hasattr(opts, k), 'Invalid %s attribute in '\
+          'BisectOptions.' % k
+      setattr(opts, k, v)
 
     metric_values = opts.metric.split('/')
     if len(metric_values) != 2:
       raise RuntimeError("Invalid metric specified: [%s]" % opts.metric)
 
     opts.metric = metric_values
     opts.repeat_test_count = min(max(opts.repeat_test_count, 1), 100)
     opts.max_time_minutes = min(max(opts.max_time_minutes, 1), 60)
     opts.truncate_percent = min(max(opts.truncate_percent, 0), 25)
     opts.truncate_percent = opts.truncate_percent / 100.0
@@ skipping 55 matching lines @@
         return 0
     finally:
       bisect_test.PerformCleanup()
   except RuntimeError, e:
     print 'Error: %s' % e.message
     print
   return 1
 
 if __name__ == '__main__':
   sys.exit(main())