Adding gather_stats (based on bisect-builds.py) to gather benchmarking

tools/gpu/gather_stats.py

+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Snapshot Build Linear Benchmark Tool
+
+This script executes and gathers stats from a series of snapshot archives. It
+starts at a start revision (it will try to guess HEAD) and asks for an end
+revision. It will then linearly run across this revision range by downloading,
+unzipping, and gathers stats for each revision.
+"""
+
+import optparse
+import os
+import sys
+from distutils.version import LooseVersion
+import inspect
+
+cmd_folder = os.path.realpath(os.path.abspath(os.path.split(inspect.getfile(
+    inspect.currentframe() ))[0]))
+
+bisect_builds_dir = os.path.join(cmd_folder, "..")
+
+if bisect_builds_dir not in sys.path:
+  sys.path.insert(0, bisect_builds_dir)
+
+bisect_builds = __import__('bisect-builds')
+
+def gather_stats(platform,

[Robert Sesek, 2012/09/24 17:52:30]

Chromium style should be GatherStats.

[hartmanng, 2012/09/24 20:13:20]

Done.

+                 official_builds,
+                 execute,
+                 start_rev=0,
+                 end_rev=0,
+                 num_runs=1,
+                 stdoutdir=None):
+  """Given start and known end revisions, run sequentially
+  on archived revisions, gathering performance stats at each one.
+
+  @param platform Which build to download/run ('mac', 'win', 'linux64', etc.).
+  @param official_builds Specify build type (Chromium or Official build).
+  @param execute String specifying what command to run at each revision.
+  @param start_rev Number/tag of the revision to start on.
+  @param end_rev Number/tag of the revision to end on.
+  @param num_runs Number of times to run each build.
+  @param stdoutdir The (optional) path to an output directory."""
+
+  context = bisect_builds.PathContext(platform, start_rev, end_rev,
+                                      official_builds)
+  cwd = os.getcwd()
+
+
+  print "Downloading list of known revisions..."
+  _GetDownloadPath = lambda rev: os.path.join(cwd,
+      '%s-%s' % (str(rev), context.archive_name))
+  if official_builds:
+    revlist = context.GetOfficialBuildsList()
+  else:
+    revlist = context.GetRevList()
+
+  print 'Running on range [%s, %s].' % (revlist[0], revlist[-1])
+
+  # Initialize our revision index and fetch the first revision.
+  rev = revlist[0]
+  next_rev = 0
+  zipfile = _GetDownloadPath(rev)
+  initial_fetch = bisect_builds.DownloadJob(context, 'initial_fetch', rev,
+                                            zipfile)
+  initial_fetch.Start()
+  initial_fetch.WaitFor()
+  index = 1
+
+  while zipfile and index <= len(revlist):
+    cur_fetch = None
+    if index < len(revlist):
+      # Pre-fetch next revision
+      next_rev = revlist[index]
+      cur_fetch = bisect_builds.DownloadJob(context, 'cur_fetch', next_rev,
+                                            _GetDownloadPath(next_rev))
+      cur_fetch.Start()
+
+    # Run test on the index revision.
+    def execute_args(chrome_path):
+      return (execute % chrome_path).split(' ')
+
+    # Run test on the index revision.
+    (status, stdout, stderr) = bisect_builds.RunRevision(context,
+                                                         rev,
+                                                         zipfile,
+                                                         num_runs,
+                                                         execute_args,
+                                                         stdoutdir)
+
+    rev = next_rev
+
+    os.unlink(zipfile)
+    zipfile = None
+
+    # Wait for fetch to complete.
+    try:
+      if cur_fetch:
+        cur_fetch.WaitFor()
+        index = index + 1
+        zipfile = cur_fetch.zipfile
+    except SystemExit:
+      print "Cleaning up..."
+      for f in [_GetDownloadPath(revlist[index])]:
+        try:
+          os.unlink(f)
+        except OSError:
+          pass
+      sys.exit(0)
+
+  return 0
+
+def main():
+  usage = ('%prog [options]\n'
+           'Collects benchmark statistics on snapshot builds.\n')
+  parser = optparse.OptionParser(usage=usage)
+  # Strangely, the default help output doesn't include the choice list.
+  choices = ['mac', 'win', 'linux', 'linux64']
+            # linux-chromiumos lacks a continuous archive http://crbug.com/78158
+  parser.add_option('-a', '--archive',
+                    choices = choices,
+                    help = 'The buildbot archive to run [%s].' %
+                           '|'.join(choices))
+  parser.add_option('-o', action="store_true", dest='official_builds',
+                    help = 'Run across official ' +
+                    'Chrome builds (internal only) instead of ' +
+                    'Chromium archives.')
+  parser.add_option('-e', '--end', type = 'str',
+                    help = 'The revision to end on. Default is HEAD.')
+  parser.add_option('-s', '--start', type = 'str',
+                    help = 'The revision to start from. Default is 0.')
+  parser.add_option('-t', '--times', type = 'int',
+                    help = 'Number of times to run each build. Temporary' +
+                    ' profiles are reused. Default is 1.',
+                    default = 1)
+  parser.add_option('-x', '--execute', type='str',
+                    help = 'Command to execute for each revision. Include a ' +
+                    'single %s to specify the location of the downloaded ' +
+                    'chrome executable. Default is \'python ' +
+                    'run_scrolling_benchmark page_sets/Q32012.json ' +
+                    '--browser-executable=%s\'.',
+                    default = 'python ./run_scrolling_benchmark ' +
+                    'page_sets/Q32012.json --browser-executable=%s')
+  parser.add_option('-d', '--output-directory', '--stdoutdir', type='str',
+                    help = 'Save stdout to files for each revision.',
+                    default = None)
+  parser.add_option('-r', '--revisions', type = 'int',
+                    help = 'Number of revisions to run if a start revision ' +
+                    'isn\'t specified. Default is 500.',
+                    default = 500)
+  (opts, args) = parser.parse_args()
+
+  if opts.archive is None:
+    print 'Error: missing required parameter: --archive'
+    print
+    parser.print_help()
+    return 1
+
+  # Create the context. Initialize 0 for the revisions as they are set below.
+  context = bisect_builds.PathContext(opts.archive, 0, 0, opts.official_builds)
+  # Pick a starting point, try to get HEAD for this.
+  if opts.end:
+    end_rev = opts.end
+  else:
+    end_rev = '999.0.0.0'
+    if not opts.official_builds:
+      end_rev = GetChromiumRevision(context.GetLastChangeURL())
+
+  # Find out where to start.
+  if opts.start:
+    start_rev = opts.start
+  else:
+    if opts.official_builds:
+      r = int(end_rev[:end_rev.find('.')]) - opts.revisions + 1
+      start_rev = str(r) + '.0.0.0'
+    else:
+      start_rev = int(end_rev) - opts.revisions + 1
+
+  if opts.official_builds:
+    start_rev = LooseVersion(start_rev)
+    end_rev = LooseVersion(end_rev)
+  else:
+    start_rev = int(start_rev)
+    end_rev = int(end_rev)
+
+  if opts.official_builds and start_rev < LooseVersion('0.0.0.0') or \
+      not opts.official_builds and start_rev < 0:
+    print ('The start revision (%s) must be greater than 0.\n' % start_rev)
+    parser.print_help()
+    return 1
+
+  if start_rev > end_rev:
+    print ('The start revision (%s) must precede the end revision (%s).\n' %
+           (start_rev, end_rev))
+    parser.print_help()
+    return 1
+
+  if opts.times < 1:
+    print('Number of times to run (%d) must be greater than or equal to 1.' %
+          opts.times)
+    parser.print_help()
+    return 1
+
+  return gather_stats(opts.archive, opts.official_builds, opts.execute,
+                      start_rev, end_rev, opts.times, opts.output_directory)
+
+if __name__ == '__main__':
+  sys.exit(main())