Make bisect script to post build request to try server.

tools/bisect-perf-regression.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Performance Test Bisect Tool
 
 This script bisects a series of changelists using binary search. It starts at
 a bad revision where a performance metric has regressed, and asks for a last
 known-good revision. It will then binary search across this revision range by
@@ skipping 34 matching lines @@
 import re
 import shlex
 import shutil
 import StringIO
 import subprocess
 import sys
 import time
 import zipfile
 
 import bisect_utils
+import post_perf_builder_job
+
 
 try:
   from telemetry.page import cloud_storage
 except ImportError:
   sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), 'telemetry'))
   from telemetry.page import cloud_storage
 
 # The additional repositories that might need to be bisected.
 # If the repository has any dependant repositories (such as skia/src needs
 # skia/include and skia/gyp to be updated), specify them in the 'depends'
@@ skipping 1241 matching lines @@
     source_dir = os.path.join(build_dir, build_type)
     destination_dir = os.path.join(build_dir, '%s.bak' % build_type)
     if restore:
       source_dir, destination_dir = destination_dir, source_dir
     if os.path.exists(source_dir):
       RmTreeAndMkDir(destination_dir, skip_makedir=True)
       shutil.move(source_dir, destination_dir)
       return destination_dir
     return None
 
-  def DownloadCurrentBuild(self, sha_revision, build_type='Release'):
+  def DownloadCurrentBuild(self, revision, build_type='Release'):
     """Download the build archive for the given revision.
 
     Args:
-      sha_revision: The git SHA1 for the revision.
+      revision: The SVN revision to build.
       build_type: Target build type ('Release', 'Debug', 'Release_x64' etc.)
 
     Returns:
       True if download succeeds, otherwise False.
     """
-    # Get SVN revision for the given SHA, since builds are archived using SVN
-    # revision.
-    revision = self.source_control.SVNFindRev(sha_revision)
-    if not revision:
-      raise RuntimeError(
-          'Failed to determine SVN revision for %s' % sha_revision)
-
     abs_build_dir = os.path.abspath(
         self.builder.GetBuildOutputDirectory(self.opts, self.src_cwd))
     target_build_output_dir = os.path.join(abs_build_dir, build_type)
     # Get build target architecture.
     build_arch = self.opts.target_arch
     # File path of the downloaded archive file.
     archive_file_dest = os.path.join(abs_build_dir,
                                      GetZipFileName(revision, build_arch))
-    if FetchFromCloudStorage(self.opts.gs_bucket,
-                             GetRemoteBuildPath(revision, build_arch),
-                             abs_build_dir):
-      # Generic name for the archive, created when archive file is extracted.
-      output_dir = os.path.join(abs_build_dir,
-                                GetZipFileName(target_arch=build_arch))
-      # Unzip build archive directory.
-      try:
+    remote_build = GetRemoteBuildPath(revision, build_arch)
+    fetch_build_func = lambda: FetchFromCloudStorage(self.opts.gs_bucket,
+                                                     remote_build,
+                                                     abs_build_dir)
+    if not fetch_build_func():
+      if not self.PostBuildRequestAndWait(revision, condition=fetch_build_func):
+        raise RuntimeError('Somewthing went wrong while processing build'
+                           'request for: %s' % revision)
+
+    # Generic name for the archive, created when archive file is extracted.
+    output_dir = os.path.join(abs_build_dir,
+                              GetZipFileName(target_arch=build_arch))
+    # Unzip build archive directory.
+    try:
+      RmTreeAndMkDir(output_dir, skip_makedir=True)
+      ExtractZip(archive_file_dest, abs_build_dir)
+      if os.path.exists(output_dir):
+        self.BackupOrRestoreOutputdirectory(restore=False)
+        print 'Moving build from %s to %s' % (
+            output_dir, target_build_output_dir)
+        shutil.move(output_dir, target_build_output_dir)
+        return True
+      raise IOError('Missing extracted folder %s ' % output_dir)
+    except e:
+      print 'Somewthing went wrong while extracting archive file: %s' % e
+      self.BackupOrRestoreOutputdirectory(restore=True)
+      # Cleanup any leftovers from unzipping.
+      if os.path.exists(output_dir):
         RmTreeAndMkDir(output_dir, skip_makedir=True)
-        ExtractZip(archive_file_dest, abs_build_dir)
-        if os.path.exists(output_dir):
-          self.BackupOrRestoreOutputdirectory(restore=False)
-          print 'Moving build from %s to %s' % (
-              output_dir, target_build_output_dir)
-          shutil.move(output_dir, target_build_output_dir)
-          return True
-        raise IOError('Missing extracted folder %s ' % output_dir)
-      except e:
-        print 'Somewthing went wrong while extracting archive file: %s' % e
-        self.BackupOrRestoreOutputdirectory(restore=True)
-        # Cleanup any leftovers from unzipping.
-        if os.path.exists(output_dir):
-          RmTreeAndMkDir(output_dir, skip_makedir=True)
-      finally:
-        # Delete downloaded archive
-        if os.path.exists(archive_file_dest):
-          os.remove(archive_file_dest)
+    finally:
+      # Delete downloaded archive
+      if os.path.exists(archive_file_dest):
+        os.remove(archive_file_dest)
+    return False
+
+  def PostBuildRequestAndWait(self, revision, condition, patch=None):
+    """POSTs the build request job to the tryserver instance."""
+
+    def GetBuilderNameAndBuildTime(target_arch='ia32'):
+      """Gets builder name and buildtime in seconds based on platform."""
+      if IsWindows():
+        if Is64BitWindows() and target_arch == 'x64':
+          return ('Win x64 Bisect Builder', 3600)
+        return ('Win Bisect Builder', 3600)
+      if IsLinux():
+        return ('Linux Bisect Builder', 1800)
+      if IsMac():
+        return ('Mac Bisect Builder', 2700)
+      raise NotImplementedError('Unsupported Platform "%s".' % sys.platform)
+    if not condition:
+      return False
+
+    bot_name, build_timeout = GetBuilderNameAndBuildTime(self.opts.target_arch)
+
+    # Creates a try job description.
+    job_args = {'host': self.opts.builder_host,
+                'port': self.opts.builder_port,
+                'revision': revision,
+                'bot': bot_name,
+                'name': 'Bisect Job-%s' % revision
+               }
+    # Update patch information if supplied.
+    if patch:
+      job_args['patch'] = patch
+    # Posts job to build the revision on the server.
+    if post_perf_builder_job.PostTryJob(job_args):
+      poll_interval = 60
+      start_time = time.time()
+      while True:
+        res = condition()
+        if res:
+          return res
+        elapsed_time = time.time() - start_time
+        if elapsed_time > build_timeout:

[shatch, 2014/03/11 17:20:54]

A timeout seems fair enough for now I think. I can imagine once sheriff's are
using this, if several people post build jobs (to say, windows), that we might
be mid-build when this timeout occurs. Is the plan to add additional build
capacity to handle this? Is there any way to check what the status of the build
is, in case it's *nearly* done?

[prasadv, 2014/03/13 01:03:26]

To handle load, we requested 16 new builder bots on tryserver
(crbug.com/347707).
Regarding build status, I'll investigate a way to find this information.

On 2014/03/11 17:20:54, shatch wrote:

+          raise RuntimeError('Timed out while waiting %ds for %s build.' %
+                             (build_timeout, revision))
+        print ('Time elapsed: %ss, still waiting for %s build' %
+               (elapsed_time, revision))
+        time.sleep(poll_interval)
     return False
 
   def BuildCurrentRevision(self, depot, revision=None):
     """Builds chrome and performance_ui_tests on the current revision.
 
     Returns:
       True if the build was successful.
     """
     if self.opts.debug_ignore_build:
       return True
     cwd = os.getcwd()
     os.chdir(self.src_cwd)
     # Fetch build archive for the given revision from the cloud storage when
     # the storage bucket is passed.
     if depot == 'chromium' and self.opts.gs_bucket and revision:
+      # Get SVN revision for the given SHA, since builds are archived using SVN
+      # revision.
+      revision = self.source_control.SVNFindRev(revision)
+      if not revision:
+        raise RuntimeError(
+            'Failed to determine SVN revision for %s' % sha_revision)
       if self.DownloadCurrentBuild(revision):
         os.chdir(cwd)
         return True
       raise RuntimeError('Failed to download build archive for revision %s.\n'
                          'Unfortunately, bisection couldn\'t continue any '
                          'further. Please try running script without '
                          '--gs_bucket flag to produce local builds.' % revision)
 
+
     build_success = self.builder.Build(depot, self.opts)
     os.chdir(cwd)
     return build_success
 
   def RunGClientHooks(self):
     """Runs gclient with runhooks command.
 
     Returns:
       True if gclient reports no errors.
     """
@@ skipping 1614 matching lines @@
     self.command = None
     self.output_buildbot_annotations = None
     self.no_custom_deps = False
     self.working_directory = None
     self.extra_src = None
     self.debug_ignore_build = None
     self.debug_ignore_sync = None
     self.debug_ignore_perf_test = None
     self.gs_bucket = None
     self.target_arch = 'ia32'
+    self.builder_host = None
+    self.builder_port = None
 
   def _CreateCommandLineParser(self):
     """Creates a parser with bisect options.
 
     Returns:
       An instance of optparse.OptionParser.
     """
     usage = ('%prog [options] [-- chromium-options]\n'
              'Perform binary search on revision history to find a minimal '
              'range of revisions where a peformance metric regressed.\n')
@@ skipping 93 matching lines @@
                      type='str',
                      help=('Name of Google Storage bucket to upload or '
                      'download build. e.g., chrome-perf'))
     group.add_option('--target_arch',
                      type='choice',
                      choices=['ia32', 'x64', 'arm'],
                      default='ia32',
                      dest='target_arch',
                      help=('The target build architecture. Choices are "ia32" '
                      '(default), "x64" or "arm".'))
-
+    group.add_option('--builder_host',
+                     dest='builder_host',
+                     type='str',
+                     help=('Host address of server to produce build by posting'
+                           ' try job request.'))
+    group.add_option('--builder_port',
+                     dest='builder_port',
+                     type='int',
+                     help=('HTTP port of the server to produce build by posting'
+                           ' try job request.'))
     parser.add_option_group(group)
 
     group = optparse.OptionGroup(parser, 'Debug options')
     group.add_option('--debug_ignore_build',
                      action="store_true",
                      help='DEBUG: Don\'t perform builds.')
     group.add_option('--debug_ignore_sync',
                      action="store_true",
                      help='DEBUG: Don\'t perform syncs.')
     group.add_option('--debug_ignore_perf_test',
                      action="store_true",
                      help='DEBUG: Don\'t perform performance tests.')
     parser.add_option_group(group)
-
-
     return parser
 
   def ParseCommandLine(self):
     """Parses the command line for bisect options."""
     parser = self._CreateCommandLineParser()
     (opts, args) = parser.parse_args()
 
     try:
       if not opts.command:
         raise RuntimeError('missing required parameter: --command')
 
       if not opts.good_revision:
         raise RuntimeError('missing required parameter: --good_revision')
 
       if not opts.bad_revision:
         raise RuntimeError('missing required parameter: --bad_revision')
 
       if not opts.metric:
         raise RuntimeError('missing required parameter: --metric')
 
       if opts.gs_bucket:
         if not cloud_storage.List(opts.gs_bucket):
-          raise RuntimeError('Invalid Google Storage URL: [%s]', e)
-
+          raise RuntimeError('Invalid Google Storage: gs://%s' % opts.gs_bucket)
+        if not opts.builder_host:
+          raise RuntimeError('Must specify try server hostname, when '
+                             'gs_bucket is used: --builder_host')
+        if not opts.builder_port:
+          raise RuntimeError('Must specify try server port number, when '
+                             'gs_bucket is used: --builder_port')
       if opts.target_platform == 'cros':
         # Run sudo up front to make sure credentials are cached for later.
         print 'Sudo is required to build cros:'
         print
         RunProcess(['sudo', 'true'])
 
         if not opts.cros_board:
           raise RuntimeError('missing required parameter: --cros_board')
 
         if not opts.cros_remote_ip:
@@ skipping 84 matching lines @@
 
     if not source_control:
       raise RuntimeError("Sorry, only the git workflow is supported at the "
           "moment.")
 
     # gClient sync seems to fail if you're not in master branch.
     if (not source_control.IsInProperBranch() and
         not opts.debug_ignore_sync and
         not opts.working_directory):
       raise RuntimeError("You must switch to master branch to run bisection.")
-
     bisect_test = BisectPerformanceMetrics(source_control, opts)
     try:
       bisect_results = bisect_test.Run(opts.command,
                                        opts.bad_revision,
                                        opts.good_revision,
                                        opts.metric)
       if bisect_results['error']:
         raise RuntimeError(bisect_results['error'])
       bisect_test.FormatAndPrintResults(bisect_results)
       return 0
     finally:
       bisect_test.PerformCleanup()
   except RuntimeError, e:
     if opts.output_buildbot_annotations:
       # The perf dashboard scrapes the "results" step in order to comment on
       # bugs. If you change this, please update the perf dashboard as well.
       bisect_utils.OutputAnnotationStepStart('Results')
     print 'Error: %s' % e.message
     if opts.output_buildbot_annotations:
       bisect_utils.OutputAnnotationStepClosed()
   return 1
 
 if __name__ == '__main__':
   sys.exit(main())