Check build status by parsing tryserver buildbot JSON.

tools/bisect-perf-regression.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Performance Test Bisect Tool
 
 This script bisects a series of changelists using binary search. It starts at
 a bad revision where a performance metric has regressed, and asks for a last
 known-good revision. It will then binary search across this revision range by
@@ skipping 36 matching lines @@
 import shutil
 import StringIO
 import subprocess
 import sys
 import time
 import zipfile
 
 sys.path.append(os.path.join(os.path.dirname(__file__), 'telemetry'))
 
 import bisect_utils
-import post_perf_builder_job
+import post_perf_builder_job as bisect_builder
 from telemetry.page import cloud_storage
 
 # The additional repositories that might need to be bisected.
 # If the repository has any dependant repositories (such as skia/src needs
 # skia/include and skia/gyp to be updated), specify them in the 'depends'
 # so that they're synced appropriately.
 # Format is:
 # src: path to the working directory.
 # recurse: True if this repositry will get bisected.
 # depends: A list of other repositories that are actually part of the same
@@ skipping 1509 matching lines @@
     downloaded_file = os.path.join(
         abs_build_dir,
         GetZipFileName(revision, self.opts.target_arch, patch_sha))
 
     fetch_build_func = lambda: FetchFromCloudStorage(self.opts.gs_bucket,
                                                      source_file,
                                                      abs_build_dir)
 
     if not fetch_build_func():
       if not self.PostBuildRequestAndWait(revision,
-                                          condition=fetch_build_func,
+                                          fetch_build=fetch_build_func,
                                           patch=patch):
         raise RuntimeError('Somewthing went wrong while processing build'
                            'request for: %s' % revision)
     # Generic name for the archive, created when archive file is extracted.
     output_dir = os.path.join(
         abs_build_dir, GetZipFileName(target_arch=self.opts.target_arch))
     # Unzip build archive directory.
     try:
       RmTreeAndMkDir(output_dir, skip_makedir=True)
       ExtractZip(downloaded_file, abs_build_dir)
@@ skipping 11 matching lines @@
       self.BackupOrRestoreOutputdirectory(restore=True)
       # Cleanup any leftovers from unzipping.
       if os.path.exists(output_dir):
         RmTreeAndMkDir(output_dir, skip_makedir=True)
     finally:
       # Delete downloaded archive
       if os.path.exists(downloaded_file):
         os.remove(downloaded_file)
     return False
 
-  def PostBuildRequestAndWait(self, revision, condition, patch=None):
+  def WaitUntilBuildIsReady(self, fetch_build, bot_name, builder_host,
+                            builder_port, build_request_id, max_timeout):
+    """Waits until build is produced by bisect builder on tryserver.
+
+    Args:
+      fetch_build: Function to check and download build from cloud storage.
+      bot_name: Builder bot name on tryserver.
+      builder_host Tryserver hostname.
+      builder_port: Tryserver port.
+      build_request_id: A unique ID of the build request posted to tryserver.
+      max_timeout: Maximum time to wait for the build.
+
+    Returns:
+      True if build exists and download is successful, otherwise throws
+      RuntimeError exception when time elapse.
+    """
+    # Build number on the tryserver.
+    build_num = None
+    # Interval to check build on cloud storage.
+    poll_interval = 60
+    # Interval to check build status on tryserver.
+    status_check_interval = 600
+    last_status_check = time.time()
+    start_time = time.time()
+    while True:
+      # Checks for build on gs://chrome-perf and download if exists.
+      res = fetch_build()
+      if res:
+        return (res, 'Build successfully found')
+      elapsed_status_check = time.time() - last_status_check
+      # To avoid overloading tryserver with status check requests, we check
+      # build status for every 10 mins.
+      if elapsed_status_check > status_check_interval:
+        last_status_check = time.time()
+        if not build_num:
+          # Get the build number on tryserver for the current build.
+          build_num = bisect_builder.GetBuildNumFromBuilder(
+              build_request_id, bot_name, builder_host, builder_port)
+        # Check the status of build using the build number.
+        # Note: Build is treated as PENDING if build number is not found
+        # on the the tryserver.
+        build_status, status_link = bisect_builder.GetBuildStatus(
+            build_num, bot_name, builder_host, builder_port)
+        if build_status == bisect_builder.FAILED:
+          return (False, 'Failed to produce build, log: %s' % status_link)
+      elapsed_time = time.time() - start_time
+      if elapsed_time > max_timeout:
+        return (False, 'Timed out: %ss without build' % max_timeout)
+
+      print 'Time elapsed: %ss without build.' % elapsed_time
+      time.sleep(poll_interval)
+
+  def PostBuildRequestAndWait(self, revision, fetch_build, patch=None):
     """POSTs the build request job to the tryserver instance."""
 
     def GetBuilderNameAndBuildTime(target_arch='ia32'):
       """Gets builder bot name and buildtime in seconds based on platform."""
       # Bot names should match the one listed in tryserver.chromium's
       # master.cfg which produces builds for bisect.
       if IsWindows():
         if Is64BitWindows() and target_arch == 'x64':
           return ('win_perf_bisect_builder', MAX_WIN_BUILD_TIME)
         return ('win_perf_bisect_builder', MAX_WIN_BUILD_TIME)
       if IsLinux():
         return ('linux_perf_bisect_builder', MAX_LINUX_BUILD_TIME)
       if IsMac():
         return ('mac_perf_bisect_builder', MAX_MAC_BUILD_TIME)
       raise NotImplementedError('Unsupported Platform "%s".' % sys.platform)
-    if not condition:
+    if not fetch_build:
       return False
 
     bot_name, build_timeout = GetBuilderNameAndBuildTime(self.opts.target_arch)
-
-    # Create a unique ID for each build request posted to try server builders.
+    builder_host = self.opts.builder_host
+    builder_port = self.opts.builder_port
+    # Create a unique ID for each build request posted to tryserver builders.
     # This ID is added to "Reason" property in build's json.
-    # TODO: Use this id to track the build status.
-    build_request_id = GetSHA1HexDigest('%s-%s' % (revision, patch))
+    build_request_id = GetSHA1HexDigest(
+        '%s-%s-%s' % (revision, patch, time.time()))
 
     # Creates a try job description.
-    job_args = {'host': self.opts.builder_host,
-                'port': self.opts.builder_port,
+    job_args = {'host': builder_host,
+                'port': builder_port,
                 'revision': 'src@%s' % revision,
                 'bot': bot_name,
                 'name': build_request_id
                }
     # Update patch information if supplied.
     if patch:
       job_args['patch'] = patch
     # Posts job to build the revision on the server.
-    if post_perf_builder_job.PostTryJob(job_args):
-      poll_interval = 60
-      start_time = time.time()
-      while True:
-        res = condition()
-        if res:
-          return res
-        elapsed_time = time.time() - start_time
-        if elapsed_time > build_timeout:
-          raise RuntimeError('Timed out while waiting %ds for %s build.' %
-                             (build_timeout, revision))
-        print ('Time elapsed: %ss, still waiting for %s build' %
-               (elapsed_time, revision))
-        time.sleep(poll_interval)
+    if bisect_builder.PostTryJob(job_args):
+      status, error_msg = self.WaitUntilBuildIsReady(fetch_build,
+                                                     bot_name,
+                                                     builder_host,
+                                                     builder_port,
+                                                     build_request_id,
+                                                     build_timeout)
+      if not status:
+        raise RuntimeError('%s [revision: %s]' % (error_msg, revision))
+      return True
     return False
 
   def IsDownloadable(self, depot):
     """Checks if build is downloadable based on target platform and depot."""
     if self.opts.target_platform in ['chromium'] and self.opts.gs_bucket:
       return (depot == 'chromium' or
               'chromium' in DEPOT_DEPS_NAME[depot]['from'] or
               'v8' in DEPOT_DEPS_NAME[depot]['from'])
     return False
 
@@ skipping 2234 matching lines @@
       # The perf dashboard scrapes the "results" step in order to comment on
       # bugs. If you change this, please update the perf dashboard as well.
       bisect_utils.OutputAnnotationStepStart('Results')
     print 'Error: %s' % e.message
     if opts.output_buildbot_annotations:
       bisect_utils.OutputAnnotationStepClosed()
   return 1
 
 if __name__ == '__main__':
   sys.exit(main())