Download archives contain git or svn revisions in the filename.

tools/bisect-perf-regression.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Performance Test Bisect Tool
 
 This script bisects a series of changelists using binary search. It starts at
 a bad revision where a performance metric has regressed, and asks for a last
 known-good revision. It will then binary search across this revision range by
@@ skipping 464 matching lines @@
 
 def FetchFromCloudStorage(bucket_name, source_path, destination_path):
   """Fetches file(s) from the Google Cloud Storage.
 
   Args:
     bucket_name: Google Storage bucket name.
     source_path: Source file path.
     destination_path: Destination file path.
 
   Returns:
-    True if the fetching succeeds, otherwise False.
+    Downloaded file path if exisits, otherwise None.
   """
   target_file = os.path.join(destination_path, os.path.basename(source_path))
   try:
     if cloud_storage.Exists(bucket_name, source_path):
       print 'Fetching file from gs//%s/%s ...' % (bucket_name, source_path)
       cloud_storage.Get(bucket_name, source_path, destination_path)
       if os.path.exists(target_file):
-        return True
+        return target_file
     else:
       print ('File gs://%s/%s not found in cloud storage.' % (
           bucket_name, source_path))
   except Exception as e:
     print 'Something went wrong while fetching file from cloud: %s' % e
     if os.path.exists(target_file):
       os.remove(target_file)
-  return False
+  return None
 
 
 # This is copied from Chromium's project build/scripts/common/chromium_utils.py.
 def MaybeMakeDirectory(*path):
   """Creates an entire path, if it doesn't already exist."""
   file_path = os.path.join(*path)
   try:
     os.makedirs(file_path)
   except OSError, e:
     if e.errno != errno.EEXIST:
@@ skipping 1034 matching lines @@
     source_dir = os.path.join(build_dir, build_type)
     destination_dir = os.path.join(build_dir, '%s.bak' % build_type)
     if restore:
       source_dir, destination_dir = destination_dir, source_dir
     if os.path.exists(source_dir):
       RmTreeAndMkDir(destination_dir, skip_makedir=True)
       shutil.move(source_dir, destination_dir)
       return destination_dir
     return None
 
+  def GetBuildArchiveForRevision(self, revision, gs_bucket, target_arch,
+                                 patch_sha, out_dir):
+    """Checks and downloads build archive for a given revision.
+
+    Checks for build archive with Git hash or SVN revision. If either of the
+    file exists, then downloads the archive file.
+
+    Args:
+      revision: A Git hash revision.
+      gs_bucket: Cloud storage bucket name
+      target_arch: 32 or 64 bit build target
+      patch: A DEPS patch (used while bisecting 3rd party repositories).
+      out_dir: Build output directory where downloaded file is stored.
+
+    Returns:
+      Downloaded archive file path if exists, otherwise None.
+    """
+    # Source archive file path on cloud storage using Git revision.
+    source_file = GetRemoteBuildPath(revision, target_arch, patch_sha)
+    downloaded_archive = FetchFromCloudStorage(gs_bucket, source_file, out_dir)
+    if not downloaded_archive:
+      # Get SVN revision for the given SHA.
+      svn_revision = self.source_control.SVNFindRev(revision)
+      if svn_revision:
+        # Source archive file path on cloud storage using SVN revision.
+        source_file = GetRemoteBuildPath(svn_revision, target_arch, patch_sha)
+        return FetchFromCloudStorage(gs_bucket, source_file, out_dir)
+    return downloaded_archive
+
   def DownloadCurrentBuild(self, revision, build_type='Release', patch=None):
     """Downloads the build archive for the given revision.
 
     Args:
-      revision: The SVN revision to build.
+      revision: The Git revision to download or build.
       build_type: Target build type ('Release', 'Debug', 'Release_x64' etc.)
+      patch: A DEPS patch (used while bisecting 3rd party repositories).
 
     Returns:
       True if download succeeds, otherwise False.
     """
     patch_sha = None
     if patch:
       # Get the SHA of the DEPS changes patch.
       patch_sha = GetSHA1HexDigest(patch)
 
       # Update the DEPS changes patch with a patch to create a new file named
       # 'DEPS.sha' and add patch_sha evaluated above to it.
       patch = '%s\n%s' % (patch, DEPS_SHA_PATCH % {'deps_sha': patch_sha})
 
-    # Source archive file path on cloud storage.
-    source_file = GetRemoteBuildPath(revision, self.opts.target_arch, patch_sha)
-
     # Get Build output directory
     abs_build_dir = os.path.abspath(
         self.builder.GetBuildOutputDirectory(self.opts, self.src_cwd))
-    # Downloaded archive file path.
-    downloaded_file = os.path.join(
-        abs_build_dir,
-        GetZipFileName(revision, self.opts.target_arch, patch_sha))
 
-    fetch_build_func = lambda: FetchFromCloudStorage(self.opts.gs_bucket,
-                                                     source_file,
-                                                     abs_build_dir)
+    fetch_build_func = lambda: self.GetBuildArchiveForRevision(
+      revision, self.opts.gs_bucket, self.opts.target_arch,
+      patch_sha, abs_build_dir)
 
-    if not fetch_build_func():
-      if not self.PostBuildRequestAndWait(revision,
-                                          fetch_build=fetch_build_func,
-                                          patch=patch):
-        raise RuntimeError('Somewthing went wrong while processing build'
-                           'request for: %s' % revision)
+    # Downloaded archive file path, downloads build archive for given revision.
+    downloaded_file = fetch_build_func()
+
+    # When build archive doesn't exists, post a build request to tryserver
+    # and wait for the build to be produced.
+    if not downloaded_file:
+      downloaded_file = self.PostBuildRequestAndWait(
+          revision, fetch_build=fetch_build_func, patch=patch)
+      if not downloaded_file:
+        return False
+
     # Generic name for the archive, created when archive file is extracted.
     output_dir = os.path.join(
         abs_build_dir, GetZipFileName(target_arch=self.opts.target_arch))
     # Unzip build archive directory.
     try:
       RmTreeAndMkDir(output_dir, skip_makedir=True)
       ExtractZip(downloaded_file, abs_build_dir)
       if os.path.exists(output_dir):
         self.BackupOrRestoreOutputdirectory(restore=False)
         # Build output directory based on target(e.g. out/Release, out/Debug).
@@ skipping 21 matching lines @@
 
     Args:
       fetch_build: Function to check and download build from cloud storage.
       bot_name: Builder bot name on tryserver.
       builder_host Tryserver hostname.
       builder_port: Tryserver port.
       build_request_id: A unique ID of the build request posted to tryserver.
       max_timeout: Maximum time to wait for the build.
 
     Returns:
-      True if build exists and download is successful, otherwise throws
-      RuntimeError exception when time elapse.
+       Downloaded archive file path if exists, otherwise None.
     """
     # Build number on the tryserver.
     build_num = None
     # Interval to check build on cloud storage.
     poll_interval = 60
     # Interval to check build status on tryserver.
     status_check_interval = 600
     last_status_check = time.time()
     start_time = time.time()
     while True:
       # Checks for build on gs://chrome-perf and download if exists.
       res = fetch_build()
       if res:
         return (res, 'Build successfully found')
       elapsed_status_check = time.time() - last_status_check
       # To avoid overloading tryserver with status check requests, we check
       # build status for every 10 mins.
       if elapsed_status_check > status_check_interval:
         last_status_check = time.time()
         if not build_num:
           # Get the build number on tryserver for the current build.
           build_num = bisect_builder.GetBuildNumFromBuilder(
               build_request_id, bot_name, builder_host, builder_port)
         # Check the status of build using the build number.
         # Note: Build is treated as PENDING if build number is not found
         # on the the tryserver.
         build_status, status_link = bisect_builder.GetBuildStatus(
             build_num, bot_name, builder_host, builder_port)
         if build_status == bisect_builder.FAILED:
-          return (False, 'Failed to produce build, log: %s' % status_link)
+          return (None, 'Failed to produce build, log: %s' % status_link)
       elapsed_time = time.time() - start_time
       if elapsed_time > max_timeout:
-        return (False, 'Timed out: %ss without build' % max_timeout)
+        return (None, 'Timed out: %ss without build' % max_timeout)
 
       print 'Time elapsed: %ss without build.' % elapsed_time
       time.sleep(poll_interval)
 
   def PostBuildRequestAndWait(self, revision, fetch_build, patch=None):
-    """POSTs the build request job to the tryserver instance."""
+    """POSTs the build request job to the tryserver instance.
+
+    A try job build request is posted to tryserver.chromium.perf master,
+    and waits for the binaries to be produced and archived on cloud storage.
+    Once the build is ready and stored onto cloud, build archive is downloaded
+    into the output folder.
+
+    Args:
+      revision: A Git hash revision.
+      fetch_build: Function to check and download build from cloud storage.
+      patch: A DEPS patch (used while bisecting 3rd party repositories).
+
+    Returns:
+      Downloaded archive file path when requested build exists and download is
+      successful, otherwise None.
+    """
+    # Get SVN revision for the given SHA.
+    svn_revision = self.source_control.SVNFindRev(revision)
+    if not svn_revision:
+      raise RuntimeError(
+          'Failed to determine SVN revision for %s' % revision)
 
     def GetBuilderNameAndBuildTime(target_arch='ia32'):
       """Gets builder bot name and buildtime in seconds based on platform."""
       # Bot names should match the one listed in tryserver.chromium's
       # master.cfg which produces builds for bisect.
       if IsWindows():
         if Is64BitWindows() and target_arch == 'x64':
           return ('win_perf_bisect_builder', MAX_WIN_BUILD_TIME)
         return ('win_perf_bisect_builder', MAX_WIN_BUILD_TIME)
       if IsLinux():
         return ('linux_perf_bisect_builder', MAX_LINUX_BUILD_TIME)
       if IsMac():
         return ('mac_perf_bisect_builder', MAX_MAC_BUILD_TIME)
       raise NotImplementedError('Unsupported Platform "%s".' % sys.platform)
     if not fetch_build:
       return False
 
     bot_name, build_timeout = GetBuilderNameAndBuildTime(self.opts.target_arch)
     builder_host = self.opts.builder_host
     builder_port = self.opts.builder_port
     # Create a unique ID for each build request posted to tryserver builders.
     # This ID is added to "Reason" property in build's json.
     build_request_id = GetSHA1HexDigest(
-        '%s-%s-%s' % (revision, patch, time.time()))
+        '%s-%s-%s' % (svn_revision, patch, time.time()))
 
     # Creates a try job description.
     job_args = {'host': builder_host,
                 'port': builder_port,
-                'revision': 'src@%s' % revision,
+                'revision': 'src@%s' % svn_revision,
                 'bot': bot_name,
                 'name': build_request_id
                }
     # Update patch information if supplied.
     if patch:
       job_args['patch'] = patch
     # Posts job to build the revision on the server.
     if bisect_builder.PostTryJob(job_args):
-      status, error_msg = self.WaitUntilBuildIsReady(fetch_build,
-                                                     bot_name,
-                                                     builder_host,
-                                                     builder_port,
-                                                     build_request_id,
-                                                     build_timeout)
-      if not status:
-        raise RuntimeError('%s [revision: %s]' % (error_msg, revision))
-      return True
-    return False
+      target_file, error_msg = self.WaitUntilBuildIsReady(fetch_build,
+                                                          bot_name,
+                                                          builder_host,
+                                                          builder_port,
+                                                          build_request_id,
+                                                          build_timeout)
+      if not target_file:
+        print '%s [revision: %s]' % (error_msg, svn_revision)
+        return None
+      return target_file
+    print 'Failed to post build request for revision: [%s]' % svn_revision
+    return None
 
   def IsDownloadable(self, depot):
     """Checks if build is downloadable based on target platform and depot."""
     if self.opts.target_platform in ['chromium'] and self.opts.gs_bucket:
       return (depot == 'chromium' or
               'chromium' in DEPOT_DEPS_NAME[depot]['from'] or
               'v8' in DEPOT_DEPS_NAME[depot]['from'])
     return False
 
   def UpdateDeps(self, revision, depot, deps_file):
@@ skipping 179 matching lines @@
       return True
     cwd = os.getcwd()
     os.chdir(self.src_cwd)
     # Fetch build archive for the given revision from the cloud storage when
     # the storage bucket is passed.
     if self.IsDownloadable(depot) and revision:
       deps_patch = None
       if depot != 'chromium':
         # Create a DEPS patch with new revision for dependency repository.
         (revision, deps_patch) = self.CreateDEPSPatch(depot, revision)
-      # Get SVN revision for the given SHA, since builds are archived using SVN
-      # revision.
-      chromium_revision = self.source_control.SVNFindRev(revision)
-      if not chromium_revision:
-        raise RuntimeError(
-            'Failed to determine SVN revision for %s' % revision)
-      if self.DownloadCurrentBuild(chromium_revision, patch=deps_patch):
+      if self.DownloadCurrentBuild(revision, patch=deps_patch):
         os.chdir(cwd)
         if deps_patch:
           # Reverts the changes to DEPS file.
           self.source_control.CheckoutFileAtRevision(bisect_utils.FILE_DEPS,
                                                      revision,
                                                      cwd=self.src_cwd)
         return True
-      raise RuntimeError('Failed to download build archive for revision %s.\n'
-                         'Unfortunately, bisection couldn\'t continue any '
-                         'further. Please try running script without '
-                         '--gs_bucket flag to produce local builds.' % revision)
+      return False
 
-
+    # These codes are executed when bisect bots builds binaries locally.
     build_success = self.builder.Build(depot, self.opts)
     os.chdir(cwd)
     return build_success
 
   def RunGClientHooks(self):
     """Runs gclient with runhooks command.
 
     Returns:
       True if gclient reports no errors.
     """
@@ skipping 2023 matching lines @@
       # The perf dashboard scrapes the "results" step in order to comment on
       # bugs. If you change this, please update the perf dashboard as well.
       bisect_utils.OutputAnnotationStepStart('Results')
     print 'Error: %s' % e.message
     if opts.output_buildbot_annotations:
       bisect_utils.OutputAnnotationStepClosed()
   return 1
 
 if __name__ == '__main__':
   sys.exit(main())