Make get_syzygy_binaries.py resistant to cloud failures.

build/get_syzygy_binaries.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """A utility script for downloading versioned Syzygy binaries."""
 
-import cStringIO
 import hashlib
 import errno
 import json
 import logging
 import optparse
 import os
 import re
 import shutil
 import stat
 import sys
 import subprocess
-import urllib2
+import tempfile
+import time
 import zipfile
 
 
 _LOGGER = logging.getLogger(os.path.basename(__file__))
 
-# The URL where official builds are archived.
-_SYZYGY_ARCHIVE_URL = ('https://syzygy-archive.commondatastorage.googleapis.com'
-    '/builds/official/%(revision)s')
+# The relative path where official builds are archived in their GS bucket.
+_SYZYGY_ARCHIVE_PATH = ('/builds/official/%(revision)s')
 
 # A JSON file containing the state of the download directory. If this file and
 # directory state do not agree, then the binaries will be downloaded and
 # installed again.
 _STATE = '.state'
 
 # This matches an integer (an SVN revision number) or a SHA1 value (a GIT hash).
 # The archive exclusively uses lowercase GIT hashes.
 _REVISION_RE = re.compile('^(?:\d+|[a-f0-9]{40})$')
 
 # This matches an MD5 hash.
 _MD5_RE = re.compile('^[a-f0-9]{32}$')
 
 # List of reources to be downloaded and installed. These are tuples with the
 # following format:
 # (basename, logging name, relative installation path, extraction filter)
 _RESOURCES = [
   ('benchmark.zip', 'benchmark', '', None),
   ('binaries.zip', 'binaries', 'exe', None),
   ('symbols.zip', 'symbols', 'exe',
       lambda x: x.filename.endswith('.dll.pdb'))]
 
 
-def _Shell(*cmd, **kw):
-  """Runs |cmd|, returns the results from Popen(cmd).communicate()."""
-  _LOGGER.debug('Executing %s.', cmd)
-  prog = subprocess.Popen(cmd, shell=True, **kw)
-
-  stdout, stderr = prog.communicate()
-  if prog.returncode != 0:
-    raise RuntimeError('Command "%s" returned %d.' % (cmd, prog.returncode))
-  return (stdout, stderr)
-
-
 def _LoadState(output_dir):
   """Loads the contents of the state file for a given |output_dir|, returning
   None if it doesn't exist.
   """
   path = os.path.join(output_dir, _STATE)
   if not os.path.exists(path):
     _LOGGER.debug('No state file found.')
     return None
   with open(path, 'rb') as f:
     _LOGGER.debug('Reading state file: %s', path)
@@ skipping 168 matching lines @@
   dirs = sorted(dirs.keys(), key=lambda x: len(x), reverse=True)
   for p in dirs:
     if os.path.exists(p) and _DirIsEmpty(p):
       _LOGGER.debug('Deleting empty directory "%s".', p)
       if not dry_run:
         _RmTree(p)
 
   return deleted
 
 
-def _Download(url):
-  """Downloads the given URL and returns the contents as a string."""
-  response = urllib2.urlopen(url)
-  if response.code != 200:
-    raise RuntimeError('Failed to download "%s".' % url)
-  return response.read()
+def _FindGsUtil():
+  """Looks for depot_tools and returns the absolute path to gsutil.py."""
+  for path in os.environ['PATH'].split(';'):

[Nico, 2015/07/14 17:58:41]

os.pathsep instead of ;

[chrisha, 2015/07/14 18:05:06]

Done.

+    path = os.path.abspath(path)
+    git_cl = os.path.join(path, 'git_cl.py')
+    gs_util = os.path.join(path, 'gsutil.py')
+    if os.path.exists(git_cl) and os.path.exists(gs_util):
+      return gs_util
+  return None
+
+
+def _GsUtil(*cmd):
+  """Runs the given command in gsutil with exponential backoff and retries."""
+  gs_util = _FindGsUtil()
+  cmd = [sys.executable, gs_util] + list(cmd)
+
+  retries = 3
+  timeout = 4  # Seconds.
+  while True:
+    _LOGGER.debug('Running %s', cmd)
+    prog = subprocess.Popen(cmd, shell=True)
+    prog.communicate()
+
+    # Stop retrying on success.
+    if prog.returncode == 0:
+      return
+
+    # Raise a permanent failure if retries have been exhausted.
+    if retries == 0:
+      raise RuntimeError('Command "%s" returned %d.' % (cmd, prog.returncode))
+
+    _LOGGER.debug('Sleeping %d seconds and trying again.', timeout)
+    time.sleep(timeout)
+    retries -= 1
+    timeout *= 2
+
+
+def _Download(resource):
+  """Downloads the given GS resource to a temporary file, returning its path."""
+  tmp = tempfile.mkstemp(suffix='syzygy_archive')
+  os.close(tmp[0])
+  url = 'gs://syzygy-archive' + resource
+  _GsUtil('cp', url, tmp[1])
+  return tmp[1]
 
 
 def _InstallBinaries(options, deleted={}):
   """Installs Syzygy binaries. This assumes that the output directory has
   already been cleaned, as it will refuse to overwrite existing files."""
   contents = {}
   state = { 'revision': options.revision, 'contents': contents }
-  archive_url = _SYZYGY_ARCHIVE_URL % { 'revision': options.revision }
+  archive_path = _SYZYGY_ARCHIVE_PATH % { 'revision': options.revision }
   if options.resources:
     resources = [(resource, resource, '', None)
                  for resource in options.resources]
   else:
     resources = _RESOURCES
   for (base, name, subdir, filt) in resources:
     # Create the output directory if it doesn't exist.
     fulldir = os.path.join(options.output_dir, subdir)
     if os.path.isfile(fulldir):
       raise Exception('File exists where a directory needs to be created: %s' %
                       fulldir)
     if not os.path.exists(fulldir):
       _LOGGER.debug('Creating directory: %s', fulldir)
       if not options.dry_run:
         os.makedirs(fulldir)
 
-    # Download the archive.
-    url = archive_url + '/' + base
-    _LOGGER.debug('Retrieving %s archive at "%s".', name, url)
-    data = _Download(url)
+    # Download and read the archive.
+    resource = archive_path + '/' + base
+    _LOGGER.debug('Retrieving %s archive at "%s".', name, resource)
+    path = _Download(resource)
 
     _LOGGER.debug('Unzipping %s archive.', name)
-    archive = zipfile.ZipFile(cStringIO.StringIO(data))
+    archive = zipfile.ZipFile(open(path, 'rb'))

[Nico, 2015/07/14 17:58:41]

nit: `with open(path, 'rb') as path:`, then you don't need the explicit close
call

[chrisha, 2015/07/14 18:05:06]

Done.

     for entry in archive.infolist():
       if not filt or filt(entry):
         fullpath = os.path.normpath(os.path.join(fulldir, entry.filename))
         relpath = os.path.relpath(fullpath, options.output_dir)
         if os.path.exists(fullpath):
           # If in a dry-run take into account the fact that the file *would*
           # have been deleted.
           if options.dry_run and relpath in deleted:
             pass
           else:
             raise Exception('Path already exists: %s' % fullpath)
 
         # Extract the file and update the state dictionary.
         _LOGGER.debug('Extracting "%s".', fullpath)
         if not options.dry_run:
           archive.extract(entry.filename, fulldir)
           md5 = _Md5(fullpath)
           contents[relpath] = md5
           if sys.platform == 'cygwin':
             os.chmod(fullpath, os.stat(fullpath).st_mode | stat.S_IXUSR)
 
+    _LOGGER.debug('Removing temporary file "%s".', path)
+    archive.close()
+    os.remove(path)
+
   return state
 
 
 def _ParseCommandLine():
   """Parses the command-line and returns an options structure."""
   option_parser = optparse.OptionParser()
   option_parser.add_option('--dry-run', action='store_true', default=False,
       help='If true then will simply list actions that would be performed.')
   option_parser.add_option('--force', action='store_true', default=False,
       help='Force an installation even if the binaries are up to date.')
+  option_parser.add_option('--no-cleanup', action='store_true', default=False,
+      help='Allow installation on non-Windows platforms, and skip the forced '
+           'cleanup step.')
   option_parser.add_option('--output-dir', type='string',
       help='The path where the binaries will be replaced. Existing binaries '
            'will only be overwritten if not up to date.')
   option_parser.add_option('--overwrite', action='store_true', default=False,
       help='If specified then the installation will happily delete and rewrite '
            'the entire output directory, blasting any local changes.')
   option_parser.add_option('--revision', type='string',
       help='The SVN revision or GIT hash associated with the required version.')
   option_parser.add_option('--revision-file', type='string',
       help='A text file containing an SVN revision or GIT hash.')
@@ skipping 72 matching lines @@
   options = _ParseCommandLine()
 
   if options.dry_run:
     _LOGGER.debug('Performing a dry-run.')
 
   # We only care about Windows platforms, as the Syzygy binaries aren't used
   # elsewhere. However, there was a short period of time where this script
   # wasn't gated on OS types, and those OSes downloaded and installed binaries.
   # This will cleanup orphaned files on those operating systems.
   if sys.platform not in ('win32', 'cygwin'):
-    return _RemoveOrphanedFiles(options)
+    if options.no_cleanup:
+      _LOGGER.debug('Skipping usual cleanup for non-Windows platforms.')
+    else:
+      return _RemoveOrphanedFiles(options)
 
   # Load the current installation state, and validate it against the
   # requested installation.
   state, is_consistent = _GetCurrentState(options.revision, options.output_dir)
 
   # Decide whether or not an install is necessary.
   if options.force:
     _LOGGER.debug('Forcing reinstall of binaries.')
   elif is_consistent:
     # Avoid doing any work if the contents of the directory are consistent.
@@ skipping 20 matching lines @@
   # Install the new binaries. In a dry-run this will actually download the
   # archives, but it won't write anything to disk.
   state = _InstallBinaries(options, deleted)
 
   # Build and save the state for the directory.
   _SaveState(options.output_dir, state, options.dry_run)
 
 
 if __name__ == '__main__':
   main()