Make get_syzygy_binaries.py resistant to cloud failures.

build/get_syzygy_binaries.py

Index: build/get_syzygy_binaries.py
diff --git a/build/get_syzygy_binaries.py b/build/get_syzygy_binaries.py
index 2577c7cb7fde733b974ebf66e7c11fe1cd9777ba..011587186f784aba6eb782244840ed56ec532422 100755
--- a/build/get_syzygy_binaries.py
+++ b/build/get_syzygy_binaries.py
@@ -250,10 +250,23 @@ def _CleanState(output_dir, state, dry_run=False):
 
 def _Download(url):
   """Downloads the given URL and returns the contents as a string."""
-  response = urllib2.urlopen(url)
-  if response.code != 200:
-    raise RuntimeError('Failed to download "%s".' % url)
-  return response.read()
+  _LOGGER.debug('Downloading "%s".', url)
+  retries = 3
+  while retries > 0:
+    code = 0
+    try:
+      response = urllib2.urlopen(url)
+      if response.code == 200:
+        return response.read()
+      code = response.code
+    except urllib2.HTTPError, e:
+      code = e.code
+
+    _LOGGER.debug('Download failed with a %d response, retrying.',
+                  code)
+    retries -= 1

[Nico, 2015/07/10 15:04:39]

Trying again immediately likely won't work. Try sleeping 4s after the first try,
8s after the second or something like that (
https://en.wikipedia.org/wiki/Exponential_backoff etc)

[chrisha, 2015/07/14 17:51:38]

Err, yup. Meant to do that.

+
+  raise RuntimeError('Failed to download "%s".' % url)
 
 
 def _InstallBinaries(options, deleted={}):
@@ -278,13 +291,32 @@ def _InstallBinaries(options, deleted={}):
       if not options.dry_run:
         os.makedirs(fulldir)
 
-    # Download the archive.
-    url = archive_url + '/' + base
-    _LOGGER.debug('Retrieving %s archive at "%s".', name, url)
-    data = _Download(url)
-
-    _LOGGER.debug('Unzipping %s archive.', name)
-    archive = zipfile.ZipFile(cStringIO.StringIO(data))
+    # Download and unzip the archive. Try this a few times if it fails, as the
+    # cloud storage server occasionally flakes and sends incomplete data.
+    retries = 0
+    while True:
+      url = archive_url + '/' + base
+      _LOGGER.debug('Retrieving %s archive at "%s".', name, url)
+      data = _Download(url)
+
+      try:
+        _LOGGER.debug('Unzipping %s archive.', name)
+        archive = zipfile.ZipFile(cStringIO.StringIO(data))
+      except zipfile.BadZipfile, e:
+        # If retried too often then let the error continue.
+        retries += 1
+        if retries == 3:
+          raise e
+       
+        # Otherwise retry the download to see if the archive comes down clean
+        # this time.
+        _LOGGER.debug('Bad zip file, retrying download.')
+        continue
+
+      # Successfully unzipped the archive, so continue to extract files.
+      break
+
+    # Extract desired contents and install to disk.
     for entry in archive.infolist():
       if not filt or filt(entry):
         fullpath = os.path.normpath(os.path.join(fulldir, entry.filename))
@@ -316,6 +348,9 @@ def _ParseCommandLine():
       help='If true then will simply list actions that would be performed.')
   option_parser.add_option('--force', action='store_true', default=False,
       help='Force an installation even if the binaries are up to date.')
+  option_parser.add_option('--no-cleanup', action='store_true', default=False,
+      help='Allow installation on non-Windows platforms, and skip the forced '
+           'cleanup step.')
   option_parser.add_option('--output-dir', type='string',
       help='The path where the binaries will be replaced. Existing binaries '
            'will only be overwritten if not up to date.')
@@ -408,7 +443,10 @@ def main():
   # wasn't gated on OS types, and those OSes downloaded and installed binaries.
   # This will cleanup orphaned files on those operating systems.
   if sys.platform not in ('win32', 'cygwin'):
-    return _RemoveOrphanedFiles(options)
+    if options.no_cleanup:
+      _LOGGER.debug('Skipping usual cleanup for non-Windows platforms.')
+    else:
+      return _RemoveOrphanedFiles(options)
 
   # Load the current installation state, and validate it against the
   # requested installation.