Add retry logic to install-sysroot.py

build/linux/sysroot_scripts/install-sysroot.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Install Debian Wheezy sysroots for building chromium.
 """
 
 # The sysroot is needed to ensure that binaries will run on Debian Wheezy,
 # the oldest supported linux distribution.  This script can be run manually but
 # is more often run as part of gclient hooks. When run from hooks this script
 # in a no-op on non-linux platforms.
 
 # The sysroot image could be constructed from scratch based on the current
 # state or Debian Wheezy but for consistency we currently use a pre-built root
 # image. The image will normally need to be rebuilt every time chrome's build
 # dependencies are changed.
 
 import hashlib
 import platform
 import optparse
 import os
 import re
 import shutil
 import subprocess
 import sys
+import time
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 sys.path.append(os.path.dirname(os.path.dirname(SCRIPT_DIR)))
 import detect_host_arch
 import gyp_chromium
 import gyp_environment
 
 
 URL_PREFIX = 'http://commondatastorage.googleapis.com'
 URL_PATH = 'chrome-linux-sysroot/toolchain'
@@ skipping 91 matching lines @@
 
   # Finally, if we can detect a non-standard target_arch such as ARM or
   # MIPS, then install the sysroot too.
   # Don't attampt to install arm64 since this is currently and android-only
   # architecture.
   target_arch = DetectTargetArch()
   if target_arch and target_arch not in (host_arch, 'i386', 'arm64'):
     InstallSysroot(target_arch)
 
 
-def main():
+def main(args):
+  parser = optparse.OptionParser('usage: %prog [OPTIONS]', description=__doc__)
+  parser.add_option('--running-as-hook', action='store_true',
+                    default=False, help='Used when running from gclient hooks.'
+                                        ' Installs default sysroot images.')
+  parser.add_option('--arch', type='choice', choices=valid_archs,
+                    help='Sysroot architecture: %s' % ', '.join(valid_archs))
+  options, _ = parser.parse_args(args)
   if options.running_as_hook and not sys.platform.startswith('linux'):
     return 0
 
   if options.running_as_hook:
     InstallDefaultSysroots()
   else:
     if not options.arch:
-      print 'You much specify either --arch or --running-as-hook'
-      return 1
+      raise Error('You much specify either --arch or --running-as-hook')
     InstallSysroot(options.arch)
 
   return 0
 
 
+def DownloadURL(url, filename):
+  print 'Downloading %s' % url
+  sys.stdout.flush()
+  sys.stderr.flush()
+
+  # Re-try download in order to mitigate network falkiness.

[Lei Zhang, 2016/02/17 20:49:51]

typo

+  num_retries = 3
+  retry_wait = 5  # Doubled at each retry.
+  cmd = ['curl', '--fail', '-L', url, '-o', filename]
+  while True:
+    rtn = subprocess.call(cmd)
+    if rtn == 0:
+      return
+    elif num_retries:

[Lei Zhang, 2016/02/17 20:49:51]

no elif / else after a return.

+      # Curl failed.  Retry
+      print 'Retrying in %d s ...' % retry_wait
+      time.sleep(retry_wait)
+      num_retries -= 1
+      retry_wait *= 2
+    else:
+      raise Error('download command failed (%s): %s' % (rtn, cmd))

[Lei Zhang, 2016/02/17 20:49:51]

%s seems to work, but shouldn't it be %d since |rtn| is an int?

+
+
 def InstallSysroot(target_arch):
   # The sysroot directory should match the one specified in build/common.gypi.
   # TODO(thestig) Consider putting this else where to avoid having to recreate
   # it on every build.
   linux_dir = os.path.dirname(SCRIPT_DIR)
   if target_arch == 'amd64':
     sysroot = os.path.join(linux_dir, SYSROOT_DIR_AMD64)
     tarball_filename = TARBALL_AMD64
     tarball_sha1sum = TARBALL_AMD64_SHA1SUM
     revision = REVISION_AMD64
@@ skipping 23 matching lines @@
       if s.read() == url:
         print 'Debian Wheezy %s root image already up-to-date: %s' % \
             (target_arch, sysroot)
         return
 
   print 'Installing Debian Wheezy %s root image: %s' % (target_arch, sysroot)
   if os.path.isdir(sysroot):
     shutil.rmtree(sysroot)
   os.mkdir(sysroot)
   tarball = os.path.join(sysroot, tarball_filename)
-  print 'Downloading %s' % url
-  sys.stdout.flush()
-  sys.stderr.flush()
-  subprocess.check_call(['curl', '--fail', '-L', url, '-o', tarball])
+  DownloadURL(url, tarball)
   sha1sum = GetSha1(tarball)
   if sha1sum != tarball_sha1sum:
     raise Error('Tarball sha1sum is wrong.'
                 'Expected %s, actual: %s' % (tarball_sha1sum, sha1sum))
   subprocess.check_call(['tar', 'xf', tarball, '-C', sysroot])
   os.remove(tarball)
 
   with open(stamp, 'w') as s:
     s.write(url)
 
 
 if __name__ == '__main__':
-  parser = optparse.OptionParser('usage: %prog [OPTIONS]', description=__doc__)
-  parser.add_option('--running-as-hook', action='store_true',
-                    default=False, help='Used when running from gclient hooks.'
-                                        ' Installs default sysroot images.')
-  parser.add_option('--arch', type='choice', choices=valid_archs,
-                    help='Sysroot architecture: %s' % ', '.join(valid_archs))
-  options, _ = parser.parse_args()
-  sys.exit(main())
+  try:
+    sys.exit(main(sys.argv[1:]))
+  except Error as e:
+    sys.stderr.write(str(e) + '\n')
+    sys.exit(1)