add a mechanism to define clang tarball mirrors

tools/clang/scripts/update.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """This script is used to download prebuilt clang binaries.
 
 It is also used by package.py to build the prebuilt clang binaries."""
 
 import argparse
@@ skipping 52 matching lines @@
 LLVM_BUILD_TOOLS_DIR = os.path.abspath(
     os.path.join(LLVM_DIR, '..', 'llvm-build-tools'))
 STAMP_FILE = os.path.normpath(
     os.path.join(LLVM_DIR, '..', 'llvm-build', 'cr_build_revision'))
 BINUTILS_DIR = os.path.join(THIRD_PARTY_DIR, 'binutils')
 VERSION = '3.9.0'
 ANDROID_NDK_DIR = os.path.join(
     CHROMIUM_DIR, 'third_party', 'android_tools', 'ndk')
 
 # URL for pre-built binaries.
 CDS_URL = 'https://commondatastorage.googleapis.com/chromium-browser-clang'

[Nico, 2016/02/15 15:13:47]

Do you really need all this code? Isn't

 CDS_URL = os.environ.get('CDS_CLANG_BUCKET_OVERRIDE',
'https://commondatastorage.googleapis.com/chromium-browser-clang')

in this line sufficient?

[Mostyn Bramley-Moore, 2016/02/15 16:14:06]

I implemented this using a local patch that was hastily cobbled together as a
reference.  For this file, I think your much simpler suggestion makes a lot of
sense.

There are a bunch of other googleapis.com URLs referenced in the tree, which
might require similar patches, in which case some consolidation might be
worthwhile.  But we can worry about that later if it turns out to be a problem. 

 
 LLVM_REPO_URL='https://llvm.org/svn/llvm-project'
 if 'LLVM_REPO_URL' in os.environ:
   LLVM_REPO_URL = os.environ['LLVM_REPO_URL']
 
+# MIRROR_MAP is an optional environment variable, which contains zero or more
+# mappings from a URL prefix to one or more mirrors.  A mapping from a URL
+# prefix to its mirrors starts with the URL prefix and is followed by a pipe
+# char and a mirror prefix, additional mirrors are separated by pipe
+# characters.  Multiple such mappings can be separated by whitespace.
+# eg: MIRROR_MAP="http://orig1/foo|http://mirror1/foo/"
+MIRROR_MAP = {}
+if 'MIRROR_MAP' in os.environ:
+  for _mg in os.environ['MIRROR_MAP'].split():
+    _items = _mg.split('|')
+    MIRROR_MAP[_items[0]] = _items[1:]
+
+def UrlMirrors(orig_url):
+  """Return a list of mirror URLS to try (if any), followed by the
+  original URL."""
+  url_list = []
+  for url_prefix in MIRROR_MAP:
+    if orig_url.startswith(url_prefix):
+      for mirror_prefix in MIRROR_MAP[url_prefix]:
+        url_list += [ orig_url.replace(url_prefix, mirror_prefix, 1) ]
+  return url_list + [ orig_url ]
+
 
 def DownloadUrl(url, output_file):
-  """Download url into output_file."""
+  """Download url into output_file, or throw an exception."""
   CHUNK_SIZE = 4096
   TOTAL_DOTS = 10
-  num_retries = 3
+
+  sys.stdout.write('Downloading %s ' % url)
+  sys.stdout.flush()
+  response = urllib2.urlopen(url)
+  total_size = int(response.info().getheader('Content-Length').strip())
+  bytes_done = 0
+  dots_printed = 0
+  while True:
+    chunk = response.read(CHUNK_SIZE)
+    if not chunk:
+      break
+    output_file.write(chunk)
+    bytes_done += len(chunk)
+    num_dots = TOTAL_DOTS * bytes_done / total_size
+    sys.stdout.write('.' * (num_dots - dots_printed))
+    sys.stdout.flush()
+    dots_printed = num_dots
+  if bytes_done != total_size:
+    raise urllib2.URLError("only got %d of %d bytes" %
+                           (bytes_done, total_size))
+  print ' Done.'
+
+
+def DownloadMirroredUrl(orig_url, output_file):
+  """Download orig_url (or a mirror of it) into output_file."""
+  url_list = UrlMirrors(orig_url)
+  num_retries = 3  # Try each url in url_list this many times.
   retry_wait_s = 5  # Doubled at each retry.
 
-  while True:
-    try:
-      sys.stdout.write('Downloading %s ' % url)
-      sys.stdout.flush()
-      response = urllib2.urlopen(url)
-      total_size = int(response.info().getheader('Content-Length').strip())
-      bytes_done = 0
-      dots_printed = 0
-      while True:
-        chunk = response.read(CHUNK_SIZE)
-        if not chunk:
-          break
-        output_file.write(chunk)
-        bytes_done += len(chunk)
-        num_dots = TOTAL_DOTS * bytes_done / total_size
-        sys.stdout.write('.' * (num_dots - dots_printed))
-        sys.stdout.flush()
-        dots_printed = num_dots
-      if bytes_done != total_size:
-        raise urllib2.URLError("only got %d of %d bytes" %
-                               (bytes_done, total_size))
-      print ' Done.'
-      return
-    except urllib2.URLError as e:
-      sys.stdout.write('\n')
-      print e
-      if num_retries == 0 or isinstance(e, urllib2.HTTPError) and e.code == 404:
-        raise e
-      num_retries -= 1
+  for loop_num in range(num_retries):
+
+    if loop_num > 0 and len(url_list) > 0:
       print 'Retrying in %d s ...' % retry_wait_s
       time.sleep(retry_wait_s)
       retry_wait_s *= 2
 
+    # Loop over a copy of url_list since it may be modified:
+    for url in list(url_list):
+      try:
+        DownloadUrl(url, output_file)
+        return
+      except urllib2.URLError as e:
+        sys.stdout.write('\n')
+        print e
+
+        if isinstance(e, urllib2.HTTPError) and e.code == 404:
+          # Don't bother trying this URL again.
+          url_list.remove(url)
+
+  raise urllib2.URLError("Failed to download %s" % orig_url)
+
 
 def EnsureDirExists(path):
   if not os.path.exists(path):
     print "Creating directory %s" % path
     os.makedirs(path)
 
 
 def DownloadAndUnpack(url, output_dir):
   with tempfile.TemporaryFile() as f:
-    DownloadUrl(url, f)
+    DownloadMirroredUrl(url, f)
     f.seek(0)
     EnsureDirExists(output_dir)
     if url.endswith('.zip'):
       zipfile.ZipFile(f).extractall(path=output_dir)
     else:
       tarfile.open(mode='r:gz', fileobj=f).extractall(path=output_dir)
 
 
 def ReadStampFile():
   """Return the contents of the stamp file, or '' if it doesn't exist."""
@@ skipping 623 matching lines @@
     args.force_local_build = True
     if 'OS=android' not in os.environ.get('GYP_DEFINES', ''):
       # Only build the Android ASan rt on ToT bots when targetting Android.
       args.with_android = False
 
   return UpdateClang(args)
 
 
 if __name__ == '__main__':
   sys.exit(main())