use gsutil to download clang et al binaries

tools/clang/scripts/update.py

Index: tools/clang/scripts/update.py
diff --git a/tools/clang/scripts/update.py b/tools/clang/scripts/update.py
index 322359f69596d13d4adafdf12b401beb32fa1e1b..8373b569d4c65f027df8d89fd5e5469826d5896d 100755
--- a/tools/clang/scripts/update.py
+++ b/tools/clang/scripts/update.py
@@ -20,7 +20,6 @@ import sys
 import tarfile
 import tempfile
 import time
-import urllib2
 import zipfile
 
 # Do NOT CHANGE this if you don't know what you're doing -- see
@@ -69,48 +68,35 @@ VERSION = '3.9.0'
 ANDROID_NDK_DIR = os.path.join(
     CHROMIUM_DIR, 'third_party', 'android_tools', 'ndk')
 
-# URL for pre-built binaries.
-CDS_URL = 'https://commondatastorage.googleapis.com/chromium-browser-clang'
+sys.path.insert(0, os.path.join(CHROMIUM_DIR, 'build'))
+import find_depot_tools
+DEPOT_PATH = find_depot_tools.add_depot_tools_to_path()
+GSUTIL_PATH = os.path.join(DEPOT_PATH, 'gsutil.py')
+
+# Google Storage bucket where the pre-built binaries are kept.
+CDS_BUCKET = 'gs://chromium-browser-clang'
 
 LLVM_REPO_URL='https://llvm.org/svn/llvm-project'
 if 'LLVM_REPO_URL' in os.environ:
   LLVM_REPO_URL = os.environ['LLVM_REPO_URL']
 
 
-def DownloadUrl(url, output_file):
-  """Download url into output_file."""
-  CHUNK_SIZE = 4096
-  TOTAL_DOTS = 10
+def DownloadFile(source, output_file):
+  """Download google storage source into output_file."""
   num_retries = 3
   retry_wait_s = 5  # Doubled at each retry.
 
   while True:
     try:
-      sys.stdout.write('Downloading %s ' % url)
+      sys.stdout.write('Downloading %s ' % source)
       sys.stdout.flush()
-      response = urllib2.urlopen(url)
-      total_size = int(response.info().getheader('Content-Length').strip())
-      bytes_done = 0
-      dots_printed = 0
-      while True:
-        chunk = response.read(CHUNK_SIZE)
-        if not chunk:
-          break
-        output_file.write(chunk)
-        bytes_done += len(chunk)
-        num_dots = TOTAL_DOTS * bytes_done / total_size
-        sys.stdout.write('.' * (num_dots - dots_printed))
-        sys.stdout.flush()
-        dots_printed = num_dots
-      if bytes_done != total_size:
-        raise urllib2.URLError("only got %d of %d bytes" %
-                               (bytes_done, total_size))
-      print ' Done.'
+      cmd = ['python', GSUTIL_PATH, 'cp', source, output_file]
+      subprocess.check_call(cmd, stderr=open('/dev/null', 'w'))

[hans, 2016/02/12 00:39:11]

What does the error message look like for the user of update.py when gsutil
fails?

[Mostyn Bramley-Moore, 2016/02/12 00:43:33]

This is what I get on a throttled/blocked machine, without a ~/.boto config
setup (inside a retry loop):

Downloading gs://chromium-browser-clang/Linux_x64/clang-259396-1.tgz 
Command '['python', '/root/depot_tools/gsutil.py', 'cp',
'gs://chromium-browser-clang/Linux_x64/clang-259396-1.tgz', '/tmp/tmpqRDSzF']'
returned non-zero exit status 1

[hans, 2016/02/12 01:42:33]

I assume opening /dev/null doesn't work on Windows?

[hans, 2016/02/12 01:42:33]

That's pretty opaque. The current code will tell the user what went wrong.

[Mostyn Bramley-Moore, 2016/02/15 13:29:44]

Ahh, right- I copied this pattern from build/download_gold_plugin.py which is
not used on windows.

       return
-    except urllib2.URLError as e:
+    except Exception as e:
       sys.stdout.write('\n')
       print e
-      if num_retries == 0 or isinstance(e, urllib2.HTTPError) and e.code == 404:
+      if num_retries == 0: # TODO(mostynb): exit early on 404?
         raise e
       num_retries -= 1
       print 'Retrying in %d s ...' % retry_wait_s
@@ -124,12 +110,12 @@ def EnsureDirExists(path):
     os.makedirs(path)
 
 
-def DownloadAndUnpack(url, output_dir):
-  with tempfile.TemporaryFile() as f:
-    DownloadUrl(url, f)
+def DownloadAndUnpack(source, output_dir):
+  with tempfile.NamedTemporaryFile() as f:
+    DownloadFile(source, f.name)
     f.seek(0)
     EnsureDirExists(output_dir)
-    if url.endswith('.zip'):
+    if source.endswith('.zip'):
       zipfile.ZipFile(f).extractall(path=output_dir)
     else:
       tarfile.open(mode='r:gz', fileobj=f).extractall(path=output_dir)
@@ -285,7 +271,7 @@ def DownloadHostGcc(args):
   if not os.path.exists(gcc_dir):
     print 'Downloading pre-built GCC 4.8.2...'
     DownloadAndUnpack(
-        CDS_URL + '/tools/gcc482precise.tgz', LLVM_BUILD_TOOLS_DIR)
+        CDS_BUCKET + '/tools/gcc482precise.tgz', LLVM_BUILD_TOOLS_DIR)
   args.gcc_toolchain = gcc_dir
 
 
@@ -300,7 +286,7 @@ def AddCMakeToPath():
     zip_name = 'cmake322_%s.tgz' % suffix
     cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'cmake322', 'bin')
   if not os.path.exists(cmake_dir):
-    DownloadAndUnpack(CDS_URL + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR)
+    DownloadAndUnpack(CDS_BUCKET + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR)
   os.environ['PATH'] = cmake_dir + os.pathsep + os.environ.get('PATH', '')
 
 
@@ -344,18 +330,18 @@ def UpdateClang(args):
   if not args.force_local_build:
     cds_file = "clang-%s.tgz" %  PACKAGE_VERSION
     if sys.platform == 'win32' or sys.platform == 'cygwin':
-      cds_full_url = CDS_URL + '/Win/' + cds_file
+      cds_source = CDS_BUCKET + '/Win/' + cds_file
     elif sys.platform == 'darwin':
-      cds_full_url = CDS_URL + '/Mac/' + cds_file
+      cds_source = CDS_BUCKET + '/Mac/' + cds_file
     else:
       assert sys.platform.startswith('linux')
-      cds_full_url = CDS_URL + '/Linux_x64/' + cds_file
+      cds_source = CDS_BUCKET + '/Linux_x64/' + cds_file
 
     print 'Downloading prebuilt clang'
     if os.path.exists(LLVM_BUILD_DIR):
       RmTree(LLVM_BUILD_DIR)
     try:
-      DownloadAndUnpack(cds_full_url, LLVM_BUILD_DIR)
+      DownloadAndUnpack(cds_source, LLVM_BUILD_DIR)
       print 'clang %s unpacked' % PACKAGE_VERSION
       # Download the gold plugin if requested to by an environment variable.
       # This is used by the CFI ClusterFuzz bot, and it's required for official
@@ -364,7 +350,7 @@ def UpdateClang(args):
         RunCommand(['python', CHROMIUM_DIR+'/build/download_gold_plugin.py'])
       WriteStampFile(PACKAGE_VERSION)
       return 0
-    except urllib2.URLError:
+    except Exception:

[hans, 2016/02/12 01:42:33]

This exception handler is much broader than the previous one.

For example, if there is some error while unpacking the file, that error is now
no longer surfaced to the user.

       print 'Failed to download prebuilt clang %s' % cds_file
       print 'Use --force-local-build if you want to build locally.'
       print 'Exiting.'