use gsutil to download clang et al binaries

tools/clang/scripts/update.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """This script is used to download prebuilt clang binaries.
 
 It is also used by package.py to build the prebuilt clang binaries."""
 
 import argparse
 import distutils.spawn
 import glob
 import os
 import pipes
 import re
 import shutil
 import subprocess
 import stat
 import sys
 import tarfile
 import tempfile
 import time
-import urllib2
 import zipfile
 
 # Do NOT CHANGE this if you don't know what you're doing -- see
 # https://chromium.googlesource.com/chromium/src/+/master/docs/updating_clang.md
 # Reverting problematic clang rolls is safe, though.
 CLANG_REVISION = '259396'
 
 use_head_revision = 'LLVM_FORCE_HEAD_REVISION' in os.environ
 if use_head_revision:
   CLANG_REVISION = 'HEAD'
@@ skipping 28 matching lines @@
 LIBCXXABI_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxxabi')
 LLVM_BUILD_TOOLS_DIR = os.path.abspath(
     os.path.join(LLVM_DIR, '..', 'llvm-build-tools'))
 STAMP_FILE = os.path.normpath(
     os.path.join(LLVM_DIR, '..', 'llvm-build', 'cr_build_revision'))
 BINUTILS_DIR = os.path.join(THIRD_PARTY_DIR, 'binutils')
 VERSION = '3.9.0'
 ANDROID_NDK_DIR = os.path.join(
     CHROMIUM_DIR, 'third_party', 'android_tools', 'ndk')
 
-# URL for pre-built binaries.
-CDS_URL = 'https://commondatastorage.googleapis.com/chromium-browser-clang'
+sys.path.insert(0, os.path.join(CHROMIUM_DIR, 'build'))
+import find_depot_tools
+DEPOT_PATH = find_depot_tools.add_depot_tools_to_path()
+GSUTIL_PATH = os.path.join(DEPOT_PATH, 'gsutil.py')
+
+# Google Storage bucket where the pre-built binaries are kept.
+CDS_BUCKET = 'gs://chromium-browser-clang'
 
 LLVM_REPO_URL='https://llvm.org/svn/llvm-project'
 if 'LLVM_REPO_URL' in os.environ:
   LLVM_REPO_URL = os.environ['LLVM_REPO_URL']
 
 
-def DownloadUrl(url, output_file):
-  """Download url into output_file."""
-  CHUNK_SIZE = 4096
-  TOTAL_DOTS = 10
+def DownloadFile(source, output_file):
+  """Download google storage source into output_file."""
   num_retries = 3
   retry_wait_s = 5  # Doubled at each retry.
 
   while True:
     try:
-      sys.stdout.write('Downloading %s ' % url)
+      sys.stdout.write('Downloading %s ' % source)
       sys.stdout.flush()
-      response = urllib2.urlopen(url)
-      total_size = int(response.info().getheader('Content-Length').strip())
-      bytes_done = 0
-      dots_printed = 0
-      while True:
-        chunk = response.read(CHUNK_SIZE)
-        if not chunk:
-          break
-        output_file.write(chunk)
-        bytes_done += len(chunk)
-        num_dots = TOTAL_DOTS * bytes_done / total_size
-        sys.stdout.write('.' * (num_dots - dots_printed))
-        sys.stdout.flush()
-        dots_printed = num_dots
-      if bytes_done != total_size:
-        raise urllib2.URLError("only got %d of %d bytes" %
-                               (bytes_done, total_size))
-      print ' Done.'
+      cmd = ['python', GSUTIL_PATH, 'cp', source, output_file]
+      subprocess.check_call(cmd, stderr=open('/dev/null', 'w'))

[hans, 2016/02/12 00:39:11]

What does the error message look like for the user of update.py when gsutil
fails?

[Mostyn Bramley-Moore, 2016/02/12 00:43:33]

This is what I get on a throttled/blocked machine, without a ~/.boto config
setup (inside a retry loop):

Downloading gs://chromium-browser-clang/Linux_x64/clang-259396-1.tgz 
Command '['python', '/root/depot_tools/gsutil.py', 'cp',
'gs://chromium-browser-clang/Linux_x64/clang-259396-1.tgz', '/tmp/tmpqRDSzF']'
returned non-zero exit status 1

[hans, 2016/02/12 01:42:33]

I assume opening /dev/null doesn't work on Windows?

[hans, 2016/02/12 01:42:33]

That's pretty opaque. The current code will tell the user what went wrong.

[Mostyn Bramley-Moore, 2016/02/15 13:29:44]

Ahh, right- I copied this pattern from build/download_gold_plugin.py which is
not used on windows.

       return
-    except urllib2.URLError as e:
+    except Exception as e:
       sys.stdout.write('\n')
       print e
-      if num_retries == 0 or isinstance(e, urllib2.HTTPError) and e.code == 404:
+      if num_retries == 0: # TODO(mostynb): exit early on 404?
         raise e
       num_retries -= 1
       print 'Retrying in %d s ...' % retry_wait_s
       time.sleep(retry_wait_s)
       retry_wait_s *= 2
 
 
 def EnsureDirExists(path):
   if not os.path.exists(path):
     print "Creating directory %s" % path
     os.makedirs(path)
 
 
-def DownloadAndUnpack(url, output_dir):
-  with tempfile.TemporaryFile() as f:
-    DownloadUrl(url, f)
+def DownloadAndUnpack(source, output_dir):
+  with tempfile.NamedTemporaryFile() as f:
+    DownloadFile(source, f.name)
     f.seek(0)
     EnsureDirExists(output_dir)
-    if url.endswith('.zip'):
+    if source.endswith('.zip'):
       zipfile.ZipFile(f).extractall(path=output_dir)
     else:
       tarfile.open(mode='r:gz', fileobj=f).extractall(path=output_dir)
 
 
 def ReadStampFile():
   """Return the contents of the stamp file, or '' if it doesn't exist."""
   try:
     with open(STAMP_FILE, 'r') as f:
       return f.read().rstrip()
@@ skipping 135 matching lines @@
 def DownloadHostGcc(args):
   """Downloads gcc 4.8.2 and makes sure args.gcc_toolchain is set."""
   if not sys.platform.startswith('linux') or args.gcc_toolchain:
     return
   # Unconditionally download a prebuilt gcc to guarantee the included libstdc++
   # works on Ubuntu Precise.
   gcc_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'gcc482precise')
   if not os.path.exists(gcc_dir):
     print 'Downloading pre-built GCC 4.8.2...'
     DownloadAndUnpack(
-        CDS_URL + '/tools/gcc482precise.tgz', LLVM_BUILD_TOOLS_DIR)
+        CDS_BUCKET + '/tools/gcc482precise.tgz', LLVM_BUILD_TOOLS_DIR)
   args.gcc_toolchain = gcc_dir
 
 
 def AddCMakeToPath():
   """Download CMake and add it to PATH."""
   if sys.platform == 'win32':
     zip_name = 'cmake-3.2.2-win32-x86.zip'
     cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR,
                              'cmake-3.2.2-win32-x86', 'bin')
   else:
     suffix = 'Darwin' if sys.platform == 'darwin' else 'Linux'
     zip_name = 'cmake322_%s.tgz' % suffix
     cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'cmake322', 'bin')
   if not os.path.exists(cmake_dir):
-    DownloadAndUnpack(CDS_URL + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR)
+    DownloadAndUnpack(CDS_BUCKET + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR)
   os.environ['PATH'] = cmake_dir + os.pathsep + os.environ.get('PATH', '')
 
 
 vs_version = None
 def GetVSVersion():
   global vs_version
   if vs_version:
     return vs_version
 
   # Try using the toolchain in depot_tools.
@@ skipping 23 matching lines @@
     if not need_gold_plugin or os.path.exists(
         os.path.join(LLVM_BUILD_DIR, "lib/LLVMgold.so")):
       return 0
 
   # Reset the stamp file in case the build is unsuccessful.
   WriteStampFile('')
 
   if not args.force_local_build:
     cds_file = "clang-%s.tgz" %  PACKAGE_VERSION
     if sys.platform == 'win32' or sys.platform == 'cygwin':
-      cds_full_url = CDS_URL + '/Win/' + cds_file
+      cds_source = CDS_BUCKET + '/Win/' + cds_file
     elif sys.platform == 'darwin':
-      cds_full_url = CDS_URL + '/Mac/' + cds_file
+      cds_source = CDS_BUCKET + '/Mac/' + cds_file
     else:
       assert sys.platform.startswith('linux')
-      cds_full_url = CDS_URL + '/Linux_x64/' + cds_file
+      cds_source = CDS_BUCKET + '/Linux_x64/' + cds_file
 
     print 'Downloading prebuilt clang'
     if os.path.exists(LLVM_BUILD_DIR):
       RmTree(LLVM_BUILD_DIR)
     try:
-      DownloadAndUnpack(cds_full_url, LLVM_BUILD_DIR)
+      DownloadAndUnpack(cds_source, LLVM_BUILD_DIR)
       print 'clang %s unpacked' % PACKAGE_VERSION
       # Download the gold plugin if requested to by an environment variable.
       # This is used by the CFI ClusterFuzz bot, and it's required for official
       # builds on linux.
       if need_gold_plugin:
         RunCommand(['python', CHROMIUM_DIR+'/build/download_gold_plugin.py'])
       WriteStampFile(PACKAGE_VERSION)
       return 0
-    except urllib2.URLError:
+    except Exception:

[hans, 2016/02/12 01:42:33]

This exception handler is much broader than the previous one.

For example, if there is some error while unpacking the file, that error is now
no longer surfaced to the user.

       print 'Failed to download prebuilt clang %s' % cds_file
       print 'Use --force-local-build if you want to build locally.'
       print 'Exiting.'
       return 1
 
   if args.with_android and not os.path.exists(ANDROID_NDK_DIR):
     print 'Android NDK not found at ' + ANDROID_NDK_DIR
     print 'The Android NDK is needed to build a Clang whose -fsanitize=address'
     print 'works on Android. See '
     print 'https://www.chromium.org/developers/how-tos/android-build-instructions'
@@ skipping 385 matching lines @@
     args.force_local_build = True
     if 'OS=android' not in os.environ.get('GYP_DEFINES', ''):
       # Only build the Android ASan rt on ToT bots when targetting Android.
       args.with_android = False
 
   return UpdateClang(args)
 
 
 if __name__ == '__main__':
   sys.exit(main())