Rework/update toolchain script for vs2015

win_toolchain/get_toolchain_if_necessary.py

 #!/usr/bin/env python
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Downloads and unpacks a toolchain for building on Windows. The contents are
 matched by sha1 which will be updated when the toolchain is updated.
 
 Having a toolchain script in depot_tools means that it's not versioned
 directly with the source code. That is, if the toolchain is upgraded, but
@@ skipping 15 matching lines @@
 maintained, and a new 2015 script would be added.
 """
 
 import hashlib
 import json
 import optparse
 import os
 import shutil
 import subprocess
 import sys
+import tempfile
 import time
+import zipfile
 
 
 BASEDIR = os.path.dirname(os.path.abspath(__file__))
 DEPOT_TOOLS_PATH = os.path.join(BASEDIR, '..')
 sys.path.append(DEPOT_TOOLS_PATH)
-import download_from_google_storage
+try:
+  import download_from_google_storage
+except ImportError:
+  # Allow use of utility functions in this script from package_from_installed
+  # on bare VM that doesn't have a full depot_tools.
+  pass
 
 if sys.platform != 'cygwin':
   import ctypes.wintypes
   GetFileAttributes = ctypes.windll.kernel32.GetFileAttributesW
   GetFileAttributes.argtypes = (ctypes.wintypes.LPWSTR,)
   GetFileAttributes.restype = ctypes.wintypes.DWORD
   FILE_ATTRIBUTE_HIDDEN = 0x2
   FILE_ATTRIBUTE_SYSTEM = 0x4
 
 
@@ skipping 126 matching lines @@
   if (os.path.isdir(target_dir) and
       not bool(int(os.environ.get('CHROME_HEADLESS', '0')))):
     for i in range(9, 0, -1):
       sys.stdout.write(
               '\rRemoving old toolchain in %ds... (Ctrl-C to cancel)' % i)
       sys.stdout.flush()
       time.sleep(1)
     print
 
 
+def DownloadUsingGsutil(filename):
+  """Downloads the given file from Google Storage chrome-wintoolchain bucket."""
+  temp_dir = tempfile.mkdtemp()
+  assert os.path.basename(filename) == filename
+  target_path = os.path.join(temp_dir, filename)
+  gsutil = download_from_google_storage.Gsutil(
+      download_from_google_storage.GSUTIL_DEFAULT_PATH, boto_path=None)
+  code = gsutil.call('cp', 'gs://chrome-wintoolchain/' + filename, target_path)
+  if code != 0:
+    sys.exit('gsutil failed')
+  return temp_dir, target_path
+
+
+def DoTreeMirror(target_dir, tree_sha1):
+  """In order to save temporary space on bots that do not have enough space to
+  download ISOs, unpack them, and copy to the target location, the whole tree
+  is uploaded as a zip to internal storage, and then mirrored here."""
+  temp_dir, local_zip = DownloadUsingGsutil(tree_sha1 + '.zip')
+  sys.stdout.write('Extracting %s...\n' % local_zip)
+  sys.stdout.flush()
+  with zipfile.ZipFile(local_zip, 'r', zipfile.ZIP_DEFLATED, True) as zf:
+    zf.extractall(target_dir)
+  if temp_dir:
+    subprocess.check_call('rmdir /s/q "%s"' % temp_dir, shell=True)
+
+
 def main():
   if not sys.platform.startswith(('cygwin', 'win32')):
     return 0
 
   parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
   parser.add_option('--output-json', metavar='FILE',
                     help='write information about toolchain to FILE')
   options, args = parser.parse_args()
 
   if sys.platform == 'cygwin':
     # This script requires Windows Python, so invoke with depot_tools' Python.
     def winpath(path):
       return subprocess.check_output(['cygpath', '-w', path]).strip()
     python = os.path.join(DEPOT_TOOLS_PATH, 'python.bat')
     cmd = [python, winpath(__file__)]
     if options.output_json:
       cmd.extend(['--output-json', winpath(options.output_json)])
     cmd.extend(args)
     sys.exit(subprocess.call(cmd))
 
   # We assume that the Pro hash is the first one.
   desired_hashes = args
   if len(desired_hashes) == 0:
     sys.exit('Desired hashes are required.')
 
   # Move to depot_tools\win_toolchain where we'll store our files, and where
   # the downloader script is.
   os.chdir(os.path.normpath(os.path.join(BASEDIR)))
   toolchain_dir = '.'
-  target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs2013_files'))
+  if os.environ.get('GYP_MSVS_VERSION') == '2015':
+    target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs_files'))
+  else:
+    target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs2013_files'))
+  abs_target_dir = os.path.abspath(target_dir)
+
+  got_new_toolchain = False
 
   # If the current hash doesn't match what we want in the file, nuke and pave.
   # Typically this script is only run when the .sha1 one file is updated, but
   # directly calling "gclient runhooks" will also run it, so we cache
   # based on timestamps to make that case fast.
   current_hash = CalculateHash(target_dir)
   if current_hash not in desired_hashes:
     should_use_gs = False
-    if (HaveSrcInternalAccess() or 
-        LooksLikeGoogler() or 
+    if (HaveSrcInternalAccess() or
+        LooksLikeGoogler() or
         CanAccessToolchainBucket()):
       should_use_gs = True
       if not CanAccessToolchainBucket():
         RequestGsAuthentication()
     if not should_use_gs:
       print('Please follow the instructions at '
             'http://www.chromium.org/developers/how-tos/'
             'build-instructions-windows')
       return 1
     print('Windows toolchain out of date or doesn\'t exist, updating (Pro)...')
     print('  current_hash: %s' % current_hash)
     print('  desired_hashes: %s' % ', '.join(desired_hashes))
     sys.stdout.flush()
     DelayBeforeRemoving(target_dir)
     # This stays resident and will make the rmdir below fail.
     with open(os.devnull, 'wb') as nul:
       subprocess.call(['taskkill', '/f', '/im', 'mspdbsrv.exe'],
                       stdin=nul, stdout=nul, stderr=nul)
     if os.path.isdir(target_dir):
       subprocess.check_call('rmdir /s/q "%s"' % target_dir, shell=True)
-    args = [sys.executable,
-            'toolchain2013.py',
-            '--targetdir', target_dir,
-            '--sha1', desired_hashes[0],
-            '--use-gs']
-    subprocess.check_call(args)
+
+    DoTreeMirror(target_dir, desired_hashes[0])
+
+    got_new_toolchain = True
+
+  win_sdk = os.path.join(abs_target_dir, 'win_sdk')
+  try:
+    with open(os.path.join(target_dir, 'VS_VERSION'), 'rb') as f:
+      vs_version = f.read().strip()
+  except IOError:
+    # Older toolchains didn't have the VS_VERSION file, and used 'win8sdk'
+    # instead of just 'win_sdk'.
+    vs_version = '2013'
+    win_sdk = os.path.join(abs_target_dir, 'win8sdk')
+
+  data = {
+      'path': abs_target_dir,
+      'version': vs_version,
+      'win_sdk': win_sdk,
+      'wdk': os.path.join(abs_target_dir, 'wdk'),
+      'runtime_dirs': [
+        os.path.join(abs_target_dir, 'sys64'),
+        os.path.join(abs_target_dir, 'sys32'),
+      ],
+  }
+  with open(os.path.join(target_dir, '..', 'data.json'), 'w') as f:
+    json.dump(data, f)
+
+  if got_new_toolchain:
     current_hash = CalculateHash(target_dir)
     if current_hash not in desired_hashes:
       print >> sys.stderr, (
           'Got wrong hash after pulling a new toolchain. '
           'Wanted one of \'%s\', got \'%s\'.' % (
               ', '.join(desired_hashes), current_hash))
       return 1
     SaveTimestampsAndHash(target_dir, current_hash)
 
   if options.output_json:
     shutil.copyfile(os.path.join(target_dir, '..', 'data.json'),
                     options.output_json)
 
   return 0
 
 
 if __name__ == '__main__':
   sys.exit(main())