win_toolchain: Update packaging script to package win10 sdk

win_toolchain/toolchain2013.py

-#!/usr/bin/env python

[scottmg, 2015/10/03 05:04:25]

This used to pull together a toolchain from a bunch of isos that meant that
non-Googlers could have an automatically-installed toolchain too, but it's not
very possible with the new distribution mechanisms (.msp patch files).

It hasn't worked for building for a while (pre 2013 Update 2), but I forgot to
delete it.

-# Copyright 2013 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Extracts a Windows VS2013 toolchain from various downloadable pieces."""
-
-
-import ctypes
-import json
-import optparse
-import os
-import shutil
-import subprocess
-import sys
-import tempfile
-import urllib2
-
-
-BASEDIR = os.path.dirname(os.path.abspath(__file__))
-WDK_ISO_URL = (
-    'http://download.microsoft.com/download/'
-    '4/A/2/4A25C7D5-EFBE-4182-B6A9-AE6850409A78/GRMWDK_EN_7600_1.ISO')
-g_temp_dirs = []
-
-
-sys.path.append(os.path.join(BASEDIR, '..'))
-import download_from_google_storage
-
-
-def GetLongPathName(path):
-  """Converts any 8dot3 names in the path to the full name."""
-  buf = ctypes.create_unicode_buffer(260)
-  size = ctypes.windll.kernel32.GetLongPathNameW(unicode(path), buf, 260)
-  if (size > 260):
-    sys.exit('Long form of path longer than 260 chars: %s' % path)
-  return buf.value
-
-
-def RunOrDie(command):
-  subprocess.check_call(command, shell=True)
-
-
-class ScopedSubstTempDir(object):
-  """Creates a |TempDir()| and subst's a drive to the path.
-
-  This is done to avoid exceedingly long names in some .msi packages which
-  fail to extract because they exceed _MAX_PATH. Only the "subst" part of this
-  is scoped, not the temp dir, which is left for use and cleanup by the
-  caller.
-  """
-  DefineDosDevice = ctypes.windll.kernel32.DefineDosDeviceW
-  DefineDosDevice.argtypes = [ctypes.c_int, ctypes.c_wchar_p, ctypes.c_wchar_p]
-  DDD_NO_BROADCAST_SYSTEM = 0x08
-  DDD_REMOVE_DEFINITION = 0x02
-
-  def __init__(self):
-    self.real_path = TempDir()
-    self.subst_drive = None
-
-  def __enter__(self):
-    """Tries to find a subst that we can use for the temporary directory, and
-    aborts on failure."""
-    for drive in range(ord('Z'), ord('A') - 1, -1):
-      candidate = '%c:' % drive
-      if self.DefineDosDevice(
-          self.DDD_NO_BROADCAST_SYSTEM, candidate, self.real_path) != 0:
-        self.subst_drive = candidate
-        return self
-    raise RuntimeError('Unable to find a subst path')
-
-  def __exit__(self, typ, value, traceback):
-    if self.subst_drive:
-      if self.DefineDosDevice(int(self.DDD_REMOVE_DEFINITION),
-                              self.subst_drive,
-                              self.real_path) == 0:
-        raise RuntimeError('Unable to remove subst')
-
-  def ShortenedPath(self):
-    return self.subst_drive + '\\'
-
-  def RealPath(self):
-    return self.real_path
-
-
-def TempDir():
-  """Generates a temporary directory (for downloading or extracting to) and keep
-  track of the directory that's created for cleaning up later.
-  """
-  temp = tempfile.mkdtemp()
-  g_temp_dirs.append(temp)
-  return temp
-
-
-def DeleteAllTempDirs():
-  """Removes all temporary directories created by |TempDir()|."""
-  global g_temp_dirs
-  if g_temp_dirs:
-    sys.stdout.write('Cleaning up temporaries...\n')
-  for temp in g_temp_dirs:
-    # shutil.rmtree errors out on read only attributes.
-    RunOrDie('rmdir /s/q "%s"' % temp)
-  g_temp_dirs = []
-
-
-def GetMainIsoUrl(pro):
-  """Gets the main .iso URL.
-
-  If |pro| is False, downloads the Express edition.
-  """
-  prefix = 'http://download.microsoft.com/download/'
-  if pro:
-    return (prefix +
-        'A/F/1/AF128362-A6A8-4DB3-A39A-C348086472CC/VS2013_RTM_PRO_ENU.iso')
-  else:
-    return (prefix +
-        '7/2/E/72E0F986-D247-4289-B9DC-C4FB07374894/VS2013_RTM_DskExp_ENU.iso')
-
-
-def Download(url, local_path):
-  """Downloads a large-ish binary file and print some status information while
-  doing so.
-  """
-  sys.stdout.write('Downloading %s...\n' % url)
-  req = urllib2.urlopen(url)
-  content_length = int(req.headers.get('Content-Length', 0))
-  bytes_read = 0L
-  terminator = '\r' if sys.stdout.isatty() else '\n'
-  with open(local_path, 'wb') as file_handle:
-    while True:
-      chunk = req.read(1024 * 1024)
-      if not chunk:
-        break
-      bytes_read += len(chunk)
-      file_handle.write(chunk)
-      sys.stdout.write('... %d/%d%s' % (bytes_read, content_length, terminator))
-      sys.stdout.flush()
-  sys.stdout.write('\n')
-  if content_length and content_length != bytes_read:
-    sys.exit('Got incorrect number of bytes downloading %s' % url)
-
-
-def ExtractIso(iso_path):
-  """Uses 7zip to extract the contents of the given .iso (or self-extracting
-  .exe).
-  """
-  target_path = TempDir()
-  sys.stdout.write('Extracting %s...\n' % iso_path)
-  sys.stdout.flush()
-  # TODO(scottmg): Do this (and exe) manually with python code.
-  # Note that at the beginning of main() we set the working directory to 7z's
-  # location so that 7z can find its codec dll.
-  RunOrDie('7z x "%s" -y "-o%s"' % (iso_path, target_path))
-  return target_path
-
-
-def ExtractMsi(msi_path):
-  """Uses msiexec to extract the contents of the given .msi file."""
-  sys.stdout.write('Extracting %s...\n' % msi_path)
-  with ScopedSubstTempDir() as temp_dir:
-    RunOrDie('msiexec /a "%s" /qn TARGETDIR="%s"' % (
-               msi_path, temp_dir.ShortenedPath()))
-    return temp_dir.RealPath()
-
-
-def DownloadMainIso(url):
-  temp_dir = TempDir()
-  target_path = os.path.join(temp_dir, os.path.basename(url))
-  Download(url, target_path)
-  return target_path
-
-
-def DownloadSDK8():
-  """Downloads the Win8 SDK.
-
-  This one is slightly different than the simpler direct downloads. There is
-  no .ISO distribution for the Windows 8 SDK. Rather, a tool is provided that
-  is a download manager. This is used to download the various .msi files to a
-  target location. Unfortunately, this tool requires elevation for no obvious
-  reason even when only downloading, so this function will trigger a UAC
-  elevation if the script is not run from an elevated prompt. This is mostly
-  grabbed for windbg and cdb (See http://crbug.com/321187) as most of the SDK
-  is in VS2013, however we need a couple D3D related things from the SDK.
-  """
-  # Use the long path name here because because 8dot3 names don't seem to work.
-  sdk_temp_dir = GetLongPathName(TempDir())
-  target_path = os.path.join(sdk_temp_dir, 'sdksetup.exe')
-  standalone_path = os.path.join(sdk_temp_dir, 'Standalone')
-  Download(
-      ('http://download.microsoft.com/download/'
-       'F/1/3/F1300C9C-A120-4341-90DF-8A52509B23AC/standalonesdk/sdksetup.exe'),
-      target_path)
-  sys.stdout.write(
-      'Running sdksetup.exe to download Win8 SDK (may request elevation)...\n')
-  count = 0
-  while count < 5:
-    rc = subprocess.call([target_path,
-                          '/quiet',
-                          '/features', 'OptionId.WindowsDesktopDebuggers',
-                              'OptionId.WindowsDesktopSoftwareDevelopmentKit',
-                          '/layout', standalone_path])
-    if rc == 0:
-      return standalone_path
-    count += 1
-    sys.stdout.write('Windows 8 SDK failed to download, retrying.\n')
-  sys.exit('After multiple retries, couldn\'t download Win8 SDK')
-
-
-def DownloadWDKIso():
-  wdk_temp_dir = TempDir()
-  target_path = os.path.join(wdk_temp_dir, 'GRMWDK_EN_7600_1.ISO')
-  Download(WDK_ISO_URL, target_path)
-  return target_path
-
-
-def DownloadUsingGsutil(filename):
-  """Downloads the given file from Google Storage chrome-wintoolchain bucket."""
-  temp_dir = TempDir()
-  assert os.path.basename(filename) == filename
-  target_path = os.path.join(temp_dir, filename)
-  gsutil = download_from_google_storage.Gsutil(
-      download_from_google_storage.GSUTIL_DEFAULT_PATH, boto_path=None)
-  code = gsutil.call('cp', 'gs://chrome-wintoolchain/' + filename, target_path)
-  if code != 0:
-    sys.exit('gsutil failed')
-  return target_path
-
-
-def GetVSInternal():
-  """Uses gsutil to pull the toolchain from internal Google Storage bucket."""
-  return DownloadUsingGsutil('VS2013_RTM_PRO_ENU.iso')
-
-
-def GetSDKInternal():
-  """Downloads a zipped copy of the SDK from internal Google Storage bucket,
-  and extracts it."""
-  zip_file = DownloadUsingGsutil('Standalone.zip')
-  return ExtractIso(zip_file)
-
-
-class SourceImages(object):
-  """Local paths for components. |wdk_path| may be None if it's unnecessary for
-  the given configuration."""
-  def __init__(self, vs_path, sdk8_path, wdk_path):
-    self.vs_path = vs_path
-    self.sdk8_path = sdk8_path
-    self.wdk_path = wdk_path
-
-
-def GetSourceImages(local_dir, pro):
-  """Downloads the various sources that we need.
-
-  Of note: Because Express does not include ATL, there's an additional download
-  of the 7.1 WDK which is the latest publically accessible source for ATL. When
-  |pro| this is not necessary (and CHROME_HEADLESS always implies Pro).
-  """
-  if pro and not local_dir:
-    sys.exit('Non-Express must be used with --local')
-  url = GetMainIsoUrl(pro)
-  if local_dir:
-    wdk_path = (os.path.join(local_dir, os.path.basename(WDK_ISO_URL))
-                if not pro else None)
-    return SourceImages(os.path.join(local_dir, os.path.basename(url)),
-                        os.path.join(local_dir, 'Standalone'),
-                        wdk_path=wdk_path)
-  else:
-    # Note that we do the SDK first, as it might cause an elevation prompt.
-    sdk8_path = DownloadSDK8()
-    vs_path = DownloadMainIso(url)
-    wdk_path = DownloadWDKIso() if not pro else None
-    return SourceImages(vs_path, sdk8_path, wdk_path=wdk_path)
-
-
-def ExtractMsiList(root_dir, packages):
-  """Extracts the contents of a list of .msi files from an already extracted
-  .iso file.
-
-  |packages| is a list of pairs (msi, required). If required is not True, the
-  msi is optional (this is set for packages that are in Pro but not Express).
-  """
-  results = []
-  for (package, required) in packages:
-    path_to_package = os.path.join(root_dir, package)
-    if not os.path.exists(path_to_package) and not required:
-      continue
-    results.append(ExtractMsi(path_to_package))
-  return results
-
-
-def ExtractComponents(image):
-  vs_packages = [
-      (r'vcRuntimeAdditional_amd64\vc_runtimeAdditional_x64.msi', True),
-      (r'vcRuntimeAdditional_x86\vc_runtimeAdditional_x86.msi', True),
-      (r'vcRuntimeDebug_amd64\vc_runtimeDebug_x64.msi', True),
-      (r'vcRuntimeDebug_x86\vc_runtimeDebug_x86.msi', True),
-      (r'vcRuntimeMinimum_amd64\vc_runtimeMinimum_x64.msi', True),
-      (r'vcRuntimeMinimum_x86\vc_runtimeMinimum_x86.msi', True),
-      (r'vc_compilerCore86\vc_compilerCore86.msi', True),
-      (r'vc_compilerCore86res\vc_compilerCore86res.msi', True),
-      (r'vc_compilerx64nat\vc_compilerx64nat.msi', False),
-      (r'vc_compilerx64natres\vc_compilerx64natres.msi', False),
-      (r'vc_compilerx64x86\vc_compilerx64x86.msi', False),
-      (r'vc_compilerx64x86res\vc_compilerx64x86res.msi', False),
-      (r'vc_librarycore86\vc_librarycore86.msi', True),
-      (r'vc_libraryDesktop\x64\vc_LibraryDesktopX64.msi', True),
-      (r'vc_libraryDesktop\x86\vc_LibraryDesktopX86.msi', True),
-      (r'vc_libraryextended\vc_libraryextended.msi', False),
-      (r'professionalcore\Setup\vs_professionalcore.msi', False),
-      (r'vc_libraryselectablemfc\vc_libraryselectablemfc.msi', False),
-    ]
-  extracted_iso = ExtractIso(image.vs_path)
-  result = ExtractMsiList(os.path.join(extracted_iso, 'packages'), vs_packages)
-
-  sdk_packages = [
-      (r'X86 Debuggers And Tools-x86_en-us.msi', True),
-      (r'X64 Debuggers And Tools-x64_en-us.msi', True),
-      (r'SDK Debuggers-x86_en-us.msi', True),
-      (r'Windows Software Development Kit-x86_en-us.msi', True),
-      (r'Windows Software Development Kit for Metro style Apps-x86_en-us.msi',
-          True),
-    ]
-  result.extend(ExtractMsiList(os.path.join(image.sdk8_path, 'Installers'),
-                               sdk_packages))
-
-  if image.wdk_path:
-    # This image will only be set when using Express, when we need the WDK
-    # headers and libs to supplement Express with ATL.
-    wdk_packages = [
-      (r'headers.msi', True),
-      (r'libs_x86fre.msi', True),
-      (r'libs_x64fre.msi', True),
-    ]
-    extracted_iso = ExtractIso(image.wdk_path)
-    result.extend(ExtractMsiList(os.path.join(extracted_iso, 'WDK'),
-                                 wdk_packages))
-
-  return result
-
-
-def CopyToFinalLocation(extracted_dirs, target_dir):
-  sys.stdout.write('Copying to final location...\n')
-  mappings = {
-      'Program Files\\Microsoft Visual Studio 12.0\\VC\\': 'VC\\',
-      'Program Files\\Microsoft Visual Studio 12.0\\DIA SDK\\': 'DIA SDK\\',
-      'System64\\': 'sys64\\',
-      'System\\': 'sys32\\',
-      'Windows Kits\\8.1\\': 'win8sdk\\',
-      'WinDDK\\7600.16385.win7_wdk.100208-1538\\': 'wdk\\',
-  }
-  matches = []
-  for extracted_dir in extracted_dirs:
-    for root, _, filenames in os.walk(extracted_dir):
-      for filename in filenames:
-        matches.append((extracted_dir, os.path.join(root, filename)))
-
-  copies = []
-  for prefix, full_path in matches:
-    # +1 for trailing \.
-    partial_path = full_path[len(prefix) + 1:]
-    for map_from, map_to in mappings.iteritems():
-      if partial_path.startswith(map_from):
-        target_path = os.path.join(map_to, partial_path[len(map_from):])
-        copies.append((full_path, os.path.join(target_dir, target_path)))
-
-  for full_source, full_target in copies:
-    target_dir = os.path.dirname(full_target)
-    if not os.path.isdir(target_dir):
-      os.makedirs(target_dir)
-    shutil.copy2(full_source, full_target)
-
-
-def GenerateSetEnvCmd(target_dir, pro):
-  """Generate a batch file that gyp expects to exist to set up the compiler
-  environment.
-
-  This is normally generated by a full install of the SDK, but we
-  do it here manually since we do not do a full install."""
-  with open(os.path.join(
-        target_dir, r'win8sdk\bin\SetEnv.cmd'), 'w') as f:
-    f.write('@echo off\n'
-            ':: Generated by win_toolchain\\toolchain2013.py.\n'
-            # Common to x86 and x64
-            'set PATH=%~dp0..\\..\\Common7\\IDE;%PATH%\n'
-            'set INCLUDE=%~dp0..\\..\\win8sdk\\Include\\um;'
-               '%~dp0..\\..\\win8sdk\\Include\\shared;'
-               '%~dp0..\\..\\VC\\include;'
-               '%~dp0..\\..\\VC\\atlmfc\\include\n'
-            'if "%1"=="/x64" goto x64\n')
-
-    # x86. If we're Pro, then use the amd64_x86 cross (we don't support x86
-    # host at all).
-    if pro:
-      f.write('set PATH=%~dp0..\\..\\win8sdk\\bin\\x86;'
-                '%~dp0..\\..\\VC\\bin\\amd64_x86;'
-                '%~dp0..\\..\\VC\\bin\\amd64;'  # Needed for mspdb120.dll.
-                '%PATH%\n')
-    else:
-      f.write('set PATH=%~dp0..\\..\\win8sdk\\bin\\x86;'
-                '%~dp0..\\..\\VC\\bin;%PATH%\n')
-    f.write('set LIB=%~dp0..\\..\\VC\\lib;'
-               '%~dp0..\\..\\win8sdk\\Lib\\winv6.3\\um\\x86;'
-               '%~dp0..\\..\\VC\\atlmfc\\lib\n'
-            'goto :EOF\n')
-
-    # Express does not include a native 64 bit compiler, so we have to use
-    # the x86->x64 cross.
-    if not pro:
-      # x86->x64 cross.
-      f.write(':x64\n'
-              'set PATH=%~dp0..\\..\\win8sdk\\bin\\x64;'
-                 '%~dp0..\\..\\VC\\bin\\x86_amd64;'
-                 # Needed for mspdb120.dll. Must be after above though, so
-                 # that cl.exe is the x86_amd64 one.
-                 '%~dp0..\\..\\VC\\bin;'
-                 '%PATH%\n')
-    else:
-      # x64 native.
-      f.write(':x64\n'
-              'set PATH=%~dp0..\\..\\win8sdk\\bin\\x64;'
-                 '%~dp0..\\..\\VC\\bin\\amd64;'
-                 '%PATH%\n')
-    f.write('set LIB=%~dp0..\\..\\VC\\lib\\amd64;'
-               '%~dp0..\\..\\win8sdk\\Lib\\winv6.3\\um\\x64;'
-               '%~dp0..\\..\\VC\\atlmfc\\lib\\amd64\n')
-
-
-def DoTreeMirror(target_dir, tree_sha1):
-  """In order to save temporary space on bots that do not have enough space to
-  download ISOs, unpack them, and copy to the target location, the whole tree
-  is uploaded as a zip to internal storage, and then mirrored here."""
-  local_zip = DownloadUsingGsutil(tree_sha1 + '.zip')
-  sys.stdout.write('Extracting %s...\n' % local_zip)
-  sys.stdout.flush()
-  RunOrDie('7z x "%s" -y "-o%s"' % (local_zip, target_dir))
-
-
-def main():
-  parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
-  parser.add_option('--targetdir', metavar='DIR',
-                    help='put toolchain into DIR',
-                    default=os.path.join(BASEDIR, 'win_toolchain_2013'))
-  parser.add_option('--noclean', action='store_false', dest='clean',
-                    help='do not remove temp files',
-                    default=True)
-  parser.add_option('--local', metavar='DIR',
-                    help='use downloaded files from DIR')
-  parser.add_option('--express',
-                    help='use VS Express instead of Pro', action='store_true')
-  parser.add_option('--sha1',
-                    help='tree sha1 that can be used to mirror an internal '
-                         'copy (used if --use-gs)')
-  parser.add_option('--use-gs',
-                    help='Use internal servers to pull isos',
-                    default=bool(int(os.environ.get('CHROME_HEADLESS', 0))),
-                    action='store_true')
-  options, _ = parser.parse_args()
-  try:
-    target_dir = os.path.abspath(options.targetdir)
-    if os.path.exists(target_dir):
-      parser.error('%s already exists. Please [re]move it or use '
-                  '--targetdir to select a different target.\n' %
-                  target_dir)
-    # Set the working directory to 7z subdirectory. 7-zip doesn't find its
-    # codec dll very well, so this is the simplest way to make sure it runs
-    # correctly, as we don't otherwise care about working directory.
-    os.chdir(os.path.join(BASEDIR, '7z'))
-    if options.use_gs and options.sha1:
-      options.express = False
-      DoTreeMirror(target_dir, options.sha1)
-    else:
-      images = GetSourceImages(options.local, not options.express)
-      extracted = ExtractComponents(images)
-      CopyToFinalLocation(extracted, target_dir)
-      GenerateSetEnvCmd(target_dir, not options.express)
-
-    data = {
-        'path': target_dir,
-        'version': '2013e' if options.express else '2013',
-        'win8sdk': os.path.join(target_dir, 'win8sdk'),
-        'wdk': os.path.join(target_dir, 'wdk'),
-        'runtime_dirs': [
-          os.path.join(target_dir, 'sys64'),
-          os.path.join(target_dir, 'sys32'),
-        ],
-    }
-    with open(os.path.join(target_dir, '..', 'data.json'), 'w') as f:
-      json.dump(data, f)
-  finally:
-    if options.clean:
-      DeleteAllTempDirs()
-
-
-if __name__ == '__main__':
-  sys.exit(main())