tools addition to automate setting up windows toolchain

tools/win/toolchain/toolchain.py

+# Copyright 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Extracts a Windows toolchain suitable for building Chrome from various
+# downloadable pieces.
+
+
+import os
+import shutil
+import sys
+import tempfile
+import urllib2
+
+
+g_temp_dirs = []
+
+
+def TempDir():
+  """Generate a temporary directory (for downloading or extracting to) and keep
+  track of the directory that's created for cleaning up later."""
+  global g_temp_dirs
+  temp = tempfile.mkdtemp()
+  g_temp_dirs.append(temp)
+  return temp
+
+
+def DeleteAllTempDirs():
+  """Remove all temporary directories created by |TempDir()|."""
+  global g_temp_dirs
+  sys.stdout.write('Cleaning up temporaries...\n')
+  for temp in g_temp_dirs:
+    # shutil.rmtree errors out on read only attributes.
+    os.system('rmdir /s/q "%s"' % temp)
+  g_temp_dirs = []
+
+
+def Download(url, local_path):
+  """Download a large-ish binary file and print some status information while
+  doing so."""
+  sys.stdout.write('Downloading %s...' % url)
+  req = urllib2.urlopen(url)
+  content_length = int(req.headers.get('Content-Length', 0))
+  bytes_read = 0
+  with open(local_path, 'wb') as file:
+    while True:
+      chunk = req.read(1024 * 1024)
+      if not chunk:
+        break
+      bytes_read = len(chunk)
+      file.write(chunk)
+      sys.stdout.write('.')
+  sys.stdout.write('\n')
+  if content_length and content_length != bytes_read:
+    raise SystemExit('Got incorrect number of bytes downloading %s' % url)
+
+
+def DownloadSDK71Iso():
+  sdk7_temp_dir = TempDir()
+  target_path = os.path.join(sdk7_temp_dir, 'GRMSDKX_EN_DVD.iso')
+  Download(
+      ('http://download.microsoft.com/download/'
+       'F/1/0/F10113F5-B750-4969-A255-274341AC6BCE/GRMSDKX_EN_DVD.iso'),
+      target_path)
+  return target_path
+
+
+def DownloadWDKIso():
+  wdk_temp_dir = TempDir()
+  target_path = os.path.join(wdk_temp_dir, 'GRMWDK_EN_7600_1.ISO')
+  Download(
+      ('http://download.microsoft.com/download/'
+       '4/A/2/4A25C7D5-EFBE-4182-B6A9-AE6850409A78/GRMWDK_EN_7600_1.ISO'),
+      target_path)
+  return target_path
+
+
+def DownloadWDKUpdate():

[alexeypa (please no reviews), 2013/01/02 19:33:57]

nit: This is not WDK update. This is SDK 7.1 update.

[scottmg, 2013/01/02 21:00:48]

Oops, thanks. Done.

+  wdk_update_temp_dir = TempDir()
+  target_path = os.path.join(wdk_update_temp_dir, 'VC-Compiler-KB2519277.exe')
+  Download(
+      ('http://download.microsoft.com/download/'
+        '7/5/0/75040801-126C-4591-BCE4-4CD1FD1499AA/VC-Compiler-KB2519277.exe'),
+      target_path)
+  return target_path
+
+
+def DownloadDirectXSDK():
+  dxsdk_temp_dir = TempDir()
+  target_path = os.path.join(dxsdk_temp_dir, 'DXSDK_Jun10.exe')
+  Download(
+      ('http://download.microsoft.com/download/'
+       'A/E/7/AE743F1F-632B-4809-87A9-AA1BB3458E31/DXSDK_Jun10.exe'),
+      target_path)
+  return target_path
+
+
+def DownloadSDK8():
+  """Download the Win8 SDK. This one is slightly different than the simple
+  ones above. There is no .ISO distribution for the Windows 8 SDK. Rather, a
+  tool is provided that is a download manager. This is used to download the
+  various .msi files to a target location. Unfortunately, this tool requires
+  elevation for no obvious reason even when only downloading, so this function
+  will trigger a UAC elevation if the script is not run from an elevated
+  prompt."""
+  sdk_temp_dir = TempDir()
+  target_path = os.path.join(sdk_temp_dir, 'sdksetup.exe')
+  standalone_path = os.path.join(sdk_temp_dir, 'Standalone')
+  Download(
+      ('http://download.microsoft.com/download/'
+       'F/1/3/F1300C9C-A120-4341-90DF-8A52509B23AC/standalonesdk/sdksetup.exe'),
+      target_path)
+  count = 0
+  while count < 5:
+    rc = os.system(target_path + ' /quiet '
+              '/features OptionId.WindowsDesktopSoftwareDevelopmentKit '
+              '/layout ' + standalone_path)
+    if rc == 0:
+      return standalone_path
+      break
+    count += 1
+    sys.stdout.write('Windows 8 SDK failed to download, retrying.\n')
+  raise SystemExit("After multiple retries, couldn't download Win8 SDK")
+
+
+def ExtractIso(iso_path):
+  """Use 7zip to extract the contents of the given .iso (or self-extracting
+  .exe)."""
+  target_path = TempDir()
+  sys.stdout.write('Extracting %s...\n' % iso_path)
+  # TODO(scottmg): Do this (and exe) manually with python code.
+  if os.system('7z x "%s" -y "-o%s" > nul' % (iso_path, target_path)) != 0:
+    raise SystemExit("Couldn't extract %s" % iso_path)
+  return target_path
+
+
+ExtractExe = ExtractIso
+
+
+def ExtractMsi(msi_path):
+  """Use msiexec to extract the contents of the given .msi file."""
+  sys.stdout.write('Extracting %s...\n' % msi_path)
+  target_path = TempDir()
+  if os.system(
+      'msiexec /a "%s" /qn TARGETDIR=%s' % (msi_path, target_path)) != 0:
+    raise SystemExit("Couldn't extract %s" % msi_path)
+  os.unlink(os.path.join(target_path, os.path.basename(msi_path)))

[alexeypa (please no reviews), 2013/01/02 19:33:57]

Is this needed? The script deletes all temporary files it created at the end.

[scottmg, 2013/01/02 21:00:48]

Done.

+  return target_path
+
+
+def PullFrom(list_of_path_pairs, source_root, target_dir):
+  """Each pair in |list_of_path_pairs| is (from, to). Join the 'from' with
+  |source_root| and the 'to' with |target_dir| and perform a recursive copy."""
+  for source, destination in list_of_path_pairs:
+    full_source = os.path.join(source_root, source)
+    full_target = os.path.join(target_dir, destination)
+    rc = os.system('robocopy /s "%s" "%s" >nul' % (full_source, full_target))
+    if (rc & 8) != 0 or (rc & 16) != 0:
+      # ref: http://ss64.com/nt/robocopy-exit.html
+      raise SystemExit("Couldn't copy %s to %s" % (full_source, full_target))
+
+
+def PatchAsyncInfo(target_dir):
+  """Apply patch from
+  http://www.chromium.org/developers/how-tos/build-instructions-windows for
+  asyncinfo.h."""
+  sys.stdout.write('Patching asyncinfo.h...\n')
+  asyncinfo_h_path = os.path.join(
+      target_dir, r'win8sdk\Include\winrt\asyncinfo.h')
+  with open(asyncinfo_h_path, 'rb') as f:
+    asyncinfo_h = f.read()
+  patched = asyncinfo_h.replace(
+      'enum class AsyncStatus {', 'enum AsyncStatus {')
+  with open(asyncinfo_h_path, 'wb') as f:
+    f.write(patched)
+
+
+
+def GenerateSetEnvCmd(target_dir):
+  """Generate a batch file that gyp expects to exist to set up the compiler
+  environment. This is normally generated by a full install of the SDK, but we
+  do it here manually since we do not do a full install."""
+  with open(os.path.join(
+        target_dir, r'win8sdk\bin\SetEnv.cmd'), 'w') as file:
+    file.write('@echo off\n')
+    file.write(':: Generated by tools\\win\\toolchain\\toolchain.py.\n')
+    # Common to x86 and x64
+    file.write('set PATH=%s;%%PATH%%\n' % (
+        os.path.join(target_dir, r'Common7\IDE')))
+    file.write('set INCLUDE=%s;%s;%s\n' % (
+        os.path.join(target_dir, r'win8sdk\Include\um'),
+        os.path.join(target_dir, r'win8sdk\Include\shared'),
+        os.path.join(target_dir, r'VC\include')))
+    file.write('if "%1"=="/x64" goto x64\n')
+
+    # x86 only.
+    file.write('set PATH=%s;%s;%s;%%PATH%%\n' % (
+        os.path.join(target_dir, r'win8sdk\bin\x86'),
+        os.path.join(target_dir, r'VC\bin'),
+        os.path.join(target_dir, r'WDK\bin')))
+    file.write('set LIB=%s;%s\n' % (
+        os.path.join(target_dir, r'VC\lib'),
+        os.path.join(target_dir, r'win8sdk\Lib\win8\um\x86')))
+    file.write('goto done\n')
+
+    # x64 only.
+    file.write(':x64\n')
+    file.write('set PATH=%s;%s;%s;%%PATH%%\n' % (
+        os.path.join(target_dir, r'win8sdk\bin\x64'),
+        os.path.join(target_dir, r'VC\bin\amd64'),
+        os.path.join(target_dir, r'WDK\bin\amd64')))
+    file.write('set LIB=%s;%s\n' % (
+        os.path.join(target_dir, r'VC\lib\amd64'),
+        os.path.join(target_dir, r'win8sdk\Lib\win8\um\x64')))
+
+    file.write(':done\n')
+
+
+def GenerateTopLevelEnv(target_dir):
+  """Generate a batch file that sets up various environment variables that let
+  the Chromium build files and gyp find SDKs and tools."""
+  with open(os.path.join(target_dir, r'env.bat'), 'w') as file:
+    file.write('@echo off\n')
+    file.write(':: Generated by tools\\win\\toolchain\\toolchain.py.\n')
+    file.write('set GYP_DEFINES=windows_sdk_path="%s" '
+                'component=shared_library\n' % (
+                    os.path.join(target_dir, 'win8sdk')))
+    file.write('set GYP_MSVS_VERSION=2010e\n')
+    file.write('set GYP_MSVS_OVERRIDE_PATH=%s\n' % target_dir)
+    file.write('set GYP_GENERATORS=ninja\n')
+    file.write('set WDK_DIR=%s\n' % os.path.join(target_dir, r'WDK'))
+    file.write('set DXSDK_DIR=%s\n' % os.path.join(target_dir, r'DXSDK'))
+    file.write('set WindowsSDKDir=%s\n' %
+        os.path.join(target_dir, r'win8sdk'))
+    file.write('echo Environment set for toolchain in %s.\n' % target_dir)
+    file.write('cd /d %s\\..\n' % target_dir)
+
+
+def main():
+  try:
+    target_dir = os.path.abspath('win_toolchain')
+    os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
+    if len(sys.argv) == 2 and sys.argv[1] == 'local':
+      sdk_path = r'C:\Users\Scott\Desktop\wee\Standalone'
+      wdk_iso = r'c:\users\scott\desktop\wee\GRMWDK_EN_7600_1.ISO'
+      wdk_update = r'c:\users\scott\desktop\wee\VC-Compiler-KB2519277.exe'
+      sdk7_path = r'C:\Users\Scott\Desktop\wee\GRMSDKX_EN_DVD.ISO'
+      dxsdk_path = r'C:\Users\Scott\Desktop\wee\DXSDK_Jun10.exe'
+    else:
+      # Note that we do the Win8 SDK first so that its silly UAC prompt
+      # happens before the user wanders off to get coffee.
+      sdk_path = DownloadSDK8()
+      wdk_iso = DownloadWDKIso()
+      wdk_update = DownloadWDKUpdate()
+      sdk7_path = DownloadSDK71Iso()
+      dxsdk_path = DownloadDirectXSDK()
+
+    extracted_sdk7 = ExtractIso(sdk7_path)
+    extracted_vc_x86 = \
+        ExtractMsi(os.path.join(extracted_sdk7,
+                                r'Setup\vc_stdx86\vc_stdx86.msi'))
+    extracted_vc_x64 = \
+        ExtractMsi(os.path.join(extracted_sdk7,
+                                r'Setup\\vc_stdamd64\vc_stdamd64.msi'))
+
+    extracted_wdk = ExtractIso(wdk_iso)
+    extracted_buildtools_x86 = \
+        ExtractMsi(os.path.join(extracted_wdk, 'WDK\\buildtools_x86fre.msi'))
+    extracted_buildtools_x64 = \
+        ExtractMsi(os.path.join(extracted_wdk, 'WDK\\buildtools_x64fre.msi'))
+    extracted_libs_x86 = \
+        ExtractMsi(os.path.join(extracted_wdk, 'WDK\\libs_x86fre.msi'))
+    extracted_libs_x64 = \
+        ExtractMsi(os.path.join(extracted_wdk, 'WDK\\libs_x64fre.msi'))
+    extracted_headers = \
+        ExtractMsi(os.path.join(extracted_wdk, 'WDK\\headers.msi'))
+
+    extracted_update = ExtractExe(wdk_update)
+    extracted_update_x86 = \
+        ExtractMsi(os.path.join(extracted_update, 'vc_stdx86.msi'))
+    extracted_update_x64 = \
+        ExtractMsi(os.path.join(extracted_update, 'vc_stdamd64.msi'))
+
+    sdk_msi_path = os.path.join(
+        sdk_path, 'Installers\\Windows Software Development Kit-x86_en-us.msi')
+    extracted_sdk_path = ExtractMsi(sdk_msi_path)
+
+    sdk_metro_msi_path = os.path.join(
+        sdk_path,
+        'Installers',
+        'Windows Software Development Kit for Metro style Apps-x86_en-us.msi')
+    extracted_metro_sdk_path = ExtractMsi(sdk_metro_msi_path)
+
+    extracted_dxsdk = ExtractExe(dxsdk_path)
+
+    sys.stdout.write('Pulling together required pieces...\n')
+
+    # Note that order is important because some of the older ones are
+    # overwritten by updates.
+    from_vcupdate_x86 = [
+        (r'Program Files\Microsoft Visual Studio 10.0', '.'),
+        (r'Win\System', r'VC\bin'),
+        ]
+    PullFrom(from_vcupdate_x86, extracted_update_x86, target_dir)
+
+    from_vcupdate_x64 = [
+        (r'Program Files(64)\Microsoft Visual Studio 10.0', '.'),
+        (r'Win\System64', r'VC\bin\amd64'),
+        ]
+    PullFrom(from_vcupdate_x64, extracted_update_x64, target_dir)

[alexeypa (please no reviews), 2013/01/02 19:33:57]

nit: One small concern I have is that PullFrom() does not fully recreates what
the installer does, so the environment created by this script can be different
from the "stock" environment in subtle ways.

For instance VC-Compiler-KB2519277.exe deletes ammintrin.h header breaking the
build. While removing this header is a bug in the updater, there could be other
headers/libraries that should actually be deleted.

I suspect this is not a real problem but it might be worth comparing the list of
files we get via the traditional install and this script.

[scottmg, 2013/01/02 21:00:48]

It took me a while to figure out what was going on with ammintrin.h! I am
concerned about subtle differences too. My handwave-y theory was that if it's
able to build Chromium and run some set of tests on a bot, it's probably close
enough. Having said that, I'm sure we'll need to improve the "install emulation"
over time.

+
+    from_sdk7_x86 =[
+        (r'Program Files\Microsoft Visual Studio 10.0', '.'),
+        (r'Win\System', r'VC\bin'),
+        ]
+    PullFrom(from_sdk7_x86, extracted_vc_x86, target_dir)
+
+    from_sdk7_x64 =[
+        (r'Program Files(64)\Microsoft Visual Studio 10.0', '.'),
+        (r'Win\System64', r'VC\bin\amd64'),
+        ]
+    PullFrom(from_sdk7_x64, extracted_vc_x64, target_dir)
+
+    from_sdk = [(r'Windows Kits\8.0', r'win8sdk')]
+    PullFrom(from_sdk, extracted_sdk_path, target_dir)
+
+    from_metro_sdk = [(r'Windows Kits\8.0', r'win8sdk')]
+    PullFrom(from_sdk, extracted_metro_sdk_path, target_dir)
+
+    from_buildtools_x86 = [
+        (r'WinDDK\7600.16385.win7_wdk.100208-1538\bin\x86', r'WDK\bin'),
+        (r'WinDDK\7600.16385.win7_wdk.100208-1538\bin\amd64', r'WDK\bin'),
+        ]
+    PullFrom(from_buildtools_x86, extracted_buildtools_x86, target_dir)
+
+    from_buildtools_x64 = [
+        (r'WinDDK\7600.16385.win7_wdk.100208-1538\bin\amd64', r'WDK\bin'),
+        ]
+    PullFrom(from_buildtools_x64, extracted_buildtools_x64, target_dir)
+
+    from_libs_x86 = [
+        (r'WinDDK\7600.16385.win7_wdk.100208-1538\lib', r'WDK\lib'),
+        ]
+    PullFrom(from_libs_x86, extracted_libs_x86, target_dir)
+
+    from_libs_x64 = [
+        (r'WinDDK\7600.16385.win7_wdk.100208-1538\lib', r'WDK\lib'),
+        ]
+    PullFrom(from_libs_x64, extracted_libs_x64, target_dir)
+
+    from_headers = [
+        (r'WinDDK\7600.16385.win7_wdk.100208-1538\inc', r'WDK\inc'),
+        ]
+    PullFrom(from_headers, extracted_headers, target_dir)
+
+    from_dxsdk = [
+        (r'DXSDK\Include', r'DXSDK\Include'),
+        (r'DXSDK\Lib', r'DXSDK\Lib'),
+        (r'DXSDK\Redist', r'DXSDK\Redist'),
+        ]
+    PullFrom(from_dxsdk, extracted_dxsdk, target_dir)
+
+    PatchAsyncInfo()
+
+    GenerateSetEnvCmd(target_dir)
+    GenerateTopLevelEnv(target_dir)
+  finally:
+    DeleteAllTempDirs()
+
+  sys.stdout.write(
+      '\nIn a (clean) cmd shell, you can now run\n\n'
+      '  %s\\env.bat\n\n'
+      'then\n\n'
+      "  gclient runhooks (or gclient sync if you haven't pulled deps yet)\n"
+      '  ninja -C out\Debug chrome\n\n'
+      'Note that this script intentionally does not modify any global\n'
+      'settings like the registry, or system environment variables, so you\n'
+      'will need to run the above env.bat whenever you start a new\n'
+      'shell.\n\n' % target_dir)
+
+
+if __name__ == '__main__':
+  main()