tools addition to automate setting up windows toolchain

tools/win/toolchain/toolchain.py

+# Copyright 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Extracts a Windows toolchain suitable for building Chrome from various
+# downloadable pieces.
+
+
+import os
+import shutil
+import sys
+import tempfile
+import urllib2
+
+
+g_temp_dirs = []
+
+
+def TempDir():
+  """Generate a temporary directory (for downloading or extracting to) and keep
+  track of the directory that's created for cleaning up later."""
+  global g_temp_dirs
+  temp = tempfile.mkdtemp()
+  g_temp_dirs.append(temp)
+  return temp
+
+
+def DeleteAllTempDirs():
+  """Remove all temporary directories created by |TempDir()|."""
+  global g_temp_dirs
+  print 'Cleaning up temporaries...'
+  for temp in g_temp_dirs:
+    # shutil.rmtree errors out on read only attributes.
+    os.system('rmdir /s/q "%s"' % temp)
+  g_temp_dirs = []
+
+
+def Download(url, local_path):
+  """Download a large-ish binary file and print some status information while
+  doing so."""
+  sys.stdout.write('Downloading %s...' % url)

[Roger McFarlane (Chromium), 2013/01/02 18:46:23]

You're got print and sys.stdout.write() used in various spots. Maybe pick one
and use it consistently.

If you were trying to avoid the newline from print: you can add a trailing comma
to suppress the newline.

[scottmg, 2013/01/02 21:00:47]

Yeah, I had that in ps #17, but it also needs sys.stdout.softspace = 0 to avoid
the magic space behaviour of print. I'll switch to sys.stdout.write everywhere I
guess.

+  req = urllib2.urlopen(url)
+  with open(local_path, 'wb') as file:
+    while True:
+      chunk = req.read(1024 * 1024)
+      if not chunk:
+        break
+      file.write(chunk)
+      sys.stdout.write('.')
+  print

[Roger McFarlane (Chromium), 2013/01/02 18:46:23]

an errant "print"?

It would be nice to track the total bytes read, compare it with the
Content-Length (if given in the response), and raise/retry on mismatch. 

[scottmg, 2013/01/02 21:00:47]

was for \n.
Done.

+
+
+def DownloadSDK71Iso():
+  sdk7_temp_dir = TempDir()
+  target_path = os.path.join(sdk7_temp_dir, 'GRMSDKX_EN_DVD.iso')
+  Download(
+      ('http://download.microsoft.com/download/'
+       'F/1/0/F10113F5-B750-4969-A255-274341AC6BCE/GRMSDKX_EN_DVD.iso'),
+      target_path)
+  return target_path
+
+
+def DownloadWDKIso():
+  wdk_temp_dir = TempDir()
+  target_path = os.path.join(wdk_temp_dir, 'GRMWDK_EN_7600_1.ISO')
+  Download(
+      ('http://download.microsoft.com/download/'
+       '4/A/2/4A25C7D5-EFBE-4182-B6A9-AE6850409A78/GRMWDK_EN_7600_1.ISO'),
+      target_path)
+  return target_path
+
+
+def DownloadWDKUpdate():
+  wdk_update_temp_dir = TempDir()
+  target_path = os.path.join(wdk_update_temp_dir, 'VC-Compiler-KB2519277.exe')
+  Download(
+      ('http://download.microsoft.com/download/'
+        '7/5/0/75040801-126C-4591-BCE4-4CD1FD1499AA/VC-Compiler-KB2519277.exe'),
+      target_path)
+  return target_path
+
+
+def DownloadDirectXSDK():
+  dxsdk_temp_dir = TempDir()
+  target_path = os.path.join(dxsdk_temp_dir, 'DXSDK_Jun10.exe')
+  Download(
+      ('http://download.microsoft.com/download/'
+       'A/E/7/AE743F1F-632B-4809-87A9-AA1BB3458E31/DXSDK_Jun10.exe'),
+      target_path)
+  return target_path
+
+
+def DownloadSDK8():
+  """Download the Win8 SDK. This one is slightly different than the simple ones
+  above. There is no .ISO distribution for the Windows 8 SDK. Rather, a tool
+  is provided that is a download manager. This is used to download the various
+  .msis to a target location. Unfortunately, this tool requires elevation for

[Roger McFarlane (Chromium), 2013/01/02 18:46:23]

.MSIs or .msi files

[scottmg, 2013/01/02 21:00:47]

Done.

+  no obvious reason even when only downloading, so this function will trigger
+  a UAC elevation if the script is not run from an elevated prompt."""
+  sdk_temp_dir = TempDir()
+  target_path = os.path.join(sdk_temp_dir, 'sdksetup.exe')
+  standalone_path = os.path.join(sdk_temp_dir, 'Standalone')
+  Download(
+      ('http://download.microsoft.com/download/'
+       'F/1/3/F1300C9C-A120-4341-90DF-8A52509B23AC/standalonesdk/sdksetup.exe'),
+      target_path)
+  count = 0
+  while count < 5:
+    rc = os.system(target_path + ' /quiet '
+              '/features OptionId.WindowsDesktopSoftwareDevelopmentKit '
+              '/layout ' + standalone_path)
+    if rc == 0:
+      return standalone_path
+      break
+    count += 1
+    print 'Windows 8 SDK failed to download, retrying.'
+  raise SystemExit("After multiple retries, couldn't download Win8 SDK")
+
+
+def ExtractIso(iso_path):
+  """Use 7zip to extract the contents of the given .iso (or self-extracting
+  .exe)."""
+  target_path = TempDir()
+  print 'Extracting %s...' % iso_path
+  # TODO(scottmg): Do this (and exe) manually with python code.
+  if os.system('7z x "%s" -y "-o%s" > nul' % (iso_path, target_path)) != 0:
+    raise SystemExit("Couldn't extract %s" % iso_path)

[Roger McFarlane (Chromium), 2013/01/02 18:46:23]

maybe infer the path to 7z; otherwise, you're assuming that the user is running
the script from the same directory.

  os.path.join(os.path.dirname(__file__), '7z.exe'))

[scottmg, 2013/01/02 21:00:47]

Done.

+  return target_path
+
+
+ExtractExe = ExtractIso
+
+
+def ExtractMsi(msi_path):
+  """Use msiexec to extract the contents of the given .msi file."""
+  print 'Extracting %s...' % msi_path
+  target_path = TempDir()
+  if os.system(
+      'msiexec /a "%s" /qn TARGETDIR=%s' % (msi_path, target_path)) != 0:
+    raise SystemExit("Couldn't extract %s" % msi_path)
+  os.unlink(os.path.join(target_path, os.path.basename(msi_path)))
+  return target_path
+
+
+def PullFrom(list_of_path_pairs, source_root, target_dir):
+  """Each pair in |list_of_path_pairs| is (from, to). Join the 'from' with
+  |source_root| and the 'to' with |target_dir| and perform a recursive copy."""
+  for source, destination in list_of_path_pairs:
+    full_source = os.path.join(source_root, source)
+    full_target = os.path.join(target_dir, destination)
+    rc = os.system('robocopy /s "%s" "%s" >nul' % (full_source, full_target))
+    if (rc & 8) != 0 or (rc & 16) != 0:
+      # ref: http://ss64.com/nt/robocopy-exit.html
+      raise SystemExit("Couldn't copy %s to %s" % (full_source, full_target))
+
+
+def GenerateSetEnvCmd(target_dir):
+  """Generate a batch file that gyp expects to exist to set up the compiler
+  environment. This is normally generated by a full install of the SDK, but we
+  do it here manually since we do not do a full install."""
+  with open(os.path.join(
+        target_dir, r'win8sdk\bin\SetEnv.cmd'), 'w') as file:
+    file.write('@echo off\n')
+    file.write(':: Generated by tools\\win\\toolchain\\toolchain.py.\n')
+    # Common to x86 and x64
+    file.write('set PATH=%s;%%PATH%%\n' % (
+        os.path.join(target_dir, r'Common7\IDE')))
+    file.write('set INCLUDE=%s;%s;%s\n' % (
+        os.path.join(target_dir, r'win8sdk\Include\um'),
+        os.path.join(target_dir, r'win8sdk\Include\shared'),
+        os.path.join(target_dir, r'VC\include')))
+    file.write('if "%1"=="/x64" goto x64\n')
+
+    # x86 only.
+    file.write('set PATH=%s;%s;%s;%%PATH%%\n' % (
+        os.path.join(target_dir, r'win8sdk\bin\x86'),
+        os.path.join(target_dir, r'VC\bin'),
+        os.path.join(target_dir, r'WDK\bin')))
+    file.write('set LIB=%s;%s\n' % (
+        os.path.join(target_dir, r'VC\lib'),
+        os.path.join(target_dir, r'win8sdk\Lib\win8\um\x86')))
+    file.write('goto done\n')
+
+    # x64 only.
+    file.write(':x64\n')
+    file.write('set PATH=%s;%s;%s;%%PATH%%\n' % (
+        os.path.join(target_dir, r'win8sdk\bin\x64'),
+        os.path.join(target_dir, r'VC\bin\amd64'),
+        os.path.join(target_dir, r'WDK\bin\amd64')))
+    file.write('set LIB=%s;%s\n' % (
+        os.path.join(target_dir, r'VC\lib\amd64'),
+        os.path.join(target_dir, r'win8sdk\Lib\win8\um\x64')))
+
+    file.write(':done\n')
+
+
+def GenerateTopLevelEnv(target_dir):
+  """Generate a batch file that sets up various environment variables that let
+  the Chromium build files and gyp find SDKs and tools."""
+  with open(os.path.join(target_dir, r'env.bat'), 'w') as file:
+    file.write('@echo off\n')
+    file.write(':: Generated by tools\\win\\toolchain\\toolchain.py.\n')
+    file.write('set GYP_DEFINES=windows_sdk_path="%s" '
+                'component=shared_library\n' % (
+                    os.path.join(target_dir, 'win8sdk')))
+    file.write('set GYP_MSVS_VERSION=2010e\n')
+    file.write('set GYP_MSVS_OVERRIDE_PATH=%s\n' % target_dir)
+    file.write('set GYP_GENERATORS=ninja\n')
+    file.write('set WDK_DIR=%s\n' % os.path.join(target_dir, r'WDK'))
+    file.write('set DXSDK_DIR=%s\n' % os.path.join(target_dir, r'DXSDK'))
+    file.write('set WindowsSDKDir=%s\n' %
+        os.path.join(target_dir, r'win8sdk'))
+    file.write('echo Environment set for toolchain in %s.\n' % target_dir)
+    file.write('cd /d %s\\..\n' % target_dir)
+
+
+def main():

[Roger McFarlane (Chromium), 2013/01/02 18:46:23]

nit: Can this easiliy be broken down further? maybe with a structure or
dictionary tying it all together? If not, add some more running commentary.

[scottmg, 2013/01/02 21:00:47]

Done.

+  try:
+    target_dir = os.path.abspath('win_toolchain')
+    os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
+    if len(sys.argv) == 2 and sys.argv[1] == 'local':

[scottmg, 2013/01/02 17:32:10]

will remove this after reviewed

+      sdk_path = r'C:\Users\Scott\Desktop\wee\Standalone'
+      wdk_iso = r'c:\users\scott\desktop\wee\GRMWDK_EN_7600_1.ISO'
+      wdk_update = r'c:\users\scott\desktop\wee\VC-Compiler-KB2519277.exe'
+      sdk7_path = r'C:\Users\Scott\Desktop\wee\GRMSDKX_EN_DVD.ISO'
+      dxsdk_path = r'C:\Users\Scott\Desktop\wee\DXSDK_Jun10.exe'
+    else:
+      # Note that we do the Win8 SDK first so that its silly UAC prompt
+      # happens before the user wanders off to get coffee.

[Roger McFarlane (Chromium), 2013/01/02 18:46:23]

Ha! Nice comment.

+      sdk_path = DownloadSDK8()
+      wdk_iso = DownloadWDKIso()
+      wdk_update = DownloadWDKUpdate()
+      sdk7_path = DownloadSDK71Iso()
+      dxsdk_path = DownloadDirectXSDK()
+
+    extracted_sdk7 = ExtractIso(sdk7_path)
+    extracted_vc_x86 = \
+        ExtractMsi(os.path.join(extracted_sdk7,
+                                r'Setup\vc_stdx86\vc_stdx86.msi'))
+    extracted_vc_x64 = \
+        ExtractMsi(os.path.join(extracted_sdk7,
+                                r'Setup\\vc_stdamd64\vc_stdamd64.msi'))
+
+    extracted_wdk = ExtractIso(wdk_iso)
+    extracted_buildtools_x86 = \
+        ExtractMsi(os.path.join(extracted_wdk, 'WDK\\buildtools_x86fre.msi'))
+    extracted_buildtools_x64 = \
+        ExtractMsi(os.path.join(extracted_wdk, 'WDK\\buildtools_x64fre.msi'))
+    extracted_libs_x86 = \
+        ExtractMsi(os.path.join(extracted_wdk, 'WDK\\libs_x86fre.msi'))
+    extracted_libs_x64 = \
+        ExtractMsi(os.path.join(extracted_wdk, 'WDK\\libs_x64fre.msi'))
+    extracted_headers = \
+        ExtractMsi(os.path.join(extracted_wdk, 'WDK\\headers.msi'))
+
+    extracted_update = ExtractExe(wdk_update)
+    extracted_update_x86 = \
+        ExtractMsi(os.path.join(extracted_update, 'vc_stdx86.msi'))
+    extracted_update_x64 = \
+        ExtractMsi(os.path.join(extracted_update, 'vc_stdamd64.msi'))
+
+    sdk_msi_path = os.path.join(
+        sdk_path, 'Installers\\Windows Software Development Kit-x86_en-us.msi')
+    extracted_sdk_path = ExtractMsi(sdk_msi_path)
+
+    sdk_metro_msi_path = os.path.join(
+        sdk_path,
+        'Installers',
+        'Windows Software Development Kit for Metro style Apps-x86_en-us.msi')
+    extracted_metro_sdk_path = ExtractMsi(sdk_metro_msi_path)
+
+    extracted_dxsdk = ExtractExe(dxsdk_path)
+
+    print 'Pulling together required pieces...'
+
+    # Note that order is important because some of the older ones are
+    # overwritten by updates.
+    from_vcupdate_x86 = [
+        (r'Program Files\Microsoft Visual Studio 10.0', '.'),
+        (r'Win\System', r'VC\bin'),
+        ]
+    PullFrom(from_vcupdate_x86, extracted_update_x86, target_dir)
+
+    from_vcupdate_x64 = [
+        (r'Program Files(64)\Microsoft Visual Studio 10.0', '.'),
+        (r'Win\System64', r'VC\bin\amd64'),
+        ]
+    PullFrom(from_vcupdate_x64, extracted_update_x64, target_dir)
+
+    from_sdk7_x86 =[
+        (r'Program Files\Microsoft Visual Studio 10.0', '.'),
+        (r'Win\System', r'VC\bin'),
+        ]
+    PullFrom(from_sdk7_x86, extracted_vc_x86, target_dir)
+
+    from_sdk7_x64 =[
+        (r'Program Files(64)\Microsoft Visual Studio 10.0', '.'),
+        (r'Win\System64', r'VC\bin\amd64'),
+        ]
+    PullFrom(from_sdk7_x64, extracted_vc_x64, target_dir)
+
+    from_sdk = [(r'Windows Kits\8.0', r'win8sdk')]
+    PullFrom(from_sdk, extracted_sdk_path, target_dir)
+
+    from_metro_sdk = [(r'Windows Kits\8.0', r'win8sdk')]
+    PullFrom(from_sdk, extracted_metro_sdk_path, target_dir)
+
+    from_buildtools_x86 = [
+        (r'WinDDK\7600.16385.win7_wdk.100208-1538\bin\x86', r'WDK\bin'),
+        (r'WinDDK\7600.16385.win7_wdk.100208-1538\bin\amd64', r'WDK\bin'),
+        ]
+    PullFrom(from_buildtools_x86, extracted_buildtools_x86, target_dir)
+
+    from_buildtools_x64 = [
+        (r'WinDDK\7600.16385.win7_wdk.100208-1538\bin\amd64', r'WDK\bin'),
+        ]
+    PullFrom(from_buildtools_x64, extracted_buildtools_x64, target_dir)
+
+    from_libs_x86 = [
+        (r'WinDDK\7600.16385.win7_wdk.100208-1538\lib', r'WDK\lib'),
+        ]
+    PullFrom(from_libs_x86, extracted_libs_x86, target_dir)
+
+    from_libs_x64 = [
+        (r'WinDDK\7600.16385.win7_wdk.100208-1538\lib', r'WDK\lib'),
+        ]
+    PullFrom(from_libs_x64, extracted_libs_x64, target_dir)
+
+    from_headers = [
+        (r'WinDDK\7600.16385.win7_wdk.100208-1538\inc', r'WDK\inc'),
+        ]
+    PullFrom(from_headers, extracted_headers, target_dir)
+
+    from_dxsdk = [
+        (r'DXSDK\Include', r'DXSDK\Include'),
+        (r'DXSDK\Lib', r'DXSDK\Lib'),
+        (r'DXSDK\Redist', r'DXSDK\Redist'),
+        ]
+    PullFrom(from_dxsdk, extracted_dxsdk, target_dir)
+
+    # Apply patch from

[Roger McFarlane (Chromium), 2013/01/02 18:46:23]

put this in its own function?

[scottmg, 2013/01/02 21:00:47]

Done.

+    # http://www.chromium.org/developers/how-tos/build-instructions-windows
+    # for asyncinfo.h:
+    print 'Patching asyncinfo.h...'
+    asyncinfo_h_path = os.path.join(
+        target_dir, r'win8sdk\Include\winrt\asyncinfo.h')
+    asyncinfo_h = open(asyncinfo_h_path, 'rb').read()

[Roger McFarlane (Chromium), 2013/01/02 18:46:23]

Do the read inside a with clause?

[scottmg, 2013/01/02 21:00:47]

Sure. (fwiw, in "regular" CPython, it's guaranteed to be closed at the end of
the statement, which is why the write succeeds without a sharing violation.)

+    patched = asyncinfo_h.replace(
+        'enum class AsyncStatus {', 'enum AsyncStatus {')
+    open(asyncinfo_h_path, 'wb').write(patched)

[Roger McFarlane (Chromium), 2013/01/02 18:46:23]

Do the write inside a with clause?

[scottmg, 2013/01/02 21:00:47]

Done.

+
+    GenerateSetEnvCmd(target_dir)
+    GenerateTopLevelEnv(target_dir)
+  finally:
+    DeleteAllTempDirs()
+
+  print ('\nIn a (clean) cmd shell, you can now run\n\n'
+         '  %s\\env.bat\n\n'
+         'then\n\n'
+         "  gclient runhooks (or gclient sync if you haven't pulled deps yet)\n"
+         '  ninja -C out\Debug chrome\n\n'
+         'Note that this script intentionally does not modify any global\n'
+         'settings like the registry, or system environment variables, so you\n'
+         'will need to run the above env.bat whenever you start a new\n'
+         'shell.\n' % target_dir)
+
+
+if __name__ == '__main__':
+  main()