Automatic Windows toolchain

win_toolchain/toolchain2013.py

+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Extracts a Windows VS2013 toolchain from various downloadable pieces.

[M-A Ruel, 2014/01/12 16:39:13]

comment -> docstring

[scottmg, 2014/01/13 18:21:25]

Done.

+
+
+import ctypes
+from optparse import OptionParser
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import urllib2
+
+
+g_temp_dirs = []
+
+
+def GetLongPathName(path):
+  """Converts any 8dot3 names in the path to the full name."""
+  buf = ctypes.create_unicode_buffer(260)
+  size = ctypes.windll.kernel32.GetLongPathNameW(unicode(path), buf, 260)
+  if (size > 260):
+    raise SystemExit('Long form of path longer than 260 chars: %s' % path)
+  return buf.value
+
+
+def RunOrDie(command):
+  rc = subprocess.call(command, shell=True)
+  if rc != 0:
+    raise SystemExit('%s failed.' % command)
+
+
+def TempDir():
+  """Generate a temporary directory (for downloading or extracting to) and keep

[M-A Ruel, 2014/01/12 16:39:13]

Generates

[scottmg, 2014/01/13 18:21:25]

Done.

+  track of the directory that's created for cleaning up later."""
+  global g_temp_dirs
+  temp = tempfile.mkdtemp()
+  g_temp_dirs.append(temp)
+  return temp
+
+
+def DeleteAllTempDirs():
+  """Remove all temporary directories created by |TempDir()|."""

[M-A Ruel, 2014/01/12 16:39:13]

Removes

Imperative everywhere

[scottmg, 2014/01/13 18:21:25]

Done.

+  global g_temp_dirs
+  if g_temp_dirs:
+    sys.stdout.write('Cleaning up temporaries...\n')
+  for temp in g_temp_dirs:
+    # shutil.rmtree errors out on read only attributes.
+    RunOrDie('rmdir /s/q "%s"' % temp)
+  g_temp_dirs = []
+
+
+def GetIsoUrl(pro):
+  """Get the .iso path."""

[M-A Ruel, 2014/01/12 16:39:13]

\n\nIf pro is False, downloads the express edition.

It's not a path, it's an url.

[scottmg, 2014/01/13 18:21:25]

Done.

+  prefix = 'http://download.microsoft.com/download/'
+  if pro:
+    return (prefix +
+        'A/F/1/AF128362-A6A8-4DB3-A39A-C348086472CC/VS2013_RTM_PRO_ENU.iso')
+  else:
+    return (prefix +
+        '7/2/E/72E0F986-D247-4289-B9DC-C4FB07374894/VS2013_RTM_DskExp_ENU.iso')
+
+
+def Download(url, local_path):
+  """Download a large-ish binary file and print some status information while
+  doing so."""
+  sys.stdout.write('Downloading %s...\n' % url)
+  req = urllib2.urlopen(url)
+  content_length = int(req.headers.get('Content-Length', 0))
+  bytes_read = 0

[M-A Ruel, 2014/01/12 16:39:13]

0L
otherwise it'll overflow on python 32 bits on 2gb file.

[scottmg, 2014/01/13 18:21:25]

Done.

+  terminator = '\r' if sys.stdout.isatty() else '\n'
+  with open(local_path, 'wb') as file:
+    while True:
+      chunk = req.read(1024 * 1024)
+      if not chunk:
+        break
+      bytes_read += len(chunk)
+      file.write(chunk)
+      sys.stdout.write('... %d/%d%s' % (bytes_read, content_length, terminator))
+      sys.stdout.flush()
+  sys.stdout.write('\n')
+  if content_length and content_length != bytes_read:
+    raise SystemExit('Got incorrect number of bytes downloading %s' % url)
+
+
+def ExtractIso(iso_path):
+  """Use 7zip to extract the contents of the given .iso (or self-extracting
+  .exe)."""
+  target_path = TempDir()
+  sys.stdout.write('Extracting %s...\n' % iso_path)
+  sys.stdout.flush()
+  # TODO(scottmg): Do this (and exe) manually with python code.
+  # Note that at the beginning of main() we set the working directory to 7z's
+  # location.
+  RunOrDie('7z x "%s" -y "-o%s" >nul' % (iso_path, target_path))
+  return target_path
+
+
+ExtractExe = ExtractIso

[M-A Ruel, 2014/01/12 16:39:13]

Why?

[scottmg, 2014/01/13 18:21:25]

Oops, removed. It was necessary in the 2010/2012 version of the script.

+
+
+def ExtractMsi(msi_path):
+  """Use msiexec to extract the contents of the given .msi file."""
+  sys.stdout.write('Extracting %s...\n' % msi_path)
+  target_path = TempDir()
+  RunOrDie('msiexec /a "%s" /qn TARGETDIR="%s"' % (msi_path, target_path))
+  return target_path
+
+
+def DownloadMainIso(url):
+  temp_dir = TempDir()
+  target_path = os.path.join(temp_dir, os.path.basename(url))
+  Download(url, target_path)
+  return target_path
+
+
+def GetSourceImage(local_dir, pro):
+  url = GetIsoUrl(pro)
+  if local_dir:
+    return os.path.join(local_dir, os.path.basename(url))
+  else:
+    return DownloadMainIso(url)
+
+
+def ExtractMsiList(iso_dir, packages):
+  results = []
+  for (package, skippable) in packages:

[M-A Ruel, 2014/01/12 16:39:13]

I'd prefer 'required' to 'skippable'. Also a docstring to quickly explain the
'package' format.

[scottmg, 2014/01/13 18:21:25]

Done.

+    path_to_package = os.path.join(iso_dir, 'packages', package)
+    if not os.path.exists(path_to_package) and skippable:
+      sys.stdout.write('Pro-only %s skipped.\n' % package)
+      continue
+    results.append(ExtractMsi(path_to_package))
+  return results
+
+
+def ExtractComponents(image):
+  extracted_iso = ExtractIso(image)

[M-A Ruel, 2014/01/12 16:39:13]

Could you just extract the files needed? It'd save a lot of temporary space.

[scottmg, 2014/01/13 18:21:25]

I tried this thinking it sounded like a good idea (see ps3), but the .msi aren't
self-contained, they refer to other cabs, etc. so I don't know the full list.
So, unfortunately no, not in any sensible way.

+  results = ExtractMsiList(extracted_iso, [

[M-A Ruel, 2014/01/12 16:39:13]

Create the list as a named variable, then call the function. It'll be more
readable.

[scottmg, 2014/01/13 18:21:25]

Done.

+      (r'vcRuntimeAdditional_amd64\vc_runtimeAdditional_x64.msi', False),
+      (r'vcRuntimeAdditional_x86\vc_runtimeAdditional_x86.msi', False),
+      (r'vcRuntimeDebug_amd64\vc_runtimeDebug_x64.msi', False),
+      (r'vcRuntimeDebug_x86\vc_runtimeDebug_x86.msi', False),
+      (r'vcRuntimeMinimum_amd64\vc_runtimeMinimum_x64.msi', False),
+      (r'vcRuntimeMinimum_x86\vc_runtimeMinimum_x86.msi', False),
+      (r'vc_compilerCore86\vc_compilerCore86.msi', False),
+      (r'vc_compilerCore86res\vc_compilerCore86res.msi', False),
+      (r'vc_compilerx64nat\vc_compilerx64nat.msi', True),
+      (r'vc_compilerx64natres\vc_compilerx64natres.msi', True),
+      (r'vc_compilerx64x86\vc_compilerx64x86.msi', True),
+      (r'vc_compilerx64x86res\vc_compilerx64x86res.msi', True),
+      (r'vc_librarycore86\vc_librarycore86.msi', False),
+      (r'vc_libraryDesktop\x64\vc_LibraryDesktopX64.msi', False),
+      (r'vc_libraryDesktop\x86\vc_LibraryDesktopX86.msi', False),
+      (r'vc_libraryextended\vc_libraryextended.msi', True),
+      (r'Windows_SDK\Windows Software Development Kit-x86_en-us.msi', False),
+      ('Windows_SDK\\'
+       r'Windows Software Development Kit for Metro style Apps-x86_en-us.msi',
+          False),
+    ])
+  return results
+
+
+def CopyToFinalLocation(extracted_dirs, target_dir):
+  sys.stdout.write('Copying to final location...\n')
+  mappings = {
+      'Program Files\\Microsoft Visual Studio 12.0\\': '.\\',

[M-A Ruel, 2014/01/12 16:39:13]

Sort keys

[scottmg, 2014/01/13 18:21:25]

Done.

+      'Windows Kits\\8.0\\': 'win8sdk\\',
+      'System64\\': 'sys64\\',
+      'System\\': 'sys32\\',
+  }
+  matches = []
+  for extracted_dir in extracted_dirs:
+    for root, dirnames, filenames in os.walk(extracted_dir):
+      for filename in filenames:
+        matches.append((extracted_dir, os.path.join(root, filename)))
+
+  copies = []
+  for prefix, full_path in matches:
+    partial_path = full_path[len(prefix) + 1:]  # +1 for trailing \

[M-A Ruel, 2014/01/12 16:39:13]

comment on line before. Note that the trailing \ may have bad side effects, use
'\\' just in case even if it is in a comment. (At worst it only confuses the
text editor or the code reviewer).

[scottmg, 2014/01/13 18:21:25]

How about a trailing '.'? Done.

+    #print 'partial_path', partial_path

[M-A Ruel, 2014/01/12 16:39:13]

Remove these, or use logging.info()
A --verbose flag would be useful.

[scottmg, 2014/01/13 18:21:25]

Done.

+    for map_from, map_to in mappings.iteritems():
+      #print 'map_from:', map_from, ', map_to:', map_to
+      if partial_path.startswith(map_from):
+        target_path = os.path.join(map_to, partial_path[len(map_from):])
+        copies.append((full_path, os.path.join(target_dir, target_path)))
+
+  for full_source, full_target in copies:
+    target_dir = os.path.dirname(full_target)
+    if not os.path.isdir(target_dir):
+      os.makedirs(target_dir)
+    shutil.copy2(full_source, full_target)
+
+
+def GenerateSetEnvCmd(target_dir, pro):
+  """Generate a batch file that gyp expects to exist to set up the compiler
+  environment. This is normally generated by a full install of the SDK, but we

[M-A Ruel, 2014/01/12 16:39:13]

empty line between 1 liner summary and description.

[scottmg, 2014/01/13 18:21:25]

Done.

+  do it here manually since we do not do a full install."""
+  with open(os.path.join(
+        target_dir, r'win8sdk\bin\SetEnv.cmd'), 'w') as file:

[M-A Ruel, 2014/01/12 16:39:13]

nit: Personally, I'd use a single call per functional block. And I'd use 'f'
instead of 'file', which is a python intrinsic.

f.write(
    '@echo off\n'
    ':: Generated by win_toolchain\\toolchain2013.py.\n'
    '\n'
    'set PATH=%~dp0..\\..\\Common7\\IDE;%PATH%\n'
    'set INCLUDE=%~dp0..\\..\\win8sdk\\Include\\um;'
      '%~dp0..\\..\\win8sdk\\Include\\shared;'
      '%~dp0..\\..\\VC\\include;'
      '%~dp0..\\..\\VC\\atlmfc\\include\n'
    'if "%1"=="/x64" goto x64\n')

[scottmg, 2014/01/13 18:21:25]

Done.

+    file.write('@echo off\n')
+    file.write(':: Generated by win_toolchain\\toolchain2013.py.\n')
+    # Common to x86 and x64
+    file.write('set PATH=%~dp0..\\..\\Common7\\IDE;%PATH%\n')
+    file.write('set INCLUDE=%~dp0..\\..\\win8sdk\\Include\\um;'
+               '%~dp0..\\..\\win8sdk\\Include\\shared;'
+               '%~dp0..\\..\\VC\\include;'
+               '%~dp0..\\..\\VC\\atlmfc\\include\n')
+    file.write('if "%1"=="/x64" goto x64\n')
+
+    # x86. If we're Pro, then use the amd64_x86 cross (we don't support x86

[M-A Ruel, 2014/01/12 16:39:13]

Would it be possible to write the file unconditionally except for one flag so
that it is always the same file that is written? It'd be more maintainable IMHO.

[scottmg, 2014/01/13 18:21:25]

Do you mean having a flag passed to the batch file instead of a flag at
generation time? No, it's run by other tools that won't know which flag to pass.

[M-A Ruel, 2014/01/13 18:29:05]

No I meant always write the same file except for a 
set PRO=1
or 0 at the top of the file.

But I don't really mind.

+    # host at all).
+    if pro:
+      file.write('set PATH=%~dp0..\\..\\win8sdk\\bin\\x86;'
+                '%~dp0..\\..\\VC\\bin\\amd64_x86;'
+                '%~dp0..\\..\\VC\\bin\\amd64;'  # Needed for mspdb120.dll.
+                '%PATH%\n')
+    else:
+      file.write('set PATH=%~dp0..\\..\\win8sdk\\bin\\x86;'
+                '%~dp0..\\..\\VC\\bin;%PATH%\n')
+    file.write('set LIB=%~dp0..\\..\\VC\\lib;'
+               '%~dp0..\\..\\win8sdk\\Lib\\win8\\um\\x86;'
+               '%~dp0..\\..\\VC\\atlmfc\\lib\n')
+    file.write('goto done\n')

[M-A Ruel, 2014/01/12 16:39:13]

FYI, you can use 'goto :EOF\n'

[scottmg, 2014/01/13 18:21:25]

Done.

+
+    # Express does not include a native 64 bit compiler, so we have to use
+    # the x86->x64 cross.
+    if not pro:
+      # x86->x64 cross.
+      file.write(':x64\n')
+      file.write('set PATH=%~dp0..\\..\\win8sdk\\bin\\x64;'
+                 '%~dp0..\\..\\VC\\bin\\x86_amd64;'
+                 '%PATH%\n')
+    else:
+      # x64 native.
+      file.write(':x64\n')
+      file.write('set PATH=%~dp0..\\..\\win8sdk\\bin\\x64;'
+                 '%~dp0..\\..\\VC\\bin\\amd64;'
+                 '%PATH%\n')
+    file.write('set LIB=%~dp0..\\..\\VC\\lib\\amd64;'
+               '%~dp0..\\..\\win8sdk\\Lib\\win8\\um\\x64;'
+               '%~dp0..\\..\\VC\\atlmfc\\lib\\amd64\n')
+    file.write(':done\n')
+
+
+def main():
+  parser = OptionParser()

[M-A Ruel, 2014/01/12 16:39:13]

parser = OptionParser(description=sys.moduels[__name__].__doc__)

[scottmg, 2014/01/13 18:21:25]

Done.

+  parser.add_option('--targetdir', metavar='DIR',
+                    help='put toolchain into DIR',
+                    default=os.path.abspath('win_toolchain_2013'))

[M-A Ruel, 2014/01/12 16:39:13]

you probably want os.path.join(FILE_PATH, 'win_toolcain_2013') and not rely on
cwd.

[scottmg, 2014/01/13 18:21:25]

Done.

+  parser.add_option('--noclean', action='store_false', dest='clean',
+                    help='do not remove temp files',
+                    default=True)
+  parser.add_option('--local', metavar='DIR',
+                    help='use downloaded files from DIR')
+  parser.add_option('--express', metavar='EXPRESS',

[M-A Ruel, 2014/01/12 16:39:13]

do not use metavar on a store_true flag, it's confusing.

[scottmg, 2014/01/13 18:21:25]

Done.

+                    help='use VS Express instead of Pro', action='store_true')
+  options, args = parser.parse_args()
+  try:
+    target_dir = os.path.abspath(options.targetdir)
+    if os.path.exists(target_dir):
+      sys.stderr.write('%s already exists. Please [re]move it or use '

[M-A Ruel, 2014/01/12 16:39:13]

parser.error() would work fine.

[scottmg, 2014/01/13 18:21:25]

Done.

+                       '--targetdir to select a different target.\n' %
+                       target_dir)
+      return 1
+    pro = not options.express

[M-A Ruel, 2014/01/12 16:39:13]

Why use options.clean but then use pro as a standalone variable? Use one style
or the other.

[scottmg, 2014/01/13 18:21:25]

Done.

+    # Set the working directory to 7z subdirectory. 7-zip doesn't find its
+    # codec dll very well, so this is the simplest way to make sure it runs
+    # correctly, as we don't otherwise care about working directory.
+    os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)), '7z'))
+    image = GetSourceImage(options.local, pro)
+    extracted = ExtractComponents(image)
+    CopyToFinalLocation(extracted, target_dir)
+
+    GenerateSetEnvCmd(target_dir, pro)
+  finally:
+    if options.clean:
+      DeleteAllTempDirs()
+
+
+if __name__ == '__main__':
+  main()

[M-A Ruel, 2014/01/12 16:39:13]

sys.exit(main())
otherwise you are ditching the return code.

[scottmg, 2014/01/13 18:21:25]

Done.