Updates to VS packaging script for VS 2015

win_toolchain/package_from_installed.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """
 From a system-installed copy of the toolchain, packages all the required bits
 into a .zip file.
 
 It assumes default install locations for tools, in particular:
 - C:\Program Files (x86)\Microsoft Visual Studio 12.0\...
@@ skipping 13 matching lines @@
 import os
 import shutil
 import sys
 import tempfile
 import zipfile
 
 import get_toolchain_if_necessary
 
 
 VS_VERSION = None
+WIN_VERSION = None
 
 
 def BuildFileList():
   result = []
 
   # Subset of VS corresponding roughly to VC.
   paths = [
       'DIA SDK/bin',
       'DIA SDK/idl',
       'DIA SDK/include',
@@ skipping 61 matching lines @@
           else:
             result.append((final_from, dest))
 
   # Just copy the whole SDK.
   sdk_path = r'C:\Program Files (x86)\Windows Kits\10'
   for root, _, files in os.walk(sdk_path):
     for f in files:
       combined = os.path.normpath(os.path.join(root, f))
       # Some of the files in this directory are exceedingly long (and exceed
       #_MAX_PATH for any moderately long root), so exclude them. We don't need
-      # them anyway.
+      # them anyway. Exclude the Windows Performance Toolkit just to save space.
       tail = combined[len(sdk_path) + 1:]
-      if tail.startswith('References\\'):
+      if tail.startswith('References\\') or \

[scottmg, 2015/12/07 20:58:04]

Put () around this if instead of using \.

[brucedawson, 2015/12/08 01:13:49]

Done.

+          tail.startswith('Windows Performance Toolkit\\'):
         continue
+      if VS_VERSION == '2015':
+        # There may be many Include\Lib\Source directories for many different
+        # versions of Windows and packaging them all wastes ~450 MB
+        # (uncompressed) per version and wastes time. Only copy the specified
+        # version.
+        if tail.startswith('Include\\') or tail.startswith('Lib\\') or \
+            tail.startswith('Source\\'):

[scottmg, 2015/12/07 20:58:04]

Same here.

[brucedawson, 2015/12/08 01:13:49]

Done.

+          if tail.count(WIN_VERSION) == 0:
+            continue
       to = os.path.join('win_sdk', tail)
       result.append((combined, to))
 
   if VS_VERSION == '2015':
-    for ucrt_path in (
-        (r'C:\Program Files (x86)\Windows Kits\10\Include', 'Include'),
-        (r'C:\Program Files (x86)\Windows Kits\10\Lib', 'Lib'),
-        (r'C:\Program Files (x86)\Windows Kits\10\Source', 'Source')):
-      src, target = ucrt_path
-      for root, _, files in os.walk(src):
-        for f in files:
-          combined = os.path.normpath(os.path.join(root, f))
-          to = os.path.join('ucrt', target, combined[len(src) + 1:])
-          result.append((combined, to))
 
     system_crt_files = [

[scottmg, 2015/12/07 20:58:04]

Have you tried the package on a non-redist installed machine? Perhaps changes
needed here..

[brucedawson, 2015/12/08 01:13:49]

I have not yet tried using the package on a machine that doesn't have the redist
installed.

         'api-ms-win-core-file-l1-2-0.dll',
         'api-ms-win-core-file-l2-1-0.dll',
         'api-ms-win-core-localization-l1-2-0.dll',
         'api-ms-win-core-processthreads-l1-1-1.dll',
         'api-ms-win-core-synch-l1-2-0.dll',
         'api-ms-win-core-timezone-l1-1-0.dll',
         'api-ms-win-core-xstate-l2-1-0.dll',
         'api-ms-win-crt-conio-l1-1-0.dll',
         'api-ms-win-crt-convert-l1-1-0.dll',
         'api-ms-win-crt-environment-l1-1-0.dll',
         'api-ms-win-crt-filesystem-l1-1-0.dll',
         'api-ms-win-crt-heap-l1-1-0.dll',
         'api-ms-win-crt-locale-l1-1-0.dll',
         'api-ms-win-crt-math-l1-1-0.dll',
         'api-ms-win-crt-multibyte-l1-1-0.dll',
         'api-ms-win-crt-private-l1-1-0.dll',
         'api-ms-win-crt-process-l1-1-0.dll',
         'api-ms-win-crt-runtime-l1-1-0.dll',
         'api-ms-win-crt-stdio-l1-1-0.dll',
         'api-ms-win-crt-string-l1-1-0.dll',
         'api-ms-win-crt-time-l1-1-0.dll',
         'api-ms-win-crt-utility-l1-1-0.dll',
         'api-ms-win-eventing-provider-l1-1-0.dll',
         'ucrtbase.dll',
         'ucrtbased.dll',
     ]
+    x64_path = r'C:\Windows\Sysnative'

[scottmg, 2015/12/07 20:58:04]

Maybe rather that exists(), something like:

x64_path = 'System32' if platform.machine.endswith('64') else 'Sysnative'

+    if not os.path.exists(x64_path):
+      # When running 64-bit python the x64 DLLs will be found here
+      x64_path = r'C:\Windows\System32'
     for system_crt_file in system_crt_files:
         result.append((os.path.join(r'C:\Windows\SysWOW64', system_crt_file),
                        os.path.join('sys32', system_crt_file)))
-        result.append((os.path.join(r'C:\Windows\Sysnative', system_crt_file),
+        result.append((os.path.join(x64_path, system_crt_file),
                        os.path.join('sys64', system_crt_file)))
 
-  # Generically drop all arm stuff that we don't need.
+  # Generically drop all arm stuff that we don't need, and
+  # drop .msi files because we don't need installers.
   return [(f, t) for f, t in result if 'arm\\' not in f.lower() and
-                                       'arm64\\' not in f.lower()]
+                                       'arm64\\' not in f.lower() and
+                                       not f.lower().endswith('.msi')]
 
 
 def GenerateSetEnvCmd(target_dir):
   """Generate a batch file that gyp expects to exist to set up the compiler
   environment.
 
   This is normally generated by a full install of the SDK, but we
   do it here manually since we do not do a full install."""
   with open(os.path.join(
         target_dir, r'win_sdk\bin\SetEnv.cmd'), 'w') as f:
     f.write('@echo off\n'
             ':: Generated by win_toolchain\\package_from_installed.py.\n'
             # Common to x86 and x64
             'set PATH=%~dp0..\\..\\Common7\\IDE;%PATH%\n'
-            'set INCLUDE=%~dp0..\\..\\win_sdk\\Include\\10.0.10240.0\\um;'
-               '%~dp0..\\..\\win_sdk\\Include\\10.0.10240.0\\shared;'
-               '%~dp0..\\..\\win_sdk\\Include\\10.0.10240.0\\winrt;'
+            'set INCLUDE=%~dp0..\\..\\win_sdk\\Include\\WINVERSION\\um;'
+               '%~dp0..\\..\\win_sdk\\Include\\WINVERSION\\shared;'
+               '%~dp0..\\..\\win_sdk\\Include\\WINVERSION\\winrt;'
+               '%~dp0..\\..\\win_sdk\\Include\\WINVERSION\\ucrt;' # VS 2015

[scottmg, 2015/12/07 20:58:04]

I guess we're going to get a different package for 2013 now. That's fine since
we should be done with it soon, but it just means that until we switch to 2015
the package we're using isn't regenerate-able with this script. So in theory we
should update the 2013 package first too. But if we're going to try to flip the
switch to 2015 soon, then we can probably get away without doing it.

[brucedawson, 2015/12/08 01:13:49]

It's true that the exact package we're using can't be regenerated, but the only
difference will be one extra entry in the include and lib paths.

Even that could be avoided if desired, but it does make the script messier so I
was inclined to not bother.

                '%~dp0..\\..\\VC\\include;'
                '%~dp0..\\..\\VC\\atlmfc\\include\n'
-            'if "%1"=="/x64" goto x64\n')
+            'if "%1"=="/x64" goto x64\n'.replace("WINVERSION", WIN_VERSION))
 
     # x86. Always use amd64_x86 cross, not x86 on x86.
     f.write('set PATH=%~dp0..\\..\\win_sdk\\bin\\x86;'
               '%~dp0..\\..\\VC\\bin\\amd64_x86;'
               '%~dp0..\\..\\VC\\bin\\amd64;'  # Needed for mspdb1x0.dll.
               '%PATH%\n')
     f.write('set LIB=%~dp0..\\..\\VC\\lib;'
-               '%~dp0..\\..\\win_sdk\\Lib\\10.0.10240.0\\um\\x86;'
+               '%~dp0..\\..\\win_sdk\\Lib\\WINVERSION\\um\\x86;'
+               '%~dp0..\\..\\win_sdk\\Lib\\WINVERSION\\ucrt\\x86;' # VS 2015
                '%~dp0..\\..\\VC\\atlmfc\\lib\n'
-            'goto :EOF\n')
+            'goto :EOF\n'.replace("WINVERSION", WIN_VERSION))
 
     # x64.
     f.write(':x64\n'
             'set PATH=%~dp0..\\..\\win_sdk\\bin\\x64;'
                 '%~dp0..\\..\\VC\\bin\\amd64;'
                 '%PATH%\n')
     f.write('set LIB=%~dp0..\\..\\VC\\lib\\amd64;'
-               '%~dp0..\\..\\win_sdk\\Lib\\10.0.10240.0\\um\\x64;'
-               '%~dp0..\\..\\VC\\atlmfc\\lib\\amd64\n')
+               '%~dp0..\\..\\win_sdk\\Lib\\WINVERSION\\um\\x64;'
+               '%~dp0..\\..\\win_sdk\\Lib\\WINVERSION\\ucrt\\x64;' # VS 2015
+               '%~dp0..\\..\\VC\\atlmfc\\lib\\amd64\n'
+               .replace("WINVERSION", WIN_VERSION))
 
 
 def AddEnvSetup(files):
   """We need to generate this file in the same way that the "from pieces"
   script does, so pull that in here."""
   tempdir = tempfile.mkdtemp()
   os.makedirs(os.path.join(tempdir, 'win_sdk', 'bin'))
   GenerateSetEnvCmd(tempdir)
   files.append((os.path.join(tempdir, 'win_sdk', 'bin', 'SetEnv.cmd'),
                 'win_sdk\\bin\\SetEnv.cmd'))
@@ skipping 20 matching lines @@
   print 'Hashing...'
   sha1 = get_toolchain_if_necessary.CalculateHash(rel_dir)
   os.chdir(old_dir)
   shutil.rmtree(tempdir)
   final_name = sha1 + '.zip'
   os.rename(output, final_name)
   print 'Renamed %s to %s.' % (output, final_name)
 
 
 def main():
-  if len(sys.argv) != 2 or sys.argv[1] not in ('2013', '2015'):
-    print 'Usage: package_from_installed.py 2013|2015'
+  if len(sys.argv) < 3 or sys.argv[1] not in ('2013', '2015'):

[scottmg, 2015/12/07 20:58:04]

This is a bit much once it's more than 1. Can you switch it to using optparse?

+    print 'Usage: package_from_installed.py 2013|2015 winversion dryrun'
+    print 'winversion should be 10.0.10240.0 or similar'

[scottmg, 2015/12/07 20:58:04]

Can we not default to 10586? (w/ optparse we can have an actual default rather
than just in the help text).

(Or are you trying to maintain previous behaviour for 2013?)

+    print 'Specify dryrun to just scan for all files without packaging them.'
     return 1
 
   global VS_VERSION
   VS_VERSION = sys.argv[1]
+  global WIN_VERSION
+  WIN_VERSION = sys.argv[2]
+
+  dry_run = False
+  if len(sys.argv) > 3 and sys.argv[3].lower() == 'dryrun':
+    dry_run = True
 
   print 'Building file list...'
   files = BuildFileList()
 
   AddEnvSetup(files)
 
   if False:
     for f in files:
       print f[0], '->', f[1]
     return 0
 
   output = 'out.zip'
   if os.path.exists(output):
     os.unlink(output)
   count = 0
+  version_match_count = 0
+  total_size = 0
+  missing_files = False
   with zipfile.ZipFile(output, 'w', zipfile.ZIP_DEFLATED, True) as zf:
     for disk_name, archive_name in files:
       sys.stdout.write('\r%d/%d ...%s' % (count, len(files), disk_name[-40:]))
       sys.stdout.flush()
       count += 1
-      zf.write(disk_name, archive_name)
+      if disk_name.count(WIN_VERSION) > 0:
+        version_match_count += 1
+      if os.path.exists(disk_name):
+        if dry_run:
+          total_size += os.path.getsize(disk_name)
+        else:
+          zf.write(disk_name, archive_name)
+      else:
+        missing_files = True
+        sys.stdout.write('\r%s does not exist.\n\n' % disk_name)
+        sys.stdout.flush()
+  if dry_run:
+    sys.stdout.write('\r%1.3f GB of data in %d files, %d files for %s.%s\n' %
+        (total_size / 1e9, count, version_match_count, WIN_VERSION, ' '*50))
+    sys.stdout.flush()

[scottmg, 2015/12/07 20:58:04]

No need to .flush() here, that's just because of the \r in the prints above.

[brucedawson, 2015/12/08 01:13:49]

Done.

+    return 0
+  if missing_files:
+    raise "One or more files were missing - aborting"
+  if version_match_count == 0:
+    raise "No files found that match the specified winversion"
   sys.stdout.write('\rWrote to %s.%s\n' % (output, ' '*50))
   sys.stdout.flush()
 
   RenameToSha1(output)
 
   return 0
 
 
 if __name__ == '__main__':
   sys.exit(main())