Updates to VS packaging script for VS 2015

win_toolchain/package_from_installed.py

Index: win_toolchain/package_from_installed.py
diff --git a/win_toolchain/package_from_installed.py b/win_toolchain/package_from_installed.py
index 2762201284115e82e17d5d8457fcb508df7e2b90..869a70c8b0042bfa72780ba81b255dfd4cca92e9 100644
--- a/win_toolchain/package_from_installed.py
+++ b/win_toolchain/package_from_installed.py
@@ -31,6 +31,7 @@ import get_toolchain_if_necessary
 
 
 VS_VERSION = None
+WIN_VERSION = None
 
 
 def BuildFileList():
@@ -112,24 +113,24 @@ def BuildFileList():
       combined = os.path.normpath(os.path.join(root, f))
       # Some of the files in this directory are exceedingly long (and exceed
       #_MAX_PATH for any moderately long root), so exclude them. We don't need
-      # them anyway.
+      # them anyway. Exclude the Windows Performance Toolkit just to save space.
       tail = combined[len(sdk_path) + 1:]
-      if tail.startswith('References\\'):
+      if tail.startswith('References\\') or \

[scottmg, 2015/12/07 20:58:04]

Put () around this if instead of using \.

[brucedawson, 2015/12/08 01:13:49]

Done.

+          tail.startswith('Windows Performance Toolkit\\'):
         continue
+      if VS_VERSION == '2015':
+        # There may be many Include\Lib\Source directories for many different
+        # versions of Windows and packaging them all wastes ~450 MB
+        # (uncompressed) per version and wastes time. Only copy the specified
+        # version.
+        if tail.startswith('Include\\') or tail.startswith('Lib\\') or \
+            tail.startswith('Source\\'):

[scottmg, 2015/12/07 20:58:04]

Same here.

[brucedawson, 2015/12/08 01:13:49]

Done.

+          if tail.count(WIN_VERSION) == 0:
+            continue
       to = os.path.join('win_sdk', tail)
       result.append((combined, to))
 
   if VS_VERSION == '2015':
-    for ucrt_path in (
-        (r'C:\Program Files (x86)\Windows Kits\10\Include', 'Include'),
-        (r'C:\Program Files (x86)\Windows Kits\10\Lib', 'Lib'),
-        (r'C:\Program Files (x86)\Windows Kits\10\Source', 'Source')):
-      src, target = ucrt_path
-      for root, _, files in os.walk(src):
-        for f in files:
-          combined = os.path.normpath(os.path.join(root, f))
-          to = os.path.join('ucrt', target, combined[len(src) + 1:])
-          result.append((combined, to))
 
     system_crt_files = [

[scottmg, 2015/12/07 20:58:04]

Have you tried the package on a non-redist installed machine? Perhaps changes
needed here..

[brucedawson, 2015/12/08 01:13:49]

I have not yet tried using the package on a machine that doesn't have the redist
installed.

         'api-ms-win-core-file-l1-2-0.dll',
@@ -158,15 +159,21 @@ def BuildFileList():
         'ucrtbase.dll',
         'ucrtbased.dll',
     ]
+    x64_path = r'C:\Windows\Sysnative'

[scottmg, 2015/12/07 20:58:04]

Maybe rather that exists(), something like:

x64_path = 'System32' if platform.machine.endswith('64') else 'Sysnative'

+    if not os.path.exists(x64_path):
+      # When running 64-bit python the x64 DLLs will be found here
+      x64_path = r'C:\Windows\System32'
     for system_crt_file in system_crt_files:
         result.append((os.path.join(r'C:\Windows\SysWOW64', system_crt_file),
                        os.path.join('sys32', system_crt_file)))
-        result.append((os.path.join(r'C:\Windows\Sysnative', system_crt_file),
+        result.append((os.path.join(x64_path, system_crt_file),
                        os.path.join('sys64', system_crt_file)))
 
-  # Generically drop all arm stuff that we don't need.
+  # Generically drop all arm stuff that we don't need, and
+  # drop .msi files because we don't need installers.
   return [(f, t) for f, t in result if 'arm\\' not in f.lower() and
-                                       'arm64\\' not in f.lower()]
+                                       'arm64\\' not in f.lower() and
+                                       not f.lower().endswith('.msi')]
 
 
 def GenerateSetEnvCmd(target_dir):
@@ -181,12 +188,13 @@ def GenerateSetEnvCmd(target_dir):
             ':: Generated by win_toolchain\\package_from_installed.py.\n'
             # Common to x86 and x64
             'set PATH=%~dp0..\\..\\Common7\\IDE;%PATH%\n'
-            'set INCLUDE=%~dp0..\\..\\win_sdk\\Include\\10.0.10240.0\\um;'
-               '%~dp0..\\..\\win_sdk\\Include\\10.0.10240.0\\shared;'
-               '%~dp0..\\..\\win_sdk\\Include\\10.0.10240.0\\winrt;'
+            'set INCLUDE=%~dp0..\\..\\win_sdk\\Include\\WINVERSION\\um;'
+               '%~dp0..\\..\\win_sdk\\Include\\WINVERSION\\shared;'
+               '%~dp0..\\..\\win_sdk\\Include\\WINVERSION\\winrt;'
+               '%~dp0..\\..\\win_sdk\\Include\\WINVERSION\\ucrt;' # VS 2015

[scottmg, 2015/12/07 20:58:04]

I guess we're going to get a different package for 2013 now. That's fine since
we should be done with it soon, but it just means that until we switch to 2015
the package we're using isn't regenerate-able with this script. So in theory we
should update the 2013 package first too. But if we're going to try to flip the
switch to 2015 soon, then we can probably get away without doing it.

[brucedawson, 2015/12/08 01:13:49]

It's true that the exact package we're using can't be regenerated, but the only
difference will be one extra entry in the include and lib paths.

Even that could be avoided if desired, but it does make the script messier so I
was inclined to not bother.

                '%~dp0..\\..\\VC\\include;'
                '%~dp0..\\..\\VC\\atlmfc\\include\n'
-            'if "%1"=="/x64" goto x64\n')
+            'if "%1"=="/x64" goto x64\n'.replace("WINVERSION", WIN_VERSION))
 
     # x86. Always use amd64_x86 cross, not x86 on x86.
     f.write('set PATH=%~dp0..\\..\\win_sdk\\bin\\x86;'
@@ -194,9 +202,10 @@ def GenerateSetEnvCmd(target_dir):
               '%~dp0..\\..\\VC\\bin\\amd64;'  # Needed for mspdb1x0.dll.
               '%PATH%\n')
     f.write('set LIB=%~dp0..\\..\\VC\\lib;'
-               '%~dp0..\\..\\win_sdk\\Lib\\10.0.10240.0\\um\\x86;'
+               '%~dp0..\\..\\win_sdk\\Lib\\WINVERSION\\um\\x86;'
+               '%~dp0..\\..\\win_sdk\\Lib\\WINVERSION\\ucrt\\x86;' # VS 2015
                '%~dp0..\\..\\VC\\atlmfc\\lib\n'
-            'goto :EOF\n')
+            'goto :EOF\n'.replace("WINVERSION", WIN_VERSION))
 
     # x64.
     f.write(':x64\n'
@@ -204,8 +213,10 @@ def GenerateSetEnvCmd(target_dir):
                 '%~dp0..\\..\\VC\\bin\\amd64;'
                 '%PATH%\n')
     f.write('set LIB=%~dp0..\\..\\VC\\lib\\amd64;'
-               '%~dp0..\\..\\win_sdk\\Lib\\10.0.10240.0\\um\\x64;'
-               '%~dp0..\\..\\VC\\atlmfc\\lib\\amd64\n')
+               '%~dp0..\\..\\win_sdk\\Lib\\WINVERSION\\um\\x64;'
+               '%~dp0..\\..\\win_sdk\\Lib\\WINVERSION\\ucrt\\x64;' # VS 2015
+               '%~dp0..\\..\\VC\\atlmfc\\lib\\amd64\n'
+               .replace("WINVERSION", WIN_VERSION))
 
 
 def AddEnvSetup(files):
@@ -246,12 +257,20 @@ def RenameToSha1(output):
 
 
 def main():
-  if len(sys.argv) != 2 or sys.argv[1] not in ('2013', '2015'):
-    print 'Usage: package_from_installed.py 2013|2015'
+  if len(sys.argv) < 3 or sys.argv[1] not in ('2013', '2015'):

[scottmg, 2015/12/07 20:58:04]

This is a bit much once it's more than 1. Can you switch it to using optparse?

+    print 'Usage: package_from_installed.py 2013|2015 winversion dryrun'
+    print 'winversion should be 10.0.10240.0 or similar'

[scottmg, 2015/12/07 20:58:04]

Can we not default to 10586? (w/ optparse we can have an actual default rather
than just in the help text).

(Or are you trying to maintain previous behaviour for 2013?)

+    print 'Specify dryrun to just scan for all files without packaging them.'
     return 1
 
   global VS_VERSION
   VS_VERSION = sys.argv[1]
+  global WIN_VERSION
+  WIN_VERSION = sys.argv[2]
+
+  dry_run = False
+  if len(sys.argv) > 3 and sys.argv[3].lower() == 'dryrun':
+    dry_run = True
 
   print 'Building file list...'
   files = BuildFileList()
@@ -267,12 +286,34 @@ def main():
   if os.path.exists(output):
     os.unlink(output)
   count = 0
+  version_match_count = 0
+  total_size = 0
+  missing_files = False
   with zipfile.ZipFile(output, 'w', zipfile.ZIP_DEFLATED, True) as zf:
     for disk_name, archive_name in files:
       sys.stdout.write('\r%d/%d ...%s' % (count, len(files), disk_name[-40:]))
       sys.stdout.flush()
       count += 1
-      zf.write(disk_name, archive_name)
+      if disk_name.count(WIN_VERSION) > 0:
+        version_match_count += 1
+      if os.path.exists(disk_name):
+        if dry_run:
+          total_size += os.path.getsize(disk_name)
+        else:
+          zf.write(disk_name, archive_name)
+      else:
+        missing_files = True
+        sys.stdout.write('\r%s does not exist.\n\n' % disk_name)
+        sys.stdout.flush()
+  if dry_run:
+    sys.stdout.write('\r%1.3f GB of data in %d files, %d files for %s.%s\n' %
+        (total_size / 1e9, count, version_match_count, WIN_VERSION, ' '*50))
+    sys.stdout.flush()

[scottmg, 2015/12/07 20:58:04]

No need to .flush() here, that's just because of the \r in the prints above.

[brucedawson, 2015/12/08 01:13:49]

Done.

+    return 0
+  if missing_files:
+    raise "One or more files were missing - aborting"
+  if version_match_count == 0:
+    raise "No files found that match the specified winversion"
   sys.stdout.write('\rWrote to %s.%s\n' % (output, ' '*50))
   sys.stdout.flush()