Add the possibility to keep several version of the VS toolchain.

win_toolchain/get_toolchain_if_necessary.py

 #!/usr/bin/env python
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Downloads and unpacks a toolchain for building on Windows. The contents are
 matched by sha1 which will be updated when the toolchain is updated.
 
 Having a toolchain script in depot_tools means that it's not versioned
 directly with the source code. That is, if the toolchain is upgraded, but
@@ skipping 60 matching lines @@
   """Gets a normalized list of files under |root|."""
   assert not os.path.isabs(root)
   assert os.path.normpath(root) == root
   file_list = []
   for base, _, files in os.walk(root):
     paths = [os.path.join(base, f) for f in files]
     file_list.extend(x.lower() for x in paths)
   return sorted(file_list, key=lambda s: s.replace('/', '\\'))
 
 
-def MakeTimestampsFileName(root):
-  return os.path.join(root, '..', '.timestamps')
+def MakeTimestampsFileName(root, sha1):
+  return os.path.join(root, os.pardir, '%s.timestamps' % sha1 if sha1 else '')

[scottmg, 2016/01/28 23:04:11]

Get rid of "if sha1 else ''".

[Sébastien Marchand, 2016/02/01 19:29:29]

Done.

 
 
-def CalculateHash(root):
+def CalculateHash(root, expected_hash):
   """Calculates the sha1 of the paths to all files in the given |root| and the
-  contents of those files, and returns as a hex string."""
-  file_list = GetFileList(root)
+  contents of those files, and returns as a hex string.
 
-  # Check whether we previously saved timestamps in $root/../.timestamps. If
-  # we didn't, or they don't match, then do the full calculation, otherwise
+  |expected_hash| is the expected hash value for this toolchain if it has
+  already been installed.
+  """
+  if expected_hash:
+    full_root_path = os.path.join(root, expected_hash)
+  else:
+    full_root_path = root
+  file_list = GetFileList(full_root_path)
+  # Check whether we previously saved timestamps in $root/../{sha1}.timestamps.
+  # If we didn't, or they don't match, then do the full calculation, otherwise
   # return the saved value.
-  timestamps_file = MakeTimestampsFileName(root)
+  timestamps_file = MakeTimestampsFileName(root, expected_hash)
   timestamps_data = {'files': [], 'sha1': ''}
   if os.path.exists(timestamps_file):
     with open(timestamps_file, 'rb') as f:
       try:
         timestamps_data = json.load(f)
       except ValueError:
         # json couldn't be loaded, empty data will force a re-hash.
         pass
 
   matches = len(file_list) == len(timestamps_data['files'])
   if matches:
     for disk, cached in zip(file_list, timestamps_data['files']):
       if disk != cached[0] or os.stat(disk).st_mtime != cached[1]:
         matches = False
         break
   if matches:
     return timestamps_data['sha1']
 
   digest = hashlib.sha1()
   for path in file_list:
-    digest.update(str(path).replace('/', '\\'))
+    if expected_hash:
+      path_without_hash = (
+          str(path).replace('/', '\\').replace(expected_hash, '').replace(

[scottmg, 2016/01/28 23:04:11]

Can you trim expected_hash only from where you expect it to be at the beginning?

[Sébastien Marchand, 2016/02/01 19:29:29]

Done.

+              '\\\\', '\\'))

[scottmg, 2016/01/28 23:04:11]

Where are the double \ coming from?

[Sébastien Marchand, 2016/02/01 19:29:29]

It was because I was just removing {hash} from {root}\{hash}\foo, so the
resulting path was {root}\\foo, I've fixed this.

+    else:
+      path_without_hash = str(path).replace('/', '\\')
+    digest.update(path_without_hash)
     with open(path, 'rb') as f:
       digest.update(f.read())
   return digest.hexdigest()
 
 
+def CalculateToolchainHashes(root):
+  """Calculate the hash of the different toolchains installed in the |root|
+  directory."""
+  hashes = []
+  dir_list = [
+      d for d in os.listdir(root) if os.path.isdir(os.path.join(root, d))]
+  for d in dir_list:
+    hashes.append(CalculateHash(root, d))
+  return hashes
+
+
 def SaveTimestampsAndHash(root, sha1):
   """Saves timestamps and the final hash to be able to early-out more quickly
   next time."""
-  file_list = GetFileList(root)
+  file_list = GetFileList(os.path.join(root, sha1))
   timestamps_data = {
     'files': [[f, os.stat(f).st_mtime] for f in file_list],
     'sha1': sha1,
   }
-  with open(MakeTimestampsFileName(root), 'wb') as f:
+  with open(MakeTimestampsFileName(root, sha1), 'wb') as f:
     json.dump(timestamps_data, f)
 
 
 def HaveSrcInternalAccess():
   """Checks whether access to src-internal is available."""
   with open(os.devnull, 'w') as nul:
     if subprocess.call(
         ['svn', 'ls', '--non-interactive',
          'svn://svn.chromium.org/chrome-internal/trunk/src-internal/'],
         shell=True, stdin=nul, stdout=nul, stderr=nul) == 0:
@@ skipping 89 matching lines @@
   else:
     temp_dir, local_zip = DownloadUsingGsutil(tree_sha1 + '.zip')
   sys.stdout.write('Extracting %s...\n' % local_zip)
   sys.stdout.flush()
   with zipfile.ZipFile(local_zip, 'r', zipfile.ZIP_DEFLATED, True) as zf:
     zf.extractall(target_dir)
   if temp_dir:
     RmDir(temp_dir)
 
 
+def RemoveToolchain(root, sha1, delay_before_removing):
+  """Remove the |sha1| version of the toolchain from |root|."""
+  toolchain_target_dir = os.path.join(root, sha1)
+  if delay_before_removing:
+    DelayBeforeRemoving(toolchain_target_dir)
+  if sys.platform == 'win32':
+    # These stay resident and will make the rmdir below fail.
+    kill_list = [
+      'mspdbsrv.exe',
+      'vctip.exe', # Compiler and tools experience improvement data uploader.
+    ]
+    for process_name in kill_list:
+      with open(os.devnull, 'wb') as nul:
+        subprocess.call(['taskkill', '/f', '/im', process_name],
+                        stdin=nul, stdout=nul, stderr=nul)
+  if os.path.isdir(toolchain_target_dir):
+    RmDir(toolchain_target_dir)
+
+  timestamp_file = MakeTimestampsFileName(root, sha1)
+  if os.path.exists(timestamp_file):
+    os.remove(timestamp_file)
+
+
+def RemoveUnusedToolchains(root):
+  """Remove the versions of the toolchain that haven't been used recently."""
+  valid_toolchains = []
+  dirs_to_remove = []
+
+  for d in os.listdir(root):
+    full_path = os.path.join(root, d)
+    if os.path.isdir(full_path):
+      if not os.path.exists(MakeTimestampsFileName(root, d)):
+        dirs_to_remove.append(d)
+      else:
+        valid_toolchains.append((os.stat(full_path).st_atime, d))

[scottmg, 2016/01/28 23:04:11]

Hm, won't st_atime update for all of them every time you
CalculateToolchainHashes()?

[Sébastien Marchand, 2016/02/01 19:29:29]

No, reading a file (via read()) doesn't seem to update it's timestamp, here's an
example:

-----------------------------------------------------------
import os
import time

filename = 'delete.me'
f = open(filename, 'w').close()
creation_time = os.path.getatime(filename)
print 'Creation time: %f' % creation_time
print 'Sleeping for one second.'
print 'Reading the file.'
time.sleep(1)
f = open(filename, 'r')
f.read()
f.close()
last_access_time = os.path.getatime(filename)
print 'Access time: %f (delta: %f)' % (last_access_time, last_access_time -
creation_time)
print 'Touching file.'
os.utime(filename, None)
last_access_time = os.path.getatime(filename)
print 'Access time (after touching): %f (delta: %f)' % (last_access_time,
last_access_time - creation_time)
os.remove(filename)
-----------------------------------------------------------

On Windows it gives me the following output:
Creation time: 1454351806.138276
Sleeping for one second.
Reading the file.
Access time: 1454351806.138276 (delta: 0.000000)
Touching file.
Access time (after touching): 1454351807.143000 (delta: 1.004724)

On Linux reading the file change its timestamp:
sebmarchand@sebmarchand0-l:~/src/tmp$ python py_access_time.py 
Creation time: 1454351758.208726
Sleeping for one second.
Reading the file.
Access time: 1454351759.208748 (delta: 1.000022)
Touching file.
Access time (after touching): 1454351759.208748 (delta: 1.000022)

It looks like the NtfsDisableLastAccessUpdate is set on my machine
(https://msdn.microsoft.com/en-us/library/ff794679(v=winembedded.60).aspx), I'm
not sure why so I'll try to find a better approach.

+    elif os.path.isfile(full_path):
+      os.remove(full_path)
+
+  for d in dirs_to_remove:
+    print ('Removing %s as it doesn\'t correspond to any known toolchain.' %
+           os.path.join(root, d))
+    # Use the RemoveToolchain function to remove these directories as they might
+    # contain an older version of the toolchain.
+    RemoveToolchain(root, d, False)
+
+  # Remove the versions of the toolchains that haven't been used in the past 30
+  # days.
+  toolchain_expiration_time = 60 * 60 * 24 * 30

[scottmg, 2016/01/28 23:04:11]

I'm not sure about this heuristic now. What if a mostly-Mac dev hasn't built on
Windows in a while, will all toolchains be deleted? Maybe we could sort by date
and keep at least the most recent one?

[Sébastien Marchand, 2016/02/01 19:29:29]

No, I'm touching the timestamp file of the toolchain that has been asked for
when calling this script (the user provided sha1), so there should always be at
least one toolchain installed.

+  for toolchain in valid_toolchains:
+    toolchain_age_in_sec = time.time() - toolchain[0]
+    if toolchain_age_in_sec > toolchain_expiration_time:
+      print ('Removing version %s of the Win toolchain has it hasn\'t been used'
+             ' in the past %d days.' % (toolchain[1],
+                                        toolchain_age_in_sec / 60 / 60 / 24))
+      RemoveToolchain(root, toolchain[1], True)
+
+
 def GetInstallerName():
   """Return the name of the Windows 10 Universal C Runtime installer for the
   current platform, or None if installer is not needed or not applicable.
   The registry has to be used instead of sys.getwindowsversion() because
   Python 2.7 is only manifested as being compatible up to Windows 8, so the
   version APIs helpfully return a maximum of 6.2 (Windows 8).
   """
   key_name = r'Software\Microsoft\Windows NT\CurrentVersion'
   key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, key_name)
   value, keytype = winreg.QueryValueEx(key, "CurrentVersion")
@@ skipping 72 matching lines @@
     def winpath(path):
       return subprocess.check_output(['cygpath', '-w', path]).strip()
     python = os.path.join(DEPOT_TOOLS_PATH, 'python.bat')
     cmd = [python, winpath(__file__)]
     if options.output_json:
       cmd.extend(['--output-json', winpath(options.output_json)])
     cmd.extend(args)
     sys.exit(subprocess.call(cmd))
   assert sys.platform != 'cygwin'
 
-  # We assume that the Pro hash is the first one.
-  desired_hashes = args
-  if len(desired_hashes) == 0:
-    sys.exit('Desired hashes are required.')
+  if len(args) == 0:
+    sys.exit('Desired hash is required.')
+  desired_hash = args[0]
 
   # Move to depot_tools\win_toolchain where we'll store our files, and where
   # the downloader script is.
   os.chdir(os.path.normpath(os.path.join(BASEDIR)))
   toolchain_dir = '.'
   if os.environ.get('GYP_MSVS_VERSION') == '2015':
     target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs_files'))
   else:
     target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs2013_files'))
-  abs_target_dir = os.path.abspath(target_dir)
+  if not os.path.isdir(target_dir):
+    os.mkdir(target_dir)
+  toolchain_target_dir = os.path.join(target_dir, desired_hash)
+
+  abs_toolchain_target_dir = os.path.abspath(toolchain_target_dir)
 
   got_new_toolchain = False
 
   # If the current hash doesn't match what we want in the file, nuke and pave.
   # Typically this script is only run when the .sha1 one file is updated, but
   # directly calling "gclient runhooks" will also run it, so we cache
   # based on timestamps to make that case fast.
-  current_hash = CalculateHash(target_dir)
-  if current_hash not in desired_hashes:
+  current_hashes = CalculateToolchainHashes(target_dir)
+  if desired_hash not in current_hashes:
     should_use_gs = False
     if (HaveSrcInternalAccess() or
         LooksLikeGoogler() or
         CanAccessToolchainBucket()):
       should_use_gs = True
       if not CanAccessToolchainBucket():
         RequestGsAuthentication()
     if not should_use_gs:
       print('\n\n\nPlease follow the instructions at '
             'https://www.chromium.org/developers/how-tos/'
             'build-instructions-windows\n\n')
       return 1
     print('Windows toolchain out of date or doesn\'t exist, updating (Pro)...')
-    print('  current_hash: %s' % current_hash)
-    print('  desired_hashes: %s' % ', '.join(desired_hashes))
+    print('  current_hashes: %s' % ', '.join(current_hashes))
+    print('  desired_hash: %s' % desired_hash)
     sys.stdout.flush()
-    DelayBeforeRemoving(target_dir)
-    if sys.platform == 'win32':
-      # These stay resident and will make the rmdir below fail.
-      kill_list = [
-        'mspdbsrv.exe',
-        'vctip.exe', # Compiler and tools experience improvement data uploader.
-      ]
-      for process_name in kill_list:
-        with open(os.devnull, 'wb') as nul:
-          subprocess.call(['taskkill', '/f', '/im', process_name],
-                          stdin=nul, stdout=nul, stderr=nul)
-    if os.path.isdir(target_dir):
-      RmDir(target_dir)
 
-    DoTreeMirror(target_dir, desired_hashes[0])
+    DoTreeMirror(toolchain_target_dir, desired_hash)
 
     got_new_toolchain = True
+  else:
+    # Touch the timestamp file so we know that this version of the toolchain is
+    # still used.
+    os.utime(os.path.join(target_dir, desired_hash), None)
 
-  win_sdk = os.path.join(abs_target_dir, 'win_sdk')
+  win_sdk = os.path.join(abs_toolchain_target_dir, 'win_sdk')
   try:
-    with open(os.path.join(target_dir, 'VS_VERSION'), 'rb') as f:
+    with open(os.path.join(toolchain_target_dir, 'VS_VERSION'), 'rb') as f:
       vs_version = f.read().strip()
   except IOError:
     # Older toolchains didn't have the VS_VERSION file, and used 'win8sdk'
     # instead of just 'win_sdk'.
     vs_version = '2013'
-    win_sdk = os.path.join(abs_target_dir, 'win8sdk')
+    win_sdk = os.path.join(abs_toolchain_target_dir, 'win8sdk')
 
   data = {
-      'path': abs_target_dir,
+      'path': abs_toolchain_target_dir,
       'version': vs_version,
       'win_sdk': win_sdk,
       # Added for backwards compatibility with old toolchain packages.
       'win8sdk': win_sdk,
-      'wdk': os.path.join(abs_target_dir, 'wdk'),
+      'wdk': os.path.join(abs_toolchain_target_dir, 'wdk'),
       'runtime_dirs': [
-        os.path.join(abs_target_dir, 'sys64'),
-        os.path.join(abs_target_dir, 'sys32'),
+        os.path.join(abs_toolchain_target_dir, 'sys64'),
+        os.path.join(abs_toolchain_target_dir, 'sys32'),
       ],
   }
   with open(os.path.join(target_dir, '..', 'data.json'), 'w') as f:
     json.dump(data, f)
 
   if got_new_toolchain:
-    current_hash = CalculateHash(target_dir)
-    if current_hash not in desired_hashes:
+    current_hashes = CalculateToolchainHashes(target_dir)
+    if desired_hash not in current_hashes:
       print >> sys.stderr, (
           'Got wrong hash after pulling a new toolchain. '
-          'Wanted one of \'%s\', got \'%s\'.' % (
-              ', '.join(desired_hashes), current_hash))
+          'Wanted \'%s\', got one of \'%s\'.' % (
+              desired_hash, ', '.join(current_hashes)))
       return 1
-    SaveTimestampsAndHash(target_dir, current_hash)
+    SaveTimestampsAndHash(target_dir, desired_hash)
 
   if options.output_json:
     shutil.copyfile(os.path.join(target_dir, '..', 'data.json'),
                     options.output_json)
 
   if os.environ.get('GYP_MSVS_VERSION') == '2015':
-    InstallUniversalCRTIfNeeded(abs_target_dir)
+    InstallUniversalCRTIfNeeded(abs_toolchain_target_dir)
+
+  RemoveUnusedToolchains(target_dir)
 
   return 0
 
 
 if __name__ == '__main__':
   sys.exit(main())