Add the possibility to keep several version of the VS toolchain.

win_toolchain/get_toolchain_if_necessary.py

 #!/usr/bin/env python
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Downloads and unpacks a toolchain for building on Windows. The contents are
 matched by sha1 which will be updated when the toolchain is updated.
 
 Having a toolchain script in depot_tools means that it's not versioned
 directly with the source code. That is, if the toolchain is upgraded, but
@@ skipping 60 matching lines @@
   """Gets a normalized list of files under |root|."""
   assert not os.path.isabs(root)
   assert os.path.normpath(root) == root
   file_list = []
   for base, _, files in os.walk(root):
     paths = [os.path.join(base, f) for f in files]
     file_list.extend(x.lower() for x in paths)
   return sorted(file_list, key=lambda s: s.replace('/', '\\'))
 
 
-def MakeTimestampsFileName(root):
-  return os.path.join(root, '..', '.timestamps')
+def MakeTimestampsFileName(root, sha1):
+  return os.path.join(root, os.pardir, '%s.timestamps' % sha1)
 
 
-def CalculateHash(root):
+def CalculateHash(root, expected_hash=''):

[scottmg, 2016/01/27 21:46:12]

Can we get rid of the default value for expected_hash here?

[Sébastien Marchand, 2016/01/28 22:01:15]

Done, it was here because it's used by vs_toolchain.py

   """Calculates the sha1 of the paths to all files in the given |root| and the
   contents of those files, and returns as a hex string."""
-  file_list = GetFileList(root)
-
+  file_list = GetFileList(os.path.join(root, expected_hash))
   # Check whether we previously saved timestamps in $root/../.timestamps. If

[scottmg, 2016/01/27 21:46:12]

../{sha1}.timestamps

[Sébastien Marchand, 2016/01/28 22:01:15]

Done.

   # we didn't, or they don't match, then do the full calculation, otherwise
   # return the saved value.
-  timestamps_file = MakeTimestampsFileName(root)
+  timestamps_file = MakeTimestampsFileName(root, expected_hash)
   timestamps_data = {'files': [], 'sha1': ''}
   if os.path.exists(timestamps_file):
     with open(timestamps_file, 'rb') as f:
       try:
         timestamps_data = json.load(f)
       except ValueError:
         # json couldn't be loaded, empty data will force a re-hash.
         pass
 
   matches = len(file_list) == len(timestamps_data['files'])
   if matches:
     for disk, cached in zip(file_list, timestamps_data['files']):
       if disk != cached[0] or os.stat(disk).st_mtime != cached[1]:
         matches = False
         break
   if matches:
     return timestamps_data['sha1']
 
   digest = hashlib.sha1()
   for path in file_list:
-    digest.update(str(path).replace('/', '\\'))
+    if expected_hash != '':
+      path_without_hash = (
+          str(path).replace('/', '\\').replace(expected_hash, '').replace(
+              '\\\\', '\\'))
+    else:
+      path_without_hash = str(path).replace('/', '\\')
+    digest.update(path_without_hash)
     with open(path, 'rb') as f:
       digest.update(f.read())
   return digest.hexdigest()
 
 
+def CalculateToolchainHashes(root):
+  """Calculate the hash of the different toolchains installed in the |root|
+  directory."""
+  hashes = []
+  dir_list = [
+      d for d in os.listdir(root) if os.path.isdir(os.path.join(root, d))]
+  for d in dir_list:
+    hashes.append(CalculateHash(root, d))
+  return hashes
+
+
 def SaveTimestampsAndHash(root, sha1):
   """Saves timestamps and the final hash to be able to early-out more quickly
   next time."""
-  file_list = GetFileList(root)
+  file_list = GetFileList(os.path.join(root, sha1))
   timestamps_data = {
     'files': [[f, os.stat(f).st_mtime] for f in file_list],
     'sha1': sha1,
   }
-  with open(MakeTimestampsFileName(root), 'wb') as f:
+  with open(MakeTimestampsFileName(root, sha1), 'wb') as f:
     json.dump(timestamps_data, f)
 
 
 def HaveSrcInternalAccess():
   """Checks whether access to src-internal is available."""
   with open(os.devnull, 'w') as nul:
     if subprocess.call(
         ['svn', 'ls', '--non-interactive',
          'svn://svn.chromium.org/chrome-internal/trunk/src-internal/'],
         shell=True, stdin=nul, stdout=nul, stderr=nul) == 0:
@@ skipping 181 matching lines @@
     def winpath(path):
       return subprocess.check_output(['cygpath', '-w', path]).strip()
     python = os.path.join(DEPOT_TOOLS_PATH, 'python.bat')
     cmd = [python, winpath(__file__)]
     if options.output_json:
       cmd.extend(['--output-json', winpath(options.output_json)])
     cmd.extend(args)
     sys.exit(subprocess.call(cmd))
   assert sys.platform != 'cygwin'
 
-  # We assume that the Pro hash is the first one.
-  desired_hashes = args
-  if len(desired_hashes) == 0:
-    sys.exit('Desired hashes are required.')
+  if len(args) == 0:
+    sys.exit('Desired hash is required.')
+  desired_hash = args[0]
 
   # Move to depot_tools\win_toolchain where we'll store our files, and where
   # the downloader script is.
   os.chdir(os.path.normpath(os.path.join(BASEDIR)))
   toolchain_dir = '.'
   if os.environ.get('GYP_MSVS_VERSION') == '2015':
     target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs_files'))
   else:
     target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs2013_files'))
-  abs_target_dir = os.path.abspath(target_dir)
+  if not os.path.isdir(target_dir):
+    os.mkdir(target_dir)
+  toolchain_target_dir = os.path.join(target_dir, desired_hash)
+
+  abs_toolchain_target_dir = os.path.abspath(toolchain_target_dir)
 
   got_new_toolchain = False
 
   # If the current hash doesn't match what we want in the file, nuke and pave.
   # Typically this script is only run when the .sha1 one file is updated, but
   # directly calling "gclient runhooks" will also run it, so we cache
   # based on timestamps to make that case fast.
-  current_hash = CalculateHash(target_dir)
-  if current_hash not in desired_hashes:
+  current_hashes = CalculateToolchainHashes(target_dir)
+  if desired_hash not in current_hashes:
     should_use_gs = False
     if (HaveSrcInternalAccess() or
         LooksLikeGoogler() or
         CanAccessToolchainBucket()):
       should_use_gs = True
       if not CanAccessToolchainBucket():
         RequestGsAuthentication()
     if not should_use_gs:
       print('\n\n\nPlease follow the instructions at '
             'https://www.chromium.org/developers/how-tos/'
             'build-instructions-windows\n\n')
       return 1
     print('Windows toolchain out of date or doesn\'t exist, updating (Pro)...')
-    print('  current_hash: %s' % current_hash)
-    print('  desired_hashes: %s' % ', '.join(desired_hashes))
+    print('  current_hashes: %s' % ', '.join(current_hashes))
+    print('  desired_hash: %s' % desired_hash)
     sys.stdout.flush()
-    DelayBeforeRemoving(target_dir)
+    DelayBeforeRemoving(toolchain_target_dir)

[scottmg, 2016/01/27 21:46:12]

Can we can delete from line 388-400 now? It seems like we can never delete a
toolchain here now.

[Sébastien Marchand, 2016/01/28 22:01:15]

Move to the RemoveUnusedToolchains function.

     if sys.platform == 'win32':
       # These stay resident and will make the rmdir below fail.
       kill_list = [
         'mspdbsrv.exe',
         'vctip.exe', # Compiler and tools experience improvement data uploader.
       ]
       for process_name in kill_list:
         with open(os.devnull, 'wb') as nul:
           subprocess.call(['taskkill', '/f', '/im', process_name],
                           stdin=nul, stdout=nul, stderr=nul)
-    if os.path.isdir(target_dir):
-      RmDir(target_dir)
+    if os.path.isdir(toolchain_target_dir):
+      RmDir(toolchain_target_dir)
 
-    DoTreeMirror(target_dir, desired_hashes[0])
+    DoTreeMirror(toolchain_target_dir, desired_hash)
 
     got_new_toolchain = True
 
-  win_sdk = os.path.join(abs_target_dir, 'win_sdk')
+  win_sdk = os.path.join(abs_toolchain_target_dir, 'win_sdk')
   try:
-    with open(os.path.join(target_dir, 'VS_VERSION'), 'rb') as f:
+    with open(os.path.join(toolchain_target_dir, 'VS_VERSION'), 'rb') as f:
       vs_version = f.read().strip()
   except IOError:
     # Older toolchains didn't have the VS_VERSION file, and used 'win8sdk'
     # instead of just 'win_sdk'.
     vs_version = '2013'
-    win_sdk = os.path.join(abs_target_dir, 'win8sdk')
+    win_sdk = os.path.join(abs_toolchain_target_dir, 'win8sdk')
 
   data = {
-      'path': abs_target_dir,
+      'path': abs_toolchain_target_dir,
       'version': vs_version,
       'win_sdk': win_sdk,
       # Added for backwards compatibility with old toolchain packages.
       'win8sdk': win_sdk,
-      'wdk': os.path.join(abs_target_dir, 'wdk'),
+      'wdk': os.path.join(abs_toolchain_target_dir, 'wdk'),
       'runtime_dirs': [
-        os.path.join(abs_target_dir, 'sys64'),
-        os.path.join(abs_target_dir, 'sys32'),
+        os.path.join(abs_toolchain_target_dir, 'sys64'),
+        os.path.join(abs_toolchain_target_dir, 'sys32'),
       ],
   }
   with open(os.path.join(target_dir, '..', 'data.json'), 'w') as f:
     json.dump(data, f)
 
   if got_new_toolchain:
-    current_hash = CalculateHash(target_dir)
-    if current_hash not in desired_hashes:
+    current_hashes = CalculateToolchainHashes(target_dir)
+    if desired_hash not in current_hashes:
       print >> sys.stderr, (
           'Got wrong hash after pulling a new toolchain. '
-          'Wanted one of \'%s\', got \'%s\'.' % (
-              ', '.join(desired_hashes), current_hash))
+          'Wanted \'%s\', got one of \'%s\'.' % (
+              desired_hash, ', '.join(current_hashes)))
       return 1
-    SaveTimestampsAndHash(target_dir, current_hash)
+    SaveTimestampsAndHash(target_dir, desired_hash)
 
   if options.output_json:
     shutil.copyfile(os.path.join(target_dir, '..', 'data.json'),
                     options.output_json)
 
   if os.environ.get('GYP_MSVS_VERSION') == '2015':
-    InstallUniversalCRTIfNeeded(abs_target_dir)
+    InstallUniversalCRTIfNeeded(abs_toolchain_target_dir)
 
   return 0
 
 
 if __name__ == '__main__':
   sys.exit(main())