Improve the toolchains-hash calculation.

win_toolchain/get_toolchain_if_necessary.py

 #!/usr/bin/env python
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Downloads and unpacks a toolchain for building on Windows. The contents are
 matched by sha1 which will be updated when the toolchain is updated.
 
 Having a toolchain script in depot_tools means that it's not versioned
 directly with the source code. That is, if the toolchain is upgraded, but
@@ skipping 54 matching lines @@
   # Allow use of utility functions in this script from package_from_installed
   # on bare VM that doesn't have a full depot_tools.
   pass
 
 
 def GetFileList(root):
   """Gets a normalized list of files under |root|."""
   assert not os.path.isabs(root)
   assert os.path.normpath(root) == root
   file_list = []
+  # Ignore WER ReportQueue entries that vctip/cl leave in the bin dir if/when
+  # they crash. Also ignores the content of the win_sdk/debuggers/x(86|64)/sym/
+  # directories as this is just the temporarily location that Windbg might use
+  # to store the symbol files.
+  ignored_directories = ['wer\\reportqueue',
+                         'win_sdk\\debuggers\\x86\\sym\\',
+                         'win_sdk\\debuggers\\x64\\sym\\']

[Nico, 2016/05/27 14:45:57]

this won't work on non-windows. it looks like this currently has no ill effects
though.

[Sébastien Marchand, 2016/05/27 15:16:22]

These files get created by Windbg/WER, so they shouldn't exist on a non-Windows
system ? (unless you use Windbg with Wine ? Not sure if it works)

   for base, _, files in os.walk(root):
-    paths = [os.path.join(base, f) for f in files]
-    # Ignore WER ReportQueue entries that vctip/cl leave in the bin dir if/when
-    # they crash.
-    file_list.extend(x.lower() for x in paths if 'WER\\ReportQueue' not in x)
+    paths = [os.path.join(base, f).lower() for f in files]
+    for p in paths:
+      if any(ignored_dir in p for ignored_dir in ignored_directories):
+        continue
+      file_list.append(p)
   return sorted(file_list, key=lambda s: s.replace('/', '\\'))
 
 
 def MakeTimestampsFileName(root, sha1):
   return os.path.join(root, os.pardir, '%s.timestamps' % sha1)
 
 
 def CalculateHash(root, expected_hash):
   """Calculates the sha1 of the paths to all files in the given |root| and the
   contents of those files, and returns as a hex string.
@@ skipping 22 matching lines @@
   matches = len(file_list) == len(timestamps_data['files'])
   # Don't check the timestamp of the version file as we touch this file to
   # indicates which versions of the toolchain are still being used.
   vc_dir = os.path.join(full_root_path, 'VC').lower()
   if matches:
     for disk, cached in zip(file_list, timestamps_data['files']):
       if disk != cached[0] or (
           disk != vc_dir and os.path.getmtime(disk) != cached[1]):
         matches = False
         break
+  elif os.path.exists(timestamps_file):
+    # Print some information about the extra/missing files. Don't do this if we
+    # don't have a timestamp file, as all the files will be considered as
+    # missing.

[Nico, 2016/06/22 15:09:11]

On my windows box, this just spent several minutes printing many screenfulls of
filenames. I don't know if it was missing or extra files (the header scrolled
off), but it looked like it printed every single filename. Maybe this should
just say "N missing files, first 10 here". The current experience isn't so good.

+    timestamps_data_files = []
+    for f in timestamps_data['files']:
+      timestamps_data_files.append(f[0])
+    missing_files = [f for f in timestamps_data_files if f not in file_list]
+    if len(missing_files):
+      print ('Some files are missing from the %s version of the toolchain:' %
+             expected_hash)
+      for f in missing_files:
+        print '\t%s' % f
+    extra_files = [f for f in file_list if f not in timestamps_data_files]
+    if len(extra_files):
+      print ('There\'s some extra files in the %s version of the toolchain:' %
+             expected_hash)
+      for f in extra_files:
+        print '\t%s' % f
   if matches:
     return timestamps_data['sha1']
 
   # Make long hangs when updating the toolchain less mysterious.
   print 'Calculating hash of toolchain in %s. Please wait...' % full_root_path
   sys.stdout.flush()
   digest = hashlib.sha1()
   for path in file_list:
     path_without_hash = str(path).replace('/', '\\')
     if expected_hash:
       path_without_hash = path_without_hash.replace(
           os.path.join(root, expected_hash).replace('/', '\\'), root)
     digest.update(path_without_hash)
     with open(path, 'rb') as f:
       digest.update(f.read())
   return digest.hexdigest()
 
 
-def CalculateToolchainHashes(root):
+def CalculateToolchainHashes(root, remove_corrupt_toolchains):
   """Calculate the hash of the different toolchains installed in the |root|
   directory."""
   hashes = []
   dir_list = [
       d for d in os.listdir(root) if os.path.isdir(os.path.join(root, d))]
   for d in dir_list:
-    hashes.append(CalculateHash(root, d))
+    toolchain_hash = CalculateHash(root, d)
+    if toolchain_hash != d:

[Nico, 2016/07/08 14:38:30]

Do you really want to compare toolchain_hash and d here? On My linux box, I got
several lines like

Calculating hash of toolchain in vs_files/DIA SDK. Please wait...
The hash of a version of the toolchain has an unexpected value
(5b9fe12411cc48404db2dcdfdef641f1bc213804 instead of DIA SDK), removing it.

It happened only once when an update was indeed necessary, but "5b9fe..." and
"DIA SDK" look like strings with different types.

[Sébastien Marchand, 2016/07/08 14:44:22]

It's not a bug, it's a feature :). In the past it wasn't possible to keep
several versions of the toolchain at the same time, so the win_toolchain
directory structure looked like this:

win_toolchain/vs_files/DIA_SDK
                      |win_sdk
                      |...

Now each version of the toolchain is kept in a different subdirectory:

win_toolchain/vs_files/{hash}/DIA_SDK
                             |win_sdk
                             |...

So the original copy of the toolchain (the one directly in vs_files/) will look
corrupt and each of these directory will get removed.

It should only happen once.
 

+      print ('The hash of a version of the toolchain has an unexpected value ('
+             '%s instead of %s)%s.' % (toolchain_hash, d,
+             ', removing it' if remove_corrupt_toolchains else ''))
+      if remove_corrupt_toolchains:
+        RemoveToolchain(root, d, True)
+    else:
+      hashes.append(toolchain_hash)
   return hashes
 
 
 def SaveTimestampsAndHash(root, sha1):
   """Saves timestamps and the final hash to be able to early-out more quickly
   next time."""
   file_list = GetFileList(os.path.join(root, sha1))
   timestamps_data = {
     'files': [[f, os.path.getmtime(f)] for f in file_list],
     'sha1': sha1,
@@ skipping 260 matching lines @@
   toolchain_target_dir = os.path.join(target_dir, desired_hash)
 
   abs_toolchain_target_dir = os.path.abspath(toolchain_target_dir)
 
   got_new_toolchain = False
 
   # If the current hash doesn't match what we want in the file, nuke and pave.
   # Typically this script is only run when the .sha1 one file is updated, but
   # directly calling "gclient runhooks" will also run it, so we cache
   # based on timestamps to make that case fast.
-  current_hashes = CalculateToolchainHashes(target_dir)
+  current_hashes = CalculateToolchainHashes(target_dir, True)
   if desired_hash not in current_hashes:
     should_use_gs = False
     if (HaveSrcInternalAccess() or
         LooksLikeGoogler() or
         CanAccessToolchainBucket()):
       should_use_gs = True
       if not CanAccessToolchainBucket():
         RequestGsAuthentication()
     if not should_use_gs:
       print('\n\n\nPlease follow the instructions at '
@@ skipping 33 matching lines @@
       'wdk': os.path.join(abs_toolchain_target_dir, 'wdk'),
       'runtime_dirs': [
         os.path.join(abs_toolchain_target_dir, 'sys64'),
         os.path.join(abs_toolchain_target_dir, 'sys32'),
       ],
   }
   with open(os.path.join(target_dir, '..', 'data.json'), 'w') as f:
     json.dump(data, f)
 
   if got_new_toolchain:
-    current_hashes = CalculateToolchainHashes(target_dir)
+    current_hashes = CalculateToolchainHashes(target_dir, False)
     if desired_hash not in current_hashes:
       print >> sys.stderr, (
           'Got wrong hash after pulling a new toolchain. '
           'Wanted \'%s\', got one of \'%s\'.' % (
               desired_hash, ', '.join(current_hashes)))
       return 1
     SaveTimestampsAndHash(target_dir, desired_hash)
 
   if options.output_json:
     shutil.copyfile(os.path.join(target_dir, '..', 'data.json'),
                     options.output_json)
 
   EnableCrashDumpCollection()
 
   if os.environ.get('GYP_MSVS_VERSION') == '2015':
     InstallUniversalCRTIfNeeded(abs_toolchain_target_dir)
 
   RemoveUnusedToolchains(target_dir)
 
   return 0
 
 
 if __name__ == '__main__':
   sys.exit(main())