Improve the toolchains-hash calculation.

win_toolchain/get_toolchain_if_necessary.py

 #!/usr/bin/env python
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Downloads and unpacks a toolchain for building on Windows. The contents are
 matched by sha1 which will be updated when the toolchain is updated.
 
 Having a toolchain script in depot_tools means that it's not versioned
 directly with the source code. That is, if the toolchain is upgraded, but
@@ skipping 54 matching lines @@
   # Allow use of utility functions in this script from package_from_installed
   # on bare VM that doesn't have a full depot_tools.
   pass
 
 
 def GetFileList(root):
   """Gets a normalized list of files under |root|."""
   assert not os.path.isabs(root)
   assert os.path.normpath(root) == root
   file_list = []
+  # Ignore WER ReportQueue entries that vctip/cl leave in the bin dir if/when
+  # they crash. Also ignores the content of the win_sdk/debuggers/x(86|64)/sym/
+  # directories as this is just the temporarily location that Windbg might use
+  # to store the symbol files.
+  ignored_directories = ['wer\\reportqueue',

[scottmg, 2016/05/11 19:21:36]

Does this intentionally not have a trailing \?

[Sébastien Marchand, 2016/05/11 19:59:50]

I don't know if reportqueue is the name of the directory containing the
ReportQueue entries or if it's just the beginning of the filenames (e.g.
wer\\reportqueue_foo.bar), so I just didn't changed it.

+                         'win_sdk\\debuggers\\x86\\sym\\',
+                         'win_sdk\\debuggers\\x64\\sym\\']
   for base, _, files in os.walk(root):
-    paths = [os.path.join(base, f) for f in files]
-    # Ignore WER ReportQueue entries that vctip/cl leave in the bin dir if/when
-    # they crash.
-    file_list.extend(x.lower() for x in paths if 'WER\\ReportQueue' not in x)
+    paths = [os.path.join(base, f).lower() for f in files]
+    for f in paths:

[scottmg, 2016/05/11 19:21:36]

maybe p instead of f since f is used above

[Sébastien Marchand, 2016/05/11 19:59:50]

Done.

+      if any(ignored_dir in f for ignored_dir in ignored_directories):
+        continue
+      file_list.append(f)
   return sorted(file_list, key=lambda s: s.replace('/', '\\'))
 
 
 def MakeTimestampsFileName(root, sha1):
   return os.path.join(root, os.pardir, '%s.timestamps' % sha1)
 
 
 def CalculateHash(root, expected_hash):
   """Calculates the sha1 of the paths to all files in the given |root| and the
   contents of those files, and returns as a hex string.
@@ skipping 22 matching lines @@
   matches = len(file_list) == len(timestamps_data['files'])
   # Don't check the timestamp of the version file as we touch this file to
   # indicates which versions of the toolchain are still being used.
   vc_dir = os.path.join(full_root_path, 'VC').lower()
   if matches:
     for disk, cached in zip(file_list, timestamps_data['files']):
       if disk != cached[0] or (
           disk != vc_dir and os.path.getmtime(disk) != cached[1]):
         matches = False
         break
+  elif os.path.exists(timestamps_file):
+    # Print some information about the extra/missing files. Don't do this if we
+    # don't have a timestamp file, as all the files will be considered as
+    # missing.
+    timestamps_data_files = []
+    for f in timestamps_data['files']:
+      timestamps_data_files.append(f[0])
+    missing_files = [f for f in timestamps_data_files if f not in file_list]
+    if len(missing_files):
+      print ('Some files are missing from the %s version of the toolchain:' %
+             expected_hash)
+      for f in missing_files:
+        print '\t%s' % f
+    extra_files = [f for f in file_list if f not in timestamps_data_files]
+    if len(extra_files):
+      print ('There\'s some extra files in the %s version of the toolchain:' %
+             expected_hash)
+      for f in extra_files:
+        print '\t%s' % f
   if matches:
     return timestamps_data['sha1']
 
   # Make long hangs when updating the toolchain less mysterious.
   print 'Calculating hash of toolchain in %s. Please wait...' % full_root_path
   sys.stdout.flush()
   digest = hashlib.sha1()
   for path in file_list:
     path_without_hash = str(path).replace('/', '\\')
     if expected_hash:
       path_without_hash = path_without_hash.replace(
           os.path.join(root, expected_hash).replace('/', '\\'), root)
     digest.update(path_without_hash)
     with open(path, 'rb') as f:
       digest.update(f.read())
   return digest.hexdigest()
 
 
-def CalculateToolchainHashes(root):
+def CalculateToolchainHashes(root, remove_corrupt_toolchains):

[scottmg, 2016/05/11 19:21:36]

This new argument doesn't seem to be used.

[Sébastien Marchand, 2016/05/11 19:59:50]

Oops. Fixed.

   """Calculate the hash of the different toolchains installed in the |root|
   directory."""
   hashes = []
   dir_list = [
       d for d in os.listdir(root) if os.path.isdir(os.path.join(root, d))]
   for d in dir_list:
-    hashes.append(CalculateHash(root, d))
+    toolchain_hash = CalculateHash(root, d)
+    if toolchain_hash != d:
+      print ('The hash of a version of the toolchain has an unexpected value ('
+             '%s instead of %s), removing it.' % (toolchain_hash, d))
+      RemoveToolchain(root, d, True)
+    else:
+      hashes.append(toolchain_hash)
   return hashes
 
 
 def SaveTimestampsAndHash(root, sha1):
   """Saves timestamps and the final hash to be able to early-out more quickly
   next time."""
   file_list = GetFileList(os.path.join(root, sha1))
   timestamps_data = {
     'files': [[f, os.path.getmtime(f)] for f in file_list],
     'sha1': sha1,
@@ skipping 260 matching lines @@
   toolchain_target_dir = os.path.join(target_dir, desired_hash)
 
   abs_toolchain_target_dir = os.path.abspath(toolchain_target_dir)
 
   got_new_toolchain = False
 
   # If the current hash doesn't match what we want in the file, nuke and pave.
   # Typically this script is only run when the .sha1 one file is updated, but
   # directly calling "gclient runhooks" will also run it, so we cache
   # based on timestamps to make that case fast.
-  current_hashes = CalculateToolchainHashes(target_dir)
+  current_hashes = CalculateToolchainHashes(target_dir, True)
   if desired_hash not in current_hashes:
     should_use_gs = False
     if (HaveSrcInternalAccess() or
         LooksLikeGoogler() or
         CanAccessToolchainBucket()):
       should_use_gs = True
       if not CanAccessToolchainBucket():
         RequestGsAuthentication()
     if not should_use_gs:
       print('\n\n\nPlease follow the instructions at '
@@ skipping 33 matching lines @@
       'wdk': os.path.join(abs_toolchain_target_dir, 'wdk'),
       'runtime_dirs': [
         os.path.join(abs_toolchain_target_dir, 'sys64'),
         os.path.join(abs_toolchain_target_dir, 'sys32'),
       ],
   }
   with open(os.path.join(target_dir, '..', 'data.json'), 'w') as f:
     json.dump(data, f)
 
   if got_new_toolchain:
-    current_hashes = CalculateToolchainHashes(target_dir)
+    current_hashes = CalculateToolchainHashes(target_dir, False)
     if desired_hash not in current_hashes:
       print >> sys.stderr, (
           'Got wrong hash after pulling a new toolchain. '
           'Wanted \'%s\', got one of \'%s\'.' % (
               desired_hash, ', '.join(current_hashes)))
       return 1
     SaveTimestampsAndHash(target_dir, desired_hash)
 
   if options.output_json:
     shutil.copyfile(os.path.join(target_dir, '..', 'data.json'),
                     options.output_json)
 
   EnableCrashDumpCollection()
 
   if os.environ.get('GYP_MSVS_VERSION') == '2015':
     InstallUniversalCRTIfNeeded(abs_toolchain_target_dir)
 
   RemoveUnusedToolchains(target_dir)
 
   return 0
 
 
 if __name__ == '__main__':
   sys.exit(main())