Cached timestamps for toolchain update script

tools/win/toolchain/get_toolchain_if_necessary.py

 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import hashlib
+import json
 import os
 import subprocess
 import sys
 
 
 BASEDIR = os.path.dirname(os.path.abspath(__file__))
 
 
+def GetFileList(root):
+  """Gets a normalized list of files under |root|."""
+  assert not os.path.isabs(root)
+  assert os.path.normpath(root) == root
+  file_list = []
+  for root, dirs, files in os.walk(root):
+    dirs.sort()
+    for name in sorted(f.lower() for f in files):

[M-A Ruel, 2013/11/29 00:33:30]

file_list = []
for root, dirs, files in os.walk(root):
  file_list.extend(os.path.join(root, f).lower() for f in files))
return sorted(file_list)

much simpler.

[scottmg, 2013/11/29 01:54:10]

Interestingly, it results in a different order. Previously, the directories were
sorted and the files were sorted within the directories, but now x\a.txt will
appear before x\b\a.txt followed by x\c.txt.

But anyway... Done, and updated hash.

[M-A Ruel, 2013/11/29 02:07:28]

Ah! I didn't think about that. As you prefer.

+      file_list.append(os.path.join(root, name).lower())
+  return file_list
+
+
+def MakeTimestampsFileName(root):
+  return os.path.join(root, '..', '.timestamps')
+
+
 def CalculateHash(root):
   """Calculates the sha1 of the paths to all files in the given |root| and the
   contents of those files, and returns as a hex string."""
-  assert not os.path.isabs(root)
-  assert os.path.normpath(root) == root
+  file_list = GetFileList(root)
+
+  # Check whether we previously saved timestamps in $root/../.timestamps. If
+  # we didn't, or they don't match, then do the full calculation, otherwise
+  # return the saved value.
+  timestamps_file = MakeTimestampsFileName(root)
+  timestamps_data = {'files': [], 'sha1': ''}
+  if os.path.exists(timestamps_file):
+    with open(timestamps_file, 'rb') as f:
+      try:
+        timestamps_data = json.load(f)
+      except ValueError:
+        # json couldn't be loaded, empty data will force a re-hash.
+        pass
+
+  matches = len(file_list) == len(timestamps_data['files'])

[M-A Ruel, 2013/11/29 00:33:30]

I wonder about "legitimate" junk files, like thumbs.db, that would slow it down
or worse constantly force a sync.

[scottmg, 2013/11/29 01:54:10]

Yeah... If it comes up we could blacklist some files. FWIW, it'll nuke the whole
folder when it reinstalls, so hopefully it sort of self-corrects.

+  if matches:
+    for disk, cached in zip(file_list, timestamps_data['files']):
+      if disk != cached[0] or os.stat(disk).st_mtime != cached[1]:
+        matches = False
+        break
+  if matches:
+    return timestamps_data['sha1']
+
   digest = hashlib.sha1()
-  count = 0
-  for root, dirs, files in os.walk(root):
-    dirs.sort()
-    for name in sorted(f.lower() for f in files):
-      path = os.path.join(root, name)
-      digest.update(path.lower())
-      with open(path, 'rb') as f:
-        digest.update(f.read())
+  for path in file_list:
+    digest.update(path)
+    with open(path, 'rb') as f:
+      digest.update(f.read())
   return digest.hexdigest()
 
 
+def SaveTimestampsAndHash(root, sha1):
+  """Save timestamps and the final hash to be able to early-out more quickly
+  next time."""
+  file_list = GetFileList(root)
+  timestamps_data = {
+    'files': [[f, os.stat(f).st_mtime] for f in file_list],
+    'sha1': sha1,
+  }
+  with open(MakeTimestampsFileName(root), 'wb') as f:
+    json.dump(timestamps_data, f)
+
+
 def main():
   if sys.platform not in ('win32', 'cygwin'):
     return 0
 
   if len(sys.argv) != 1:
     print >> sys.stderr, 'Unexpected arguments.'
     return 1
 
   # Move to same location as .gclient. This is a no-op when run via gclient.
   os.chdir(os.path.normpath(os.path.join(BASEDIR, '..\\..\\..\\..')))
   toolchain_dir = 'src\\third_party\\win_toolchain'
   target_dir = os.path.join(toolchain_dir, 'files')
 
   sha1path = os.path.join(toolchain_dir, 'toolchain.sha1')
   desired_hash = ''
   if os.path.isfile(sha1path):
     with open(sha1path, 'rb') as f:
       desired_hash = f.read().strip()
 
   # If the current hash doesn't match what we want in the file, nuke and pave.
-  # Note that this script is only run when a .sha1 file is updated (per DEPS)
-  # so this relatively expensive step of hashing everything only happens when
-  # the toolchain is updated.
+  # Typically this script is only run when the .sha1 one file is updated, but
+  # directly calling "gclient runhooks" will also run it, so we cache
+  # based on timestamps to make that case fast.
   current_hash = CalculateHash(target_dir)
   if current_hash != desired_hash:
     print 'Windows toolchain out of date or doesn\'t exist, updating...'
     if os.path.isdir(target_dir):
       subprocess.check_call('rmdir /s/q "%s"' % target_dir, shell=True)
     subprocess.check_call([
         sys.executable,
         'src\\tools\\win\\toolchain\\toolchain2013.py',
         '--targetdir', target_dir])
+    current_hash = CalculateHash(target_dir)
+    if current_hash != desired_hash:
+      print >> sys.stderr, (
+          'Got wrong hash after pulling a new toolchain. '
+          'Wanted \'%s\', got \'%s\'.' % (
+              desired_hash, current_hash))
+      return 1
+    SaveTimestampsAndHash(target_dir, current_hash)
 
-  current_hash = CalculateHash(target_dir)
-  if current_hash != desired_hash:
-    print >> sys.stderr, (
-        'Got wrong hash after pulling a new toolchain. '
-        'Wanted \'%s\', got \'%s\'.' % (
-            desired_hash, current_hash))
-    return 1
   return 0
 
 
 if __name__ == '__main__':
   sys.exit(main())