Cached timestamps for toolchain update script

tools/win/toolchain/get_toolchain_if_necessary.py

Index: tools/win/toolchain/get_toolchain_if_necessary.py
diff --git a/tools/win/toolchain/get_toolchain_if_necessary.py b/tools/win/toolchain/get_toolchain_if_necessary.py
index e93e6f029fa4352b3f9d39ec5242c25d18df6f97..5e63c649acf654e818973945a0f2f32051700175 100644
--- a/tools/win/toolchain/get_toolchain_if_necessary.py
+++ b/tools/win/toolchain/get_toolchain_if_necessary.py
@@ -3,6 +3,7 @@
 # found in the LICENSE file.
 
 import hashlib
+import json
 import os
 import subprocess
 import sys
@@ -11,23 +12,69 @@ import sys
 BASEDIR = os.path.dirname(os.path.abspath(__file__))
 
 
-def CalculateHash(root):
-  """Calculates the sha1 of the paths to all files in the given |root| and the
-  contents of those files, and returns as a hex string."""
+def GetFileList(root):
+  """Gets a normalized list of files under |root|."""
   assert not os.path.isabs(root)
   assert os.path.normpath(root) == root
-  digest = hashlib.sha1()
-  count = 0
+  file_list = []
   for root, dirs, files in os.walk(root):
     dirs.sort()
     for name in sorted(f.lower() for f in files):

[M-A Ruel, 2013/11/29 00:33:30]

file_list = []
for root, dirs, files in os.walk(root):
  file_list.extend(os.path.join(root, f).lower() for f in files))
return sorted(file_list)

much simpler.

[scottmg, 2013/11/29 01:54:10]

Interestingly, it results in a different order. Previously, the directories were
sorted and the files were sorted within the directories, but now x\a.txt will
appear before x\b\a.txt followed by x\c.txt.

But anyway... Done, and updated hash.

[M-A Ruel, 2013/11/29 02:07:28]

Ah! I didn't think about that. As you prefer.

-      path = os.path.join(root, name)
-      digest.update(path.lower())
-      with open(path, 'rb') as f:
-        digest.update(f.read())
+      file_list.append(os.path.join(root, name).lower())
+  return file_list
+
+
+def MakeTimestampsFileName(root):
+  return os.path.join(root, '..', '.timestamps')
+
+
+def CalculateHash(root):
+  """Calculates the sha1 of the paths to all files in the given |root| and the
+  contents of those files, and returns as a hex string."""
+  file_list = GetFileList(root)
+
+  # Check whether we previously saved timestamps in $root/../.timestamps. If
+  # we didn't, or they don't match, then do the full calculation, otherwise
+  # return the saved value.
+  timestamps_file = MakeTimestampsFileName(root)
+  timestamps_data = {'files': [], 'sha1': ''}
+  if os.path.exists(timestamps_file):
+    with open(timestamps_file, 'rb') as f:
+      try:
+        timestamps_data = json.load(f)
+      except ValueError:
+        # json couldn't be loaded, empty data will force a re-hash.
+        pass
+
+  matches = len(file_list) == len(timestamps_data['files'])

[M-A Ruel, 2013/11/29 00:33:30]

I wonder about "legitimate" junk files, like thumbs.db, that would slow it down
or worse constantly force a sync.

[scottmg, 2013/11/29 01:54:10]

Yeah... If it comes up we could blacklist some files. FWIW, it'll nuke the whole
folder when it reinstalls, so hopefully it sort of self-corrects.

+  if matches:
+    for disk, cached in zip(file_list, timestamps_data['files']):
+      if disk != cached[0] or os.stat(disk).st_mtime != cached[1]:
+        matches = False
+        break
+  if matches:
+    return timestamps_data['sha1']
+
+  digest = hashlib.sha1()
+  for path in file_list:
+    digest.update(path)
+    with open(path, 'rb') as f:
+      digest.update(f.read())
   return digest.hexdigest()
 
 
+def SaveTimestampsAndHash(root, sha1):
+  """Save timestamps and the final hash to be able to early-out more quickly
+  next time."""
+  file_list = GetFileList(root)
+  timestamps_data = {
+    'files': [[f, os.stat(f).st_mtime] for f in file_list],
+    'sha1': sha1,
+  }
+  with open(MakeTimestampsFileName(root), 'wb') as f:
+    json.dump(timestamps_data, f)
+
+
 def main():
   if sys.platform not in ('win32', 'cygwin'):
     return 0
@@ -48,9 +95,9 @@ def main():
       desired_hash = f.read().strip()
 
   # If the current hash doesn't match what we want in the file, nuke and pave.
-  # Note that this script is only run when a .sha1 file is updated (per DEPS)
-  # so this relatively expensive step of hashing everything only happens when
-  # the toolchain is updated.
+  # Typically this script is only run when the .sha1 one file is updated, but
+  # directly calling "gclient runhooks" will also run it, so we cache
+  # based on timestamps to make that case fast.
   current_hash = CalculateHash(target_dir)
   if current_hash != desired_hash:
     print 'Windows toolchain out of date or doesn\'t exist, updating...'
@@ -60,14 +107,15 @@ def main():
         sys.executable,
         'src\\tools\\win\\toolchain\\toolchain2013.py',
         '--targetdir', target_dir])
+    current_hash = CalculateHash(target_dir)
+    if current_hash != desired_hash:
+      print >> sys.stderr, (
+          'Got wrong hash after pulling a new toolchain. '
+          'Wanted \'%s\', got \'%s\'.' % (
+              desired_hash, current_hash))
+      return 1
+    SaveTimestampsAndHash(target_dir, current_hash)
 
-  current_hash = CalculateHash(target_dir)
-  if current_hash != desired_hash:
-    print >> sys.stderr, (
-        'Got wrong hash after pulling a new toolchain. '
-        'Wanted \'%s\', got \'%s\'.' % (
-            desired_hash, current_hash))
-    return 1
   return 0