Speed up presubmit cpplint checking by skipping unchanged files

tools/presubmit.py

 #!/usr/bin/env python
 #
 # Copyright 2008 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
@@ skipping 10 matching lines @@
 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
+import md5
 import optparse
 import os
 from os.path import abspath, join, dirname, basename, exists
+import pickle
 import re
 import sys
 import subprocess
 
 # Disabled LINT rules and reason.
 # build/include_what_you_use: Started giving false positives for variables
 #  named "string" and "map" assuming that you needed to include STL headers.
 
 ENABLED_LINT_RULES = """
 build/class
@@ skipping 42 matching lines @@
 whitespace/labels
 whitespace/line_length
 whitespace/newline
 whitespace/operators
 whitespace/parens
 whitespace/tab
 whitespace/todo
 """.split()
 
 
+class FileContentsCache(object):
+
+  def __init__(self, sums_file_name):
+    self.sums = {}
+    self.sums_file_name = sums_file_name
+
+  def Load(self):
+    try:
+      sums_file = None
+      try:
+        sums_file = open(self.sums_file_name, 'r')
+        self.sums = pickle.load(sums_file)
+      except IOError:
+        # File might not exist, this is OK.
+        pass
+    finally:
+      if sums_file:
+        sums_file.close()
+
+  def Save(self):
+    try:
+      sums_file = open(self.sums_file_name, 'w')
+      pickle.dump(self.sums, sums_file)
+    finally:
+      sums_file.close()
+
+  def FilterUnchangedFiles(self, files):
+    changed_or_new = []
+    for file in files:
+      try:
+        handle = open(file, "r")
+        file_sum = md5.new(handle.read()).digest()
+        if not file in self.sums or self.sums[file] != file_sum:
+          changed_or_new.append(file)
+          self.sums[file] = file_sum
+      finally:
+        handle.close()
+    return changed_or_new
+
+  def RemoveFile(self, file):
+    if file in self.sums:
+      self.sums.pop(file)
+
+
 class SourceFileProcessor(object):
   """
   Utility class that can run through a directory structure, find all relevant
   files and invoke a custom check on the files.
   """
 
   def Run(self, path):
     all_files = []
     for file in self.GetPathsToSearch():
       all_files += self.FindFilesIn(join(path, file))
@@ skipping 24 matching lines @@
   """
 
   def IsRelevant(self, name):
     return name.endswith('.cc') or name.endswith('.h')
 
   def IgnoreDir(self, name):
     return (super(CppLintProcessor, self).IgnoreDir(name)
               or (name == 'third_party'))
 
   IGNORE_LINT = ['flag-definitions.h']
-  
+
   def IgnoreFile(self, name):
     return (super(CppLintProcessor, self).IgnoreFile(name)
               or (name in CppLintProcessor.IGNORE_LINT))
 
   def GetPathsToSearch(self):
     return ['src', 'public', 'samples', join('test', 'cctest')]
 
   def ProcessFiles(self, files, path):
+    good_files_cache = FileContentsCache('.cpplint-cache')
+    good_files_cache.Load()
+    files = good_files_cache.FilterUnchangedFiles(files)
+    if len(files) == 0:
+      print 'No changes in files detected. Skipping cpplint check.'
+      return True
+
     filt = '-,' + ",".join(['+' + n for n in ENABLED_LINT_RULES])
     command = ['cpplint.py', '--filter', filt] + join(files)
     local_cpplint = join(path, "tools", "cpplint.py")
     if exists(local_cpplint):
       command = ['python', local_cpplint, '--filter', filt] + join(files)
-    process = subprocess.Popen(command)
-    return process.wait() == 0
+
+    process = subprocess.Popen(command, stderr=subprocess.PIPE)
+    LINT_ERROR_PATTERN = re.compile(r'^(.+)[:(]\d+[:)]')
+    while True:
+      out_line = process.stderr.readline()
+      if out_line == '' and process.poll() != None:
+        break
+      sys.stderr.write(out_line)
+      m = LINT_ERROR_PATTERN.match(out_line)
+      if m:
+        good_files_cache.RemoveFile(m.group(1))
+
+    good_files_cache.Save()
+    return process.returncode == 0
 
 
 COPYRIGHT_HEADER_PATTERN = re.compile(
     r'Copyright [\d-]*200[8-9] the V8 project authors. All rights reserved.')
 
 class SourceProcessor(SourceFileProcessor):
   """
   Check that all files include a copyright notice.
   """
 
@@ skipping 59 matching lines @@
     success = CppLintProcessor().Run(workspace) and success
   success = SourceProcessor().Run(workspace) and success
   if success:
     return 0
   else:
     return 1
 
 
 if __name__ == '__main__':
   sys.exit(Main())