[autoroll] cache commit_metadata operations.

recipe_engine/fetch.py

 # Copyright 2016 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 import base64
 import httplib
 import json
 import logging
 import os
+import re
 import shutil
 import sys
 import tarfile
 import tempfile
 
 # Add third party paths.
 from . import env
 from . import requests_ssl
 from . import util
+from . import types
 from .requests_ssl import requests
 
 import subprocess42
 from google.protobuf import text_format
 
 from . import package_pb2
 
 
 class FetchError(Exception):
   pass
@@ skipping 24 matching lines @@
   def updates(self, repo, revision, checkout_dir, allow_fetch,
               other_revision, paths):
     """Returns a list of revisions between |revision| and |other_revision|.
 
     Network operations are performed only if |allow_fetch| is True.
 
     If |paths| is a non-empty list, the history is scoped just to these paths.
     """
     raise NotImplementedError()
 
+  # This is a simple mapping of
+  #   repo -> git_revision -> commit_metadata()
+  # It only holds cache entries for git commits (e.g. sha1 hashes)
+  _GIT_METADATA_CACHE = {}
+
+  # This matches git commit hashes.
+  _COMMIT_RE = re.compile('^[a-fA-F0-9]{40}$')
+
   def commit_metadata(self, repo, revision, checkout_dir, allow_fetch):
+    """Cached version of commit_metadata_slow. Implementations should only
+    override commit_metadata_slow and this implementation will call it if
+    there's no cached value for (repo, revision).
+    """
+    if self._COMMIT_RE.match(revision):
+      cache = self._GIT_METADATA_CACHE.setdefault(repo, {})
+      if revision not in cache:
+        cache[revision] = types.freeze(self.commit_metadata_slow(
+          repo, revision, checkout_dir, allow_fetch))
+      return cache[revision]
+    return self.commit_metadata_slow(repo, revision, checkout_dir, allow_fetch)
+
+  def commit_metadata_slow(self, repo, revision, checkout_dir, allow_fetch):
     """Returns a dictionary of metadata about commit |revision|.
 
     The dictionary contains the following keys: author, message.
+
+    This implementation will likely do slow operations (e.g. subprocess
+    invocations, network access, etc.). Users of Backend will always want to use
+    commit_metadata() instead.
     """
     raise NotImplementedError()
 
 
 class UncleanFilesystemError(FetchError):
   pass
 
 
 class GitError(FetchError):
 
@@ skipping 91 matching lines @@
 
     args = [
         'rev-list',
         '--reverse',
         '%s..%s' % (revision, other_revision),
     ]
     if paths:
       args.extend(['--'] + paths)
     return filter(bool, git(*args).strip().split('\n'))
 
-  def commit_metadata(self, repo, revision, checkout_dir, allow_fetch):
+  def commit_metadata_slow(self, repo, revision, checkout_dir, allow_fetch):
     git = self.Git(checkout_dir=checkout_dir)
     return {
       'author': git('show', '-s', '--pretty=%aE', revision).strip(),
       'message': git('show', '-s', '--pretty=%B', revision).strip(),
     }
 
 
 class GitilesFetchError(FetchError):
   """An HTTP error that occurred during Gitiles fetching."""
 
@@ skipping 94 matching lines @@
               diff_entry['new_path'].startswith(path)):
             matched = True
             break
         if matched:
           break
       if matched or not paths:
         results.append(entry['commit'])
 
     return list(reversed(results))
 
-  def commit_metadata(self, repo, revision, checkout_dir, allow_fetch):
+  def commit_metadata_slow(self, repo, revision, checkout_dir, allow_fetch):
     if not allow_fetch:
       raise FetchNotAllowedError(
           ('requested commit metadata for %s (%s)from gitiles but fetch not '
            'allowed') % (repo, revision))
     rev_json = self._revision_metadata(repo, revision)
     return {
       'author': rev_json['author']['email'],
       'message': rev_json['message'],
     }
 
@@ skipping 27 matching lines @@
     logging.info('fetching %s', url)
 
     resp = requests.get(url)
     if resp.status_code != httplib.OK:
       raise GitilesFetchError(resp.status_code, resp.text)
 
     if not resp.text.startswith(cls._GERRIT_XSRF_HEADER):
       raise GitilesFetchError(resp.status_code, 'Missing XSRF header')
 
     return json.loads(resp.text[len(cls._GERRIT_XSRF_HEADER):])