Refactor cache-updating code to eliminate unnecessary fetches.

gclient_scm.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Gclient-specific SCM-specific operations."""
 
 from __future__ import print_function
 
 import errno
 import logging
@@ skipping 338 matching lines @@
     rev_str = ' at %s' % revision
     files = [] if file_list is not None else None
 
     printed_path = False
     verbose = []
     if options.verbose:
       self.Print('_____ %s%s' % (self.relpath, rev_str), timestamp=False)
       verbose = ['--verbose']
       printed_path = True
 
-    url = self._CreateOrUpdateCache(url, options)
-
     if revision.startswith('refs/'):
       rev_type = "branch"
     elif revision.startswith(self.remote + '/'):
       # Rewrite remote refs to their local equivalents.
       revision = 'refs/remotes/' + revision
       rev_type = "branch"
     else:
       # hash is also a tag, only make a distinction at checkout
       rev_type = "hash"
 
+    mirror = self._GetMirror(url, options)
+    if mirror:
+      url = mirror.mirror_path
+
     if (not os.path.exists(self.checkout_path) or
         (os.path.isdir(self.checkout_path) and
          not os.path.exists(os.path.join(self.checkout_path, '.git')))):
+      if mirror:
+        self._UpdateMirror(mirror, options)
       try:
         self._Clone(revision, url, options)
       except subprocess2.CalledProcessError:
         self._DeleteOrMove(options.force)
         self._Clone(revision, url, options)
       if deps_revision and deps_revision.startswith('branch-heads/'):
         deps_branch = deps_revision.replace('branch-heads/', '')
         self._Capture(['branch', deps_branch, deps_revision])
         self._Capture(['checkout', '--quiet', deps_branch])
       if file_list is not None:
         files = self._Capture(['ls-files']).splitlines()
         file_list.extend([os.path.join(self.checkout_path, f) for f in files])
       if not verbose:
         # Make the output a little prettier. It's nice to have some whitespace
         # between projects when cloning.
         self.Print('')
       return self._Capture(['rev-parse', '--verify', 'HEAD'])
 
     if not managed:
       self._UpdateBranchHeads(options, fetch=False)
       self.Print('________ unmanaged solution; skipping %s' % self.relpath)
       return self._Capture(['rev-parse', '--verify', 'HEAD'])
 
+    if mirror:
+      self._UpdateMirror(mirror, options)
+
     # See if the url has changed (the unittests use git://foo for the url, let
     # that through).
     current_url = self._Capture(['config', 'remote.%s.url' % self.remote])
     return_early = False
     # TODO(maruel): Delete url != 'git://foo' since it's just to make the
     # unit test pass. (and update the comment above)
     # Skip url auto-correction if remote.origin.gclient-auto-fix-url is set.
     # This allows devs to use experimental repos which have a different url
     # but whose branch(s) are the same as official repos.
     if (current_url.rstrip('/') != url.rstrip('/') and
@@ skipping 331 matching lines @@
             '#Initial_checkout' ) % rev)
 
     return sha1
 
   def FullUrlForRelativeUrl(self, url):
     # Strip from last '/'
     # Equivalent to unix basename
     base_url = self.url
     return base_url[:base_url.rfind('/')] + url
 
-  def _CreateOrUpdateCache(self, url, options):
-    """Make a new git mirror or update existing mirror for |url|, and return the
-    mirror URI to clone from.
-
-    If no cache-dir is specified, just return |url| unchanged.
-    """
-    if not self.cache_dir:
-      return url
+  def _GetMirror(self, url, options):
+    """Get a git_cache.Mirror object for the argument url."""
+    if not git_cache.Mirror.GetCachePath():
+      return None
     mirror_kwargs = {
         'print_func': self.filter,
         'refs': []
     }
     # TODO(hinoka): This currently just fails because lkcr/lkgr are branches
     #               not tags. This also adds 20 seconds to every bot_update
     #               run, so I'm commenting this out until lkcr/lkgr become
     #               tags.  (2014/4/24)
     # if url == CHROMIUM_SRC_URL or url + '.git' == CHROMIUM_SRC_URL:
     #  mirror_kwargs['refs'].extend(['refs/tags/lkgr', 'refs/tags/lkcr'])
     if hasattr(options, 'with_branch_heads') and options.with_branch_heads:
       mirror_kwargs['refs'].append('refs/branch-heads/*')
-    mirror = git_cache.Mirror(url, **mirror_kwargs)
+    return git_cache.Mirror(url, **mirror_kwargs)
+
+  @staticmethod
+  def _UpdateMirror(mirror, options):
+    """Update a git mirror by fetching the latest commits from the remote."""
     if options.shallow:
       # HACK(hinoka): These repositories should be super shallow.

[iannucci, 2014/06/17 23:08:17]

oh, huge barf... this is why we can't have nice things... :(

-      if 'flash' in url:
+      if 'flash' in mirror.url:
         depth = 10
       else:
         depth = 10000
     else:
       depth = None
     mirror.populate(verbose=options.verbose, bootstrap=True, depth=depth)
     mirror.unlock()
-    return mirror.mirror_path if mirror.exists() else None
 
   def _Clone(self, revision, url, options):
     """Clone a git repository from the given URL.
 
     Once we've cloned the repo, we checkout a working branch if the specified
     revision is a branch head. If it is a tag or a specific commit, then we
     leave HEAD detached as it makes future updates simpler -- in this case the
     user should first create a new branch or switch to an existing branch before
     making changes in the repo."""
     if not options.verbose:
@@ skipping 693 matching lines @@
       new_command.append('--force')
       if command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
         new_command.extend(('--accept', 'theirs-conflict'))
     elif options.manually_grab_svn_rev:
       new_command.append('--force')
       if command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
         new_command.extend(('--accept', 'postpone'))
     elif command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
       new_command.extend(('--accept', 'postpone'))
     return new_command