Add a git cache for gclient sync operations.

gclient_scm.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Gclient-specific SCM-specific operations."""
 
 import logging
 import os
 import posixpath
 import re
@@ skipping 134 matching lines @@
     if not command in commands:
       raise gclient_utils.Error('Unknown command %s' % command)
 
     if not command in dir(self):
       raise gclient_utils.Error('Command %s not implemented in %s wrapper' % (
           command, self.__class__.__name__))
 
     return getattr(self, command)(options, args, file_list)
 
 
+class GitFilter(object):
+  PERCENT_RE = re.compile('.* ([0-9]{1,2})% .*')
+
+  def __init__(self, predicate=None):
+    """Create a new GitFilter.
+
+    Args:
+      predicate (f(line)): An optional function which is invoked for every line.
+        The line will be skipped if the predicate is False.
+    """
+    self.last_time = 0
+    self.predicate = predicate
+
+  def __call__(self, line):
+    # git uses an escape sequence to clear the line; elide it.
+    esc = line.find(unichr(033))
+    if esc > -1:
+      line = line[:esc]
+    if self.predicate and not self.predicate(line):
+      return
+    now = time.time()
+    match = self.PERCENT_RE.match(line)
+    if not match:
+      self.last_time = 0
+    if (now - self.last_time) >= 2:

[szager1, 2013/07/01 23:38:29]

Maybe parameterize the suppression interval?  Two seconds may be a good interval
for interactive use; for bots, we might want to make it longer.

[iannucci, 2013/07/01 23:55:52]

I was making it shorter explicitly FOR the bots. Maybe nag_timer / 2?

[szager1, 2013/07/02 00:08:05]

That would be fine, or even something like 10; but two seconds will cause an
awful lot of log file gas for clobber checkouts.

[iannucci, 2013/07/02 00:22:27]

Ok, I set it to nag_timer / 2 (which should be 15 seconds in the default case).

+      self.last_time = now
+      print line
+
+
 class GitWrapper(SCMWrapper):
   """Wrapper for Git"""
 
   def __init__(self, url=None, root_dir=None, relpath=None):
     """Removes 'git+' fake prefix from git URL."""
     if url.startswith('git+http://') or url.startswith('git+https://'):
       url = url[4:]
     SCMWrapper.__init__(self, url, root_dir, relpath)
 
   @staticmethod
@@ skipping 125 matching lines @@
     rev_str = ' at %s' % revision
     files = []
 
     printed_path = False
     verbose = []
     if options.verbose:
       print('\n_____ %s%s' % (self.relpath, rev_str))
       verbose = ['--verbose']
       printed_path = True
 
+    url = self._CreateOrUpdateCache(url, options)
+
     if revision.startswith('refs/heads/'):
       rev_type = "branch"
     elif revision.startswith('origin/'):
       # For compatability with old naming, translate 'origin' to 'refs/heads'
       revision = revision.replace('origin/', 'refs/heads/')
       rev_type = "branch"
     else:
       # hash is also a tag, only make a distinction at checkout
       rev_type = "hash"
 
@@ skipping 20 matching lines @@
     if not os.path.exists(os.path.join(self.checkout_path, '.git')):
       raise gclient_utils.Error('\n____ %s%s\n'
                                 '\tPath is not a git repo. No .git dir.\n'
                                 '\tTo resolve:\n'
                                 '\t\trm -rf %s\n'
                                 '\tAnd run gclient sync again\n'
                                 % (self.relpath, rev_str, self.relpath))
 
     # See if the url has changed (the unittests use git://foo for the url, let
     # that through).
     current_url = self._Capture(['config', 'remote.origin.url'])

[szager1, 2013/07/01 23:38:29]

Maybe do something here to help people who are switching from non-cached to
cached, or vice-versa?

[iannucci, 2013/07/01 23:55:52]

I think that should all work as intended, since the remote url is just
different, so all the normal replacement logic should work as before, right?

[szager1, 2013/07/02 00:08:05]

Actually, what I meant was that when switching from non-cached to cached, it
would be nice if the existing checkout could be used to create the cache, rather
than re-cloning from scratch.

[iannucci, 2013/07/02 00:22:27]

Ah! Yeah we could do that with --reference, I think. Then we could repack -a
once the mirror was done. Good idea. I'll do this in a followup cl :)

     # TODO(maruel): Delete url != 'git://foo' since it's just to make the
     # unit test pass. (and update the comment above)
     # Skip url auto-correction if remote.origin.gclient-auto-fix-url is set.
     # This allows devs to use experimental repos which have a different url
     # but whose branch(s) are the same as official repos.
     if (current_url != url and
         url != 'git://foo' and
         subprocess2.capture(
             ['git', 'config', 'remote.origin.gclient-auto-fix-url'],
             cwd=self.checkout_path).strip() != 'False'):
@@ skipping 333 matching lines @@
             '#Initial_checkout' ) % rev)
 
     return sha1
 
   def FullUrlForRelativeUrl(self, url):
     # Strip from last '/'
     # Equivalent to unix basename
     base_url = self.url
     return base_url[:base_url.rfind('/')] + url
 
+  @staticmethod
+  def _CacheFolder(url, options):
+    """Transforms a url into the path to the corresponding git cache.
+
+    Ex. (assuming cache_dir == '/cache')
+      IN:  https://chromium.googlesource.com/chromium/src
+      OUT: /cache/chromium.googlesource.com-chromium-src.git

[szager1, 2013/07/01 23:38:29]

Any reason not to mimic the full directory structure?  Admittedly it's unlikely,
but this would be a naming conflict:

https://chromium.googlesource.com/foo/bar.git
https://chromium.googlesource.com/foo-bar.git

[iannucci, 2013/07/01 23:55:52]

Eh... I'm much more inclined to flatten it out, since it makes observing the
caches easier (esp. on windows). I could replace '-' with '--' to deal with this
conflict though.

[szager1, 2013/07/02 00:08:05]

I'd be satisfied with collision detection logic.

[iannucci, 2013/07/02 00:22:27]

Done.

+    """
+    idx = url.find('://')
+    if idx != -1:
+      url = url[idx+3:]
+    url = url.replace('/', '-')
+    if not url.endswith('.git'):
+      url += '.git'
+    return os.path.join(options.cache_dir, url)
+
+  def _CreateOrUpdateCache(self, url, options):
+    """Make a new git mirror or update existing mirror for |url|, and return the
+    mirror URI to clone from.
+
+    If no cache-dir is specified, just return |url| unchanged.
+    """
+    if not options.cache_dir:
+      return url
+    folder = self._CacheFolder(url, options)
+    v = ['-v'] if options.verbose else []
+    filter_fn = lambda l: '[up to date]' not in l
+    with options.cache_lock:
+      lock = options.cache_locks[folder]
+    with lock:
+      gclient_utils.safe_makedirs(options.cache_dir)
+      if not os.path.exists(os.path.join(folder, 'config')):
+        gclient_utils.rmtree(folder)
+        self._Run(['clone'] + v + ['-c', 'core.deltaBaseCacheLimit=2g',
+                                   '--progress', '--mirror', url, folder],
+                  options, git_filter=True, filter_fn=filter_fn,
+                  cwd=options.cache_dir)
+      else:
+        # Would normally use `git remote update`, but it doesn't support
+        # --progress, so use fetch instead.
+        self._Run(['fetch'] + v + ['--multiple', '--progress', '--all'],
+                  options, git_filter=True, filter_fn=filter_fn, cwd=folder)
+    return folder
+
   def _Clone(self, revision, url, options):
     """Clone a git repository from the given URL.
 
     Once we've cloned the repo, we checkout a working branch if the specified
     revision is a branch head. If it is a tag or a specific commit, then we
     leave HEAD detached as it makes future updates simpler -- in this case the
     user should first create a new branch or switch to an existing branch before
     making changes in the repo."""
     if not options.verbose:
       # git clone doesn't seem to insert a newline properly before printing
       # to stdout
       print('')
     clone_cmd = ['-c', 'core.deltaBaseCacheLimit=2g', 'clone', '--progress']
+    if options.cache_dir:
+      clone_cmd.append('--shared')

[szager1, 2013/07/01 23:38:29]

Hmm... not sure I agree with the use of --shared.  For our use case, it seems to
add risk without providing benefit.

Here's the best explanation I can find online about the difference between
--local (which is implied when cloning from a file path) and --shared:

http://lists-archives.com/git/494327-clarify-git-clone-local-shared-reference...

[iannucci, 2013/07/01 23:55:52]

--local won't work correctly on windows, or cross-device (relies on hardlinks).
I explicitly wanted shared because I want there to only be one copy of the
object per-machine.

The normal pitfalls of --shared won't apply here, since we know that the git
repos in cache are full mirrors of the remote. Any issues there (i.e. force push
of checked out refs) would be exactly the same amount of pain as cloning
directly from the remote, and the remedy is the same (clobber the non-cache
checkout).

[szager1, 2013/07/02 00:08:05]

Fair enough!

     if revision.startswith('refs/heads/'):
       clone_cmd.extend(['-b', revision.replace('refs/heads/', '')])
       detach_head = False
     else:
       detach_head = True
     if options.verbose:
       clone_cmd.append('--verbose')
     clone_cmd.extend([url, self.checkout_path])
 
     # If the parent directory does not exist, Git clone on Windows will not
     # create it, so we need to do it manually.
     parent_dir = os.path.dirname(self.checkout_path)
     if not os.path.exists(parent_dir):
       gclient_utils.safe_makedirs(parent_dir)
 
-    percent_re = re.compile('.* ([0-9]{1,2})% .*')
-    def _GitFilter(line):
-      # git uses an escape sequence to clear the line; elide it.
-      esc = line.find(unichr(033))
-      if esc > -1:
-        line = line[:esc]
-      match = percent_re.match(line)
-      if not match or not int(match.group(1)) % 10:
-        print '%s' % line
-
     for _ in range(3):
       try:
-        self._Run(clone_cmd, options, cwd=self._root_dir, filter_fn=_GitFilter,
-                  print_stdout=False)
+        self._Run(clone_cmd, options, cwd=self._root_dir, git_filter=True)
         break
       except subprocess2.CalledProcessError, e:
         # Too bad we don't have access to the actual output yet.
         # We should check for "transfer closed with NNN bytes remaining to
         # read". In the meantime, just make sure .git exists.
         if (e.returncode == 128 and
             os.path.exists(os.path.join(self.checkout_path, '.git'))):
           print(str(e))
           print('Retrying...')
           continue
@@ skipping 194 matching lines @@
             if options.verbose:
               fetch_cmd.append('--verbose')
             self._Run(fetch_cmd, options)
           break
         except subprocess2.CalledProcessError, e:
           print(str(e))
           print('Retrying in %.1f seconds...' % backoff_time)
           time.sleep(backoff_time)
           backoff_time *= 1.3
 
-  def _Run(self, args, options, **kwargs):
+  def _Run(self, args, _options, git_filter=False, **kwargs):
+    if git_filter:
+      kwargs['filter_fn'] = GitFilter(kwargs.get('filter_fn'))
+      kwargs.setdefault('print_stdout', False)
     kwargs.setdefault('cwd', self.checkout_path)
     kwargs.setdefault('print_stdout', True)
     kwargs.setdefault('nag_timer', self.nag_timer)
     kwargs.setdefault('nag_max', self.nag_max)
     stdout = kwargs.get('stdout', sys.stdout)
     stdout.write('\n________ running \'git %s\' in \'%s\'\n' % (
                  ' '.join(args), kwargs['cwd']))
     gclient_utils.CheckCallAndFilter(['git'] + args, **kwargs)
 
 
@@ skipping 379 matching lines @@
       new_command.append('--force')
       if command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
         new_command.extend(('--accept', 'theirs-conflict'))
     elif options.manually_grab_svn_rev:
       new_command.append('--force')
       if command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
         new_command.extend(('--accept', 'postpone'))
     elif command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
       new_command.extend(('--accept', 'postpone'))
     return new_command