Tweak git gc settings, and periodically re-bootstrap.

git_cache.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """A git command for managing a local cache of git repositories."""
 
 from __future__ import print_function
 import errno
 import logging
 import optparse
 import os
 import re
 import tempfile
 import time
 import subprocess
 import sys
 import urlparse
 import zipfile
 
 from download_from_google_storage import Gsutil
 import gclient_utils
 import subcommand
 
+# Analogous to gc.autopacklimit git config.
+GC_AUTOPACKLIMIT = 50
+
 try:
   # pylint: disable=E0602
   WinErr = WindowsError
 except NameError:
   class WinErr(Exception):
     pass
 
 class LockError(Exception):
   pass
 
@@ skipping 184 matching lines @@
     kwargs.setdefault('filter_fn', self.print)
     env = kwargs.get('env') or kwargs.setdefault('env', os.environ.copy())
     env.setdefault('GIT_ASKPASS', 'true')
     env.setdefault('SSH_ASKPASS', 'true')
     self.print('running "git %s" in "%s"' % (' '.join(cmd), cwd))
     gclient_utils.CheckCallAndFilter([self.git_exe] + cmd, **kwargs)
 
   def config(self, cwd=None):
     if cwd is None:
       cwd = self.mirror_path
+
+    # Don't run git-gc in a daemon.  Bad things can happen if it gets killed.
+    self.RunGit(['config', 'gc.autodetach', '0'], cwd=cwd)
+
+    # Don't combine pack files into one big pack file.  It's really slow for
+    # repositories, and there's no way to track progress and make sure it's
+    # not stuck.
+    self.RunGit(['config', 'gc.autopacklimit', '0'], cwd=cwd)
+
+    # Allocate more RAM for cache-ing delta chains, for better performance
+    # of "Resolving deltas".
     self.RunGit(['config', 'core.deltaBaseCacheLimit',
                  gclient_utils.DefaultDeltaBaseCacheLimit()], cwd=cwd)
+
     self.RunGit(['config', 'remote.origin.url', self.url], cwd=cwd)
     self.RunGit(['config', '--replace-all', 'remote.origin.fetch',
                  '+refs/heads/*:refs/heads/*'], cwd=cwd)
     for ref in self.refs:
       ref = ref.lstrip('+').rstrip('/')
       if ref.startswith('refs/'):
         refspec = '+%s:%s' % (ref, ref)
       else:
         refspec = '+refs/%s/*:refs/%s/*' % (ref, ref)
       self.RunGit(['config', '--add', 'remote.origin.fetch', refspec], cwd=cwd)
 
   def bootstrap_repo(self, directory):
     """Bootstrap the repo from Google Stroage if possible."""
 
     python_fallback = False
     if sys.platform.startswith('win') and not self.FindExecutable('7z'):
       python_fallback = True
     elif sys.platform.startswith('darwin'):
       # The OSX version of unzip doesn't support zip64.
       python_fallback = True
     elif not self.FindExecutable('unzip'):
       python_fallback = True
 
     gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
     gsutil = Gsutil(self.gsutil_exe, boto_path=None, bypass_prodaccess=True)
     # Get the most recent version of the zipfile.
     _, ls_out, _ = gsutil.check_call('ls', gs_folder)
-    ls_out_sorted = sorted(ls_out.splitlines())
+    ls_out_sorted = sorted(ls_out.splitlines(),
+                           key=lambda x: x.split('/')[-1])

[Ryan Tseng, 2014/06/13 23:04:53]

What was broken?

[szager1, 2014/06/16 04:46:26]

Whoops, nothing was broken.  I started thinking about how to handle this if the
base names use git hashes instead of svn revisions, but I didn't finish it.  I'm
going to revert this snippet, we can address the sorting issue separately.

[Ryan Tseng, 2014/06/16 17:28:22]

Ah. fwiw the bootstrap zip file uses the git number, not the svn revision
number.  So by design this relies on the filename not being a git hash.

     if not ls_out_sorted:
       # This repo is not on Google Storage.
       return False
     latest_checkout = ls_out_sorted[-1]
 
     # Download zip file to a temporary directory.
     try:
       tempdir = tempfile.mkdtemp()
       self.print('Downloading %s' % latest_checkout)
       code = gsutil.call('cp', latest_checkout, tempdir)
@@ skipping 43 matching lines @@
       v = ['-v', '--progress']
 
     d = []
     if depth:
       d = ['--depth', str(depth)]
 
 
     with Lockfile(self.mirror_path):
       # Setup from scratch if the repo is new or is in a bad state.
       tempdir = None
-      if not os.path.exists(os.path.join(self.mirror_path, 'config')):
-        gclient_utils.rmtree(self.mirror_path)
+      config_file = os.path.join(self.mirror_path, 'config')
+      pack_dir = os.path.join(self.mirror_path, 'objects', 'pack')
+      pack_files = []
+      if os.path.isdir(pack_dir):
+        pack_files = [f for f in os.listdir(pack_dir) if f.endswith('.pack')]
+
+      should_bootstrap = (not os.path.exists(config_file) or
+                          len(pack_files) > GC_AUTOPACKLIMIT)
+      if should_bootstrap:
         tempdir = tempfile.mkdtemp(
             suffix=self.basedir, dir=self.GetCachePath())
         bootstrapped = not depth and bootstrap and self.bootstrap_repo(tempdir)
-        if not bootstrapped:
+        if bootstrapped:
+          # Bootstrap succeeded; delete previous cache, if any.
+          gclient_utils.rmtree(self.mirror_path)
+        elif not os.path.exists(config_file):
+          # Bootstrap failed, no previous cache; start with a bare git dir.
           self.RunGit(['init', '--bare'], cwd=tempdir)
+        else:
+          # Bootstrap failed, previous cache exists; warn and continue.
+          logging.warn(
+              'Git cache has a lot of pack files (%d).  Tried to re-bootstrap '
+              'but failed.  Continuing with non-optimized repository.'
+              % len(pack_files))
+          gclient_utils.rmtree(tempdir)
+          tempdir = None
       else:
         if depth and os.path.exists(os.path.join(self.mirror_path, 'shallow')):
           logging.warn(
               'Shallow fetch requested, but repo cache already exists.')
         d = []
 
       rundir = tempdir or self.mirror_path
       self.config(rundir)
       fetch_cmd = ['fetch'] + v + d + ['origin']
       fetch_specs = subprocess.check_output(
@@ skipping 200 matching lines @@
     return options, args
 
 
 def main(argv):
   dispatcher = subcommand.CommandDispatcher(__name__)
   return dispatcher.execute(OptionParser(), argv)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv[1:]))