Allow 'git cache fetch' to re-bootstrap.

git_cache.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """A git command for managing a local cache of git repositories."""
 
 from __future__ import print_function
 import errno
 import logging
@@ skipping 127 matching lines @@
     return self.is_locked() and self.pid == self._read_pid()
 
 
 class Mirror(object):
 
   git_exe = 'git.bat' if sys.platform.startswith('win') else 'git'
   gsutil_exe = os.path.join(
     os.path.dirname(os.path.abspath(__file__)), 'gsutil.py')
   cachepath_lock = threading.Lock()
 
+  @staticmethod
+  def parse_fetch_spec(spec):
+    """Parses and canonicalizes a fetch spec.
+
+    Returns (fetchspec, value_regex), where value_regex can be used
+    with 'git config --replace-all'.
+    """
+    parts = spec.split(':', 1)
+    src = parts[0].lstrip('+').rstrip('/')
+    if not src.startswith('refs/'):
+      src = 'refs/heads/%s' % src
+    dest = parts[1].rstrip('/') if len(parts) > 1 else src
+    regex = r'\+%s:.*' % src.replace('*', r'\*')
+    return ('+%s:%s' % (src, dest), regex)
+
   def __init__(self, url, refs=None, print_func=None):
     self.url = url
-    self.refs = refs or []
+    self.fetch_specs = set([self.parse_fetch_spec(ref) for ref in (refs or [])])
     self.basedir = self.UrlToCacheDir(url)
     self.mirror_path = os.path.join(self.GetCachePath(), self.basedir)
     if print_func:
       self.print = self.print_without_file
       self.print_func = print_func
     else:
       self.print = print
 
   def print_without_file(self, message, **kwargs):
     self.print_func(message)
@@ skipping 68 matching lines @@
     self.RunGit(['config', 'gc.autopacklimit', '0'], cwd=cwd)
 
     # Allocate more RAM for cache-ing delta chains, for better performance
     # of "Resolving deltas".
     self.RunGit(['config', 'core.deltaBaseCacheLimit',
                  gclient_utils.DefaultDeltaBaseCacheLimit()], cwd=cwd)
 
     self.RunGit(['config', 'remote.origin.url', self.url], cwd=cwd)
     self.RunGit(['config', '--replace-all', 'remote.origin.fetch',
                  '+refs/heads/*:refs/heads/*', r'\+refs/heads/\*:.*'], cwd=cwd)
-    for ref in self.refs:
-      ref = ref.lstrip('+').rstrip('/')
-      if ref.startswith('refs/'):
-        refspec = '+%s:%s' % (ref, ref)
-        regex = r'\+%s:.*' % ref.replace('*', r'\*')
-      else:
-        refspec = '+refs/%s/*:refs/%s/*' % (ref, ref)
-        regex = r'\+refs/heads/%s:.*' % ref.replace('*', r'\*')
+    for spec, value_regex in self.fetch_specs:
       self.RunGit(
-          ['config', '--replace-all', 'remote.origin.fetch', refspec, regex],
+          ['config', '--replace-all', 'remote.origin.fetch', spec, value_regex],
           cwd=cwd)
 
   def bootstrap_repo(self, directory):
     """Bootstrap the repo from Google Stroage if possible.
 
     More apt-ly named bootstrap_repo_from_cloud_if_possible_else_do_nothing().
     """
 
     python_fallback = False
     if (sys.platform.startswith('win') and
@@ skipping 48 matching lines @@
     if retcode:
       self.print(
           'Extracting bootstrap zipfile %s failed.\n'
           'Resuming normal operations.' % filename)
       return False
     return True
 
   def exists(self):
     return os.path.isfile(os.path.join(self.mirror_path, 'config'))
 
+  def _preserve_fetchspec(self):
+    """Read and preserve remote.origin.fetch from an existing mirror."""

[iannucci, 2015/09/22 22:56:57]

I'd mention that this mutates self.fetch_specs as a side effect.

[szager1, 2015/09/22 23:11:56]

Done.

+    if not self.exists():
+      return
+    try:
+      config_fetchspecs = subprocess.check_output(
+          [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'],
+          cwd=self.mirror_path)
+      for fetchspec in config_fetchspecs.splitlines():
+        self.fetch_specs.add(self.parse_fetch_spec(fetchspec))
+    except subprocess.CalledProcessError:
+      pass

[iannucci, 2015/09/22 22:56:57]

This /seems/ like it could cause data loss on an exceptional condition (e.g.
where this fails, so never reads into self.fetch_specs, but the rest of the code
completes). What about the case where the user has 1 malformed fetch spec
followed by some good ones?

[szager1, 2015/09/22 23:11:56]

Well, "data loss" has an interesting meaning here.  In a sense, there's always
100% data loss, because this code path only happens when we intend to blow away
an existing cache and repopulate it.  The only real "data loss" is the fetch
specs in the git config file.

... which could be problematic, but at this point, the only use case for this
code path is a developer running 'git cache fetch' after having manually edited
the config file in the old cache.  Which is to say, it could hamper a developer
in a non-permanent, recoverable way, but it should never affect the slaves
because they should never have manual changes in their config's.

I added a logging.warn for this case.

+
   def _ensure_bootstrapped(self, depth, bootstrap, force=False):
     tempdir = None
-    config_file = os.path.join(self.mirror_path, 'config')
     pack_dir = os.path.join(self.mirror_path, 'objects', 'pack')
     pack_files = []
 
     if os.path.isdir(pack_dir):
       pack_files = [f for f in os.listdir(pack_dir) if f.endswith('.pack')]
 
     should_bootstrap = (force or
-                        not os.path.exists(config_file) or
+                        not self.exists() or
                         len(pack_files) > GC_AUTOPACKLIMIT)
     if should_bootstrap:
+      if self.exists():
+        # Re-bootstrapping an existing mirror; preserve existing fetch spec.
+        self._preserve_fetchspec()
       tempdir = tempfile.mkdtemp(
           prefix='_cache_tmp', suffix=self.basedir, dir=self.GetCachePath())
       bootstrapped = not depth and bootstrap and self.bootstrap_repo(tempdir)
       if bootstrapped:
         # Bootstrap succeeded; delete previous cache, if any.
         gclient_utils.rmtree(self.mirror_path)
-      elif not os.path.exists(config_file):
+      elif not self.exists():
         # Bootstrap failed, no previous cache; start with a bare git dir.
         self.RunGit(['init', '--bare'], cwd=tempdir)
       else:
         # Bootstrap failed, previous cache exists; warn and continue.
         logging.warn(
             'Git cache has a lot of pack files (%d).  Tried to re-bootstrap '
             'but failed.  Continuing with non-optimized repository.'
             % len(pack_files))
         gclient_utils.rmtree(tempdir)
         tempdir = None
@@ skipping 209 matching lines @@
   }
   if options.depth:
     kwargs['depth'] = options.depth
   mirror.populate(**kwargs)
 
 
 @subcommand.usage('Fetch new commits into cache and current checkout')
 def CMDfetch(parser, args):
   """Update mirror, and fetch in cwd."""
   parser.add_option('--all', action='store_true', help='Fetch all remotes')
+  parser.add_option('--no_bootstrap', '--no-bootstrap',
+                    action='store_true',
+                    help='Don\'t (re)bootstrap from Google Storage')
   options, args = parser.parse_args(args)
 
   # Figure out which remotes to fetch.  This mimics the behavior of regular
   # 'git fetch'.  Note that in the case of "stacked" or "pipelined" branches,
   # this will NOT try to traverse up the branching structure to find the
   # ultimate remote to update.
   remotes = []
   if options.all:
     assert not args, 'fatal: fetch --all does not take a repository argument'
     remotes = subprocess.check_output([Mirror.git_exe, 'remote']).splitlines()
@@ skipping 10 matching lines @@
         remotes = [upstream]
   if not remotes:
     remotes = ['origin']
 
   cachepath = Mirror.GetCachePath()
   git_dir = os.path.abspath(subprocess.check_output(
       [Mirror.git_exe, 'rev-parse', '--git-dir']))
   git_dir = os.path.abspath(git_dir)
   if git_dir.startswith(cachepath):
     mirror = Mirror.FromPath(git_dir)
-    mirror.populate()
+    mirror.populate(bootstrap=not options.no_bootstrap)
     return 0
   for remote in remotes:
     remote_url = subprocess.check_output(
         [Mirror.git_exe, 'config', 'remote.%s.url' % remote]).strip()
     if remote_url.startswith(cachepath):
       mirror = Mirror.FromPath(remote_url)
       mirror.print = lambda *args: None
       print('Updating git cache...')
-      mirror.populate()
+      mirror.populate(bootstrap=not options.no_bootstrap)
     subprocess.check_call([Mirror.git_exe, 'fetch', remote])
   return 0
 
 
 @subcommand.usage('[url of repo to unlock, or -a|--all]')
 def CMDunlock(parser, args):
   """Unlock one or all repos if their lock files are still around."""
   parser.add_option('--force', '-f', action='store_true',
                     help='Actually perform the action')
   parser.add_option('--all', '-a', action='store_true',
@@ skipping 62 matching lines @@
   dispatcher = subcommand.CommandDispatcher(__name__)
   return dispatcher.execute(OptionParser(), argv)
 
 
 if __name__ == '__main__':
   try:
     sys.exit(main(sys.argv[1:]))
   except KeyboardInterrupt:
     sys.stderr.write('interrupted\n')
     sys.exit(1)