Allow 'git cache fetch' to re-bootstrap.

git_cache.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """A git command for managing a local cache of git repositories."""
 
 from __future__ import print_function
 import errno
 import logging
@@ skipping 129 matching lines @@
 
 class Mirror(object):
 
   git_exe = 'git.bat' if sys.platform.startswith('win') else 'git'
   gsutil_exe = os.path.join(
     os.path.dirname(os.path.abspath(__file__)), 'gsutil.py')
   cachepath_lock = threading.Lock()
 
   def __init__(self, url, refs=None, print_func=None):
     self.url = url
-    self.refs = refs or []
+    self.refs = set([ref.lstrip('+').rstrip('/') for ref in (refs or [])])

[iannucci, 2015/09/22 20:35:26]

can we have a 'parse_fetch_spec' function so the next person knows what's
happening here?

[szager1, 2015/09/22 22:48:27]

Done.

     self.basedir = self.UrlToCacheDir(url)
     self.mirror_path = os.path.join(self.GetCachePath(), self.basedir)
     if print_func:
       self.print = self.print_without_file
       self.print_func = print_func
     else:
       self.print = print
 
   def print_without_file(self, message, **kwargs):
     self.print_func(message)
@@ skipping 69 matching lines @@
 
     # Allocate more RAM for cache-ing delta chains, for better performance
     # of "Resolving deltas".
     self.RunGit(['config', 'core.deltaBaseCacheLimit',
                  gclient_utils.DefaultDeltaBaseCacheLimit()], cwd=cwd)
 
     self.RunGit(['config', 'remote.origin.url', self.url], cwd=cwd)
     self.RunGit(['config', '--replace-all', 'remote.origin.fetch',
                  '+refs/heads/*:refs/heads/*', r'\+refs/heads/\*:.*'], cwd=cwd)
     for ref in self.refs:
-      ref = ref.lstrip('+').rstrip('/')
       if ref.startswith('refs/'):
         refspec = '+%s:%s' % (ref, ref)
         regex = r'\+%s:.*' % ref.replace('*', r'\*')
       else:
         refspec = '+refs/%s/*:refs/%s/*' % (ref, ref)
         regex = r'\+refs/heads/%s:.*' % ref.replace('*', r'\*')
       self.RunGit(
           ['config', '--replace-all', 'remote.origin.fetch', refspec, regex],
           cwd=cwd)
 
@@ skipping 56 matching lines @@
     if retcode:
       self.print(
           'Extracting bootstrap zipfile %s failed.\n'
           'Resuming normal operations.' % filename)
       return False
     return True
 
   def exists(self):
     return os.path.isfile(os.path.join(self.mirror_path, 'config'))
 
+  def _preserve_fetchspec(self):
+    """Read and preserve remote.origin.fetch from an existing mirror."""
+    if not self.exists():
+      return
+    try:
+      config_fetchspecs = subprocess.check_output(
+          [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'],
+          cwd=self.mirror_path)
+      for fetchspec in config_fetchspecs.splitlines():
+        self.refs.add(fetchspec.split(':')[0].lstrip('+').rstrip('/'))

[iannucci, 2015/09/22 20:35:26]

and here too

[szager1, 2015/09/22 22:48:27]

Done.

+    except subprocess.CalledProcessError:
+      pass
+
   def _ensure_bootstrapped(self, depth, bootstrap, force=False):
     tempdir = None
-    config_file = os.path.join(self.mirror_path, 'config')
     pack_dir = os.path.join(self.mirror_path, 'objects', 'pack')
     pack_files = []
 
     if os.path.isdir(pack_dir):
       pack_files = [f for f in os.listdir(pack_dir) if f.endswith('.pack')]
 
     should_bootstrap = (force or
-                        not os.path.exists(config_file) or
+                        not self.exists() or
                         len(pack_files) > GC_AUTOPACKLIMIT)
     if should_bootstrap:
+      if self.exists():
+        # Re-bootstrapping an existing mirror; preserve existing fetch spec.
+        self._preserve_fetchspec()
       tempdir = tempfile.mkdtemp(
           prefix='_cache_tmp', suffix=self.basedir, dir=self.GetCachePath())
       bootstrapped = not depth and bootstrap and self.bootstrap_repo(tempdir)
       if bootstrapped:
         # Bootstrap succeeded; delete previous cache, if any.
         gclient_utils.rmtree(self.mirror_path)
-      elif not os.path.exists(config_file):
+      elif not self.exists():
         # Bootstrap failed, no previous cache; start with a bare git dir.
         self.RunGit(['init', '--bare'], cwd=tempdir)
       else:
         # Bootstrap failed, previous cache exists; warn and continue.
         logging.warn(
             'Git cache has a lot of pack files (%d).  Tried to re-bootstrap '
             'but failed.  Continuing with non-optimized repository.'
             % len(pack_files))
         gclient_utils.rmtree(tempdir)
         tempdir = None
@@ skipping 209 matching lines @@
   }
   if options.depth:
     kwargs['depth'] = options.depth
   mirror.populate(**kwargs)
 
 
 @subcommand.usage('Fetch new commits into cache and current checkout')
 def CMDfetch(parser, args):
   """Update mirror, and fetch in cwd."""
   parser.add_option('--all', action='store_true', help='Fetch all remotes')
+  parser.add_option('--no_bootstrap', '--no-bootstrap',
+                    action='store_true',
+                    help='Don\'t (re)bootstrap from Google Storage')
   options, args = parser.parse_args(args)
 
   # Figure out which remotes to fetch.  This mimics the behavior of regular
   # 'git fetch'.  Note that in the case of "stacked" or "pipelined" branches,
   # this will NOT try to traverse up the branching structure to find the
   # ultimate remote to update.
   remotes = []
   if options.all:
     assert not args, 'fatal: fetch --all does not take a repository argument'
     remotes = subprocess.check_output([Mirror.git_exe, 'remote']).splitlines()
@@ skipping 10 matching lines @@
         remotes = [upstream]
   if not remotes:
     remotes = ['origin']
 
   cachepath = Mirror.GetCachePath()
   git_dir = os.path.abspath(subprocess.check_output(
       [Mirror.git_exe, 'rev-parse', '--git-dir']))
   git_dir = os.path.abspath(git_dir)
   if git_dir.startswith(cachepath):
     mirror = Mirror.FromPath(git_dir)
-    mirror.populate()
+    mirror.populate(bootstrap=not options.no_bootstrap)
     return 0
   for remote in remotes:
     remote_url = subprocess.check_output(
         [Mirror.git_exe, 'config', 'remote.%s.url' % remote]).strip()
     if remote_url.startswith(cachepath):
       mirror = Mirror.FromPath(remote_url)
       mirror.print = lambda *args: None
       print('Updating git cache...')
-      mirror.populate()
+      mirror.populate(bootstrap=not options.no_bootstrap)
     subprocess.check_call([Mirror.git_exe, 'fetch', remote])
   return 0
 
 
 @subcommand.usage('[url of repo to unlock, or -a|--all]')
 def CMDunlock(parser, args):
   """Unlock one or all repos if their lock files are still around."""
   parser.add_option('--force', '-f', action='store_true',
                     help='Actually perform the action')
   parser.add_option('--all', '-a', action='store_true',
@@ skipping 62 matching lines @@
   dispatcher = subcommand.CommandDispatcher(__name__)
   return dispatcher.execute(OptionParser(), argv)
 
 
 if __name__ == '__main__':
   try:
     sys.exit(main(sys.argv[1:]))
   except KeyboardInterrupt:
     sys.stderr.write('interrupted\n')
     sys.exit(1)