Allow 'git cache fetch' to re-bootstrap.

git_cache.py

Index: git_cache.py
diff --git a/git_cache.py b/git_cache.py
index e80923cdbb074a0c27355bd51041341fd473ed27..12948ab7747193f0e3c75ad87a881f91df0aef1b 100755
--- a/git_cache.py
+++ b/git_cache.py
@@ -147,7 +147,7 @@ class Mirror(object):
 
   def __init__(self, url, refs=None, print_func=None):
     self.url = url
-    self.refs = refs or []
+    self.refs = set([ref.lstrip('+').rstrip('/') for ref in (refs or [])])

[iannucci, 2015/09/22 20:35:26]

can we have a 'parse_fetch_spec' function so the next person knows what's
happening here?

[szager1, 2015/09/22 22:48:27]

Done.

     self.basedir = self.UrlToCacheDir(url)
     self.mirror_path = os.path.join(self.GetCachePath(), self.basedir)
     if print_func:
@@ -237,7 +237,6 @@ class Mirror(object):
     self.RunGit(['config', '--replace-all', 'remote.origin.fetch',
                  '+refs/heads/*:refs/heads/*', r'\+refs/heads/\*:.*'], cwd=cwd)
     for ref in self.refs:
-      ref = ref.lstrip('+').rstrip('/')
       if ref.startswith('refs/'):
         refspec = '+%s:%s' % (ref, ref)
         regex = r'\+%s:.*' % ref.replace('*', r'\*')
@@ -314,9 +313,21 @@ class Mirror(object):
   def exists(self):
     return os.path.isfile(os.path.join(self.mirror_path, 'config'))
 
+  def _preserve_fetchspec(self):
+    """Read and preserve remote.origin.fetch from an existing mirror."""
+    if not self.exists():
+      return
+    try:
+      config_fetchspecs = subprocess.check_output(
+          [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'],
+          cwd=self.mirror_path)
+      for fetchspec in config_fetchspecs.splitlines():
+        self.refs.add(fetchspec.split(':')[0].lstrip('+').rstrip('/'))

[iannucci, 2015/09/22 20:35:26]

and here too

[szager1, 2015/09/22 22:48:27]

Done.

+    except subprocess.CalledProcessError:
+      pass
+
   def _ensure_bootstrapped(self, depth, bootstrap, force=False):
     tempdir = None
-    config_file = os.path.join(self.mirror_path, 'config')
     pack_dir = os.path.join(self.mirror_path, 'objects', 'pack')
     pack_files = []
 
@@ -324,16 +335,19 @@ class Mirror(object):
       pack_files = [f for f in os.listdir(pack_dir) if f.endswith('.pack')]
 
     should_bootstrap = (force or
-                        not os.path.exists(config_file) or
+                        not self.exists() or
                         len(pack_files) > GC_AUTOPACKLIMIT)
     if should_bootstrap:
+      if self.exists():
+        # Re-bootstrapping an existing mirror; preserve existing fetch spec.
+        self._preserve_fetchspec()
       tempdir = tempfile.mkdtemp(
           prefix='_cache_tmp', suffix=self.basedir, dir=self.GetCachePath())
       bootstrapped = not depth and bootstrap and self.bootstrap_repo(tempdir)
       if bootstrapped:
         # Bootstrap succeeded; delete previous cache, if any.
         gclient_utils.rmtree(self.mirror_path)
-      elif not os.path.exists(config_file):
+      elif not self.exists():
         # Bootstrap failed, no previous cache; start with a bare git dir.
         self.RunGit(['init', '--bare'], cwd=tempdir)
       else:
@@ -563,6 +577,9 @@ def CMDpopulate(parser, args):
 def CMDfetch(parser, args):
   """Update mirror, and fetch in cwd."""
   parser.add_option('--all', action='store_true', help='Fetch all remotes')
+  parser.add_option('--no_bootstrap', '--no-bootstrap',
+                    action='store_true',
+                    help='Don\'t (re)bootstrap from Google Storage')
   options, args = parser.parse_args(args)
 
   # Figure out which remotes to fetch.  This mimics the behavior of regular
@@ -593,7 +610,7 @@ def CMDfetch(parser, args):
   git_dir = os.path.abspath(git_dir)
   if git_dir.startswith(cachepath):
     mirror = Mirror.FromPath(git_dir)
-    mirror.populate()
+    mirror.populate(bootstrap=not options.no_bootstrap)
     return 0
   for remote in remotes:
     remote_url = subprocess.check_output(
@@ -602,7 +619,7 @@ def CMDfetch(parser, args):
       mirror = Mirror.FromPath(remote_url)
       mirror.print = lambda *args: None
       print('Updating git cache...')
-      mirror.populate()
+      mirror.populate(bootstrap=not options.no_bootstrap)
     subprocess.check_call([Mirror.git_exe, 'fetch', remote])
   return 0