Have git cache bootstrap repo if repo is corrupt

git_cache.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """A git command for managing a local cache of git repositories."""
 
 from __future__ import print_function
 import errno
 import logging
 import optparse
 import os
 import re
 import tempfile
 import time
 import subprocess
 import sys
 import urlparse
 import zipfile
 
 from download_from_google_storage import Gsutil
 import gclient_utils
 import subcommand
 
 # Analogous to gc.autopacklimit git config.
 GC_AUTOPACKLIMIT = 50
 
+GIT_CACHE_CORRUPT_MESSAGE = 'The Git cache is corrupt!! Re-bootstrapping now.'

[Ryan Tseng, 2014/06/26 17:35:03]

I was planning on tracking the occurrence of this by having bot_update grep for
this string in the gclient sync output.

[szager1, 2014/06/26 20:25:36]

That sounds good, but how about getting rid of the '!' characters, and put all
the info on one line for easy grep-ing:

GIT_CACHE_CORRUPT_TEMPLATE = 'WARNING: Corrupt git cache for %s; saved for
debugging to %s'

...
print GIT_CACHE_CORRUPT_TEMPLATE % (remote_url, tmp_dir)

[Ryan Tseng, 2014/06/26 21:37:58]

So the point of having this as a macro up to is so I can do something like this
in bot_update:
output = call(...)
idx = output.find(git_cache.GIT_CACHE_CORRUPT_MESSAGE)
so making it a template defeats that purpose.  Maybe

GIT_CACHE_CORRUPT_MESSAGE = 'WARNING: Corrupt git cache found.'

...
print '%s cache for %s moved to %s' % (GIT_CACHE_CORRUPT_MESSAGE, remote_url,
tmp_dir)

Also the move() is in a try/except, so it might not always run.

[szager1, 2014/06/26 21:47:31]

That's fine, but it would be nice to be able to:

output = call(...)
for line in output:
  if line.startswith(git_cache.GIT_CACHE_CORRUPT_MESSAGE):
    # parse url and temp dir from line

Even though the move might fail, we still want to print the path so we can at
least try to debug it.

+
 try:
   # pylint: disable=E0602
   WinErr = WindowsError
 except NameError:
   class WinErr(Exception):
     pass
 
 class LockError(Exception):
   pass
 
+class RefsHeadsFailedToFetch(Exception):
+  pass
 
 class Lockfile(object):
   """Class to represent a cross-platform process-specific lockfile."""
 
   def __init__(self, path):
     self.path = os.path.abspath(path)
     self.lockfile = self.path + ".lock"
     self.pid = os.getpid()
 
   def _read_pid(self):
@@ skipping 193 matching lines @@
                  '+refs/heads/*:refs/heads/*'], cwd=cwd)
     for ref in self.refs:
       ref = ref.lstrip('+').rstrip('/')
       if ref.startswith('refs/'):
         refspec = '+%s:%s' % (ref, ref)
       else:
         refspec = '+refs/%s/*:refs/%s/*' % (ref, ref)
       self.RunGit(['config', '--add', 'remote.origin.fetch', refspec], cwd=cwd)
 
   def bootstrap_repo(self, directory):
-    """Bootstrap the repo from Google Stroage if possible."""
+    """Bootstrap the repo from Google Stroage if possible.
+
+    More apt-ly named bootstrap_repo_from_cloud_if_possible_else_do_nothing().
+    """
 
     python_fallback = False
     if sys.platform.startswith('win') and not self.FindExecutable('7z'):
       python_fallback = True
     elif sys.platform.startswith('darwin'):
       # The OSX version of unzip doesn't support zip64.
       python_fallback = True
     elif not self.FindExecutable('unzip'):
       python_fallback = True
 
@@ skipping 40 matching lines @@
     if retcode:
       self.print(
           'Extracting bootstrap zipfile %s failed.\n'
           'Resuming normal operations.' % filename)
       return False
     return True
 
   def exists(self):
     return os.path.isfile(os.path.join(self.mirror_path, 'config'))
 
+  def _ensure_bootstrapped(self, depth, bootstrap, force=False):
+    tempdir = None
+    config_file = os.path.join(self.mirror_path, 'config')
+    pack_dir = os.path.join(self.mirror_path, 'objects', 'pack')
+    pack_files = []
+
+    if os.path.isdir(pack_dir):
+      pack_files = [f for f in os.listdir(pack_dir) if f.endswith('.pack')]
+
+    should_bootstrap = (force or
+                        not os.path.exists(config_file) or
+                        len(pack_files) > GC_AUTOPACKLIMIT)
+    if should_bootstrap:
+      tempdir = tempfile.mkdtemp(
+          prefix='_cache_tmp', suffix=self.basedir, dir=self.GetCachePath())
+      bootstrapped = not depth and bootstrap and self.bootstrap_repo(tempdir)
+      if bootstrapped:
+        # Bootstrap succeeded; delete previous cache, if any.
+        gclient_utils.rmtree(self.mirror_path)
+      elif not os.path.exists(config_file):
+        # Bootstrap failed, no previous cache; start with a bare git dir.
+        self.RunGit(['init', '--bare'], cwd=tempdir)
+      else:
+        # Bootstrap failed, previous cache exists; warn and continue.
+        logging.warn(
+            'Git cache has a lot of pack files (%d).  Tried to re-bootstrap '
+            'but failed.  Continuing with non-optimized repository.'
+            % len(pack_files))
+        gclient_utils.rmtree(tempdir)
+        tempdir = None
+    else:
+      if depth and os.path.exists(os.path.join(self.mirror_path, 'shallow')):
+        logging.warn(
+            'Shallow fetch requested, but repo cache already exists.')
+    return tempdir
+
+  def _fetch(self, rundir, verbose, depth):
+    self.config(rundir)
+    v = []
+    d = []
+    if verbose:
+      v = ['-v', '--progress']
+    if depth:
+      d = ['--depth', str(depth)]
+    fetch_cmd = ['fetch'] + v + d + ['origin']
+    fetch_specs = subprocess.check_output(
+        [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'],
+        cwd=rundir).strip().splitlines()
+    for spec in fetch_specs:
+      try:
+        self.print('Fetching %s' % spec)
+        self.RunGit(fetch_cmd + [spec], cwd=rundir, retry=True)
+      except subprocess.CalledProcessError:
+        if spec == '+refs/heads/*:refs/heads/*':
+          raise RefsHeadsFailedToFetch
+        logging.warn('Fetch of %s failed' % spec)
+
   def populate(self, depth=None, shallow=False, bootstrap=False,
                verbose=False, ignore_lock=False):
     assert self.GetCachePath()
     if shallow and not depth:
       depth = 10000
     gclient_utils.safe_makedirs(self.GetCachePath())
 
-    v = []
-    if verbose:
-      v = ['-v', '--progress']
-
-    d = []
-    if depth:
-      d = ['--depth', str(depth)]
-
-
     lockfile = Lockfile(self.mirror_path)
     if not ignore_lock:
       lockfile.lock()
 
     try:
-      # Setup from scratch if the repo is new or is in a bad state.
-      tempdir = None
-      config_file = os.path.join(self.mirror_path, 'config')
-      pack_dir = os.path.join(self.mirror_path, 'objects', 'pack')
-      pack_files = []
-      if os.path.isdir(pack_dir):
-        pack_files = [f for f in os.listdir(pack_dir) if f.endswith('.pack')]
-
-      should_bootstrap = (not os.path.exists(config_file) or
-                          len(pack_files) > GC_AUTOPACKLIMIT)
-      if should_bootstrap:
-        tempdir = tempfile.mkdtemp(
-            prefix='_cache_tmp', suffix=self.basedir, dir=self.GetCachePath())
-        bootstrapped = not depth and bootstrap and self.bootstrap_repo(tempdir)
-        if bootstrapped:
-          # Bootstrap succeeded; delete previous cache, if any.
-          gclient_utils.rmtree(self.mirror_path)
-        elif not os.path.exists(config_file):
-          # Bootstrap failed, no previous cache; start with a bare git dir.
-          self.RunGit(['init', '--bare'], cwd=tempdir)
-        else:
-          # Bootstrap failed, previous cache exists; warn and continue.
-          logging.warn(
-              'Git cache has a lot of pack files (%d).  Tried to re-bootstrap '
-              'but failed.  Continuing with non-optimized repository.'
-              % len(pack_files))
-          gclient_utils.rmtree(tempdir)
-          tempdir = None
-      else:
-        if depth and os.path.exists(os.path.join(self.mirror_path, 'shallow')):
-          logging.warn(
-              'Shallow fetch requested, but repo cache already exists.')
-        d = []
-
-      rundir = tempdir or self.mirror_path
-      self.config(rundir)
-      fetch_cmd = ['fetch'] + v + d + ['origin']
-      fetch_specs = subprocess.check_output(
-          [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'],
-          cwd=rundir).strip().splitlines()
-      for spec in fetch_specs:
-        try:
-          self.RunGit(fetch_cmd + [spec], cwd=rundir, retry=True)
-        except subprocess.CalledProcessError:
-          logging.warn('Fetch of %s failed' % spec)
+      tempdir = self._ensure_bootstrapped(depth, bootstrap)
+      try:

[szager1, 2014/06/26 20:25:36]

I don't think you really need nested 'try' here:
tempdir = None
try:
  tempdir = self._ensure_bootstrap(depth, bootstrap)
  rundir = tempdir or self.mirror_path
  self._fetch(rundir, verbose, depth)
except RefsHeadsFailedToFetch:
  gclient_utils.rmtree(rundir)
  tempdir = self._ensure_bootstrap(depth, bootstrap, force=True)
  assert tempdir
  self._fetch(tempdir, verbose, depth)
finally:
  if tempdir:
    os.rename(tempdir, self.mirror_path)
  if not ignore_lock:
    lockfile.unlock()

[Ryan Tseng, 2014/06/26 21:37:58]

Done.

+        self._fetch(tempdir or self.mirror_path, verbose, depth)
+      except RefsHeadsFailedToFetch:
+        # This is a major failure, we need to clean and force a bootstrap.
+        self.print(GIT_CACHE_CORRUPT_MESSAGE)
+        tempdir = self._ensure_bootstrapped(depth, bootstrap, force=True)
+        self._fetch(tempdir or self.mirror_path, verbose, depth)
       if tempdir:
         os.rename(tempdir, self.mirror_path)
     finally:
       if not ignore_lock:
         lockfile.unlock()
 
   def update_bootstrap(self, prune=False):
     # The files are named <git number>.zip
     gen_number = subprocess.check_output(
         [self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip()
@@ skipping 207 matching lines @@
     return options, args
 
 
 def main(argv):
   dispatcher = subcommand.CommandDispatcher(__name__)
   return dispatcher.execute(OptionParser(), argv)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv[1:]))