Have git cache bootstrap repo if repo is corrupt

git_cache.py

Index: git_cache.py
diff --git a/git_cache.py b/git_cache.py
index 75fb29ac4a638cc87353e04b957a53f383bc7dc9..67c7b3ec7724431c21478d1b2e13fd2b596c5152 100755
--- a/git_cache.py
+++ b/git_cache.py
@@ -25,6 +25,8 @@ import subcommand
 # Analogous to gc.autopacklimit git config.
 GC_AUTOPACKLIMIT = 50
 
+GIT_CACHE_CORRUPT_MESSAGE = 'The Git cache is corrupt!! Re-bootstrapping now.'

[Ryan Tseng, 2014/06/26 17:35:03]

I was planning on tracking the occurrence of this by having bot_update grep for
this string in the gclient sync output.

[szager1, 2014/06/26 20:25:36]

That sounds good, but how about getting rid of the '!' characters, and put all
the info on one line for easy grep-ing:

GIT_CACHE_CORRUPT_TEMPLATE = 'WARNING: Corrupt git cache for %s; saved for
debugging to %s'

...
print GIT_CACHE_CORRUPT_TEMPLATE % (remote_url, tmp_dir)

[Ryan Tseng, 2014/06/26 21:37:58]

So the point of having this as a macro up to is so I can do something like this
in bot_update:
output = call(...)
idx = output.find(git_cache.GIT_CACHE_CORRUPT_MESSAGE)
so making it a template defeats that purpose.  Maybe

GIT_CACHE_CORRUPT_MESSAGE = 'WARNING: Corrupt git cache found.'

...
print '%s cache for %s moved to %s' % (GIT_CACHE_CORRUPT_MESSAGE, remote_url,
tmp_dir)

Also the move() is in a try/except, so it might not always run.

[szager1, 2014/06/26 21:47:31]

That's fine, but it would be nice to be able to:

output = call(...)
for line in output:
  if line.startswith(git_cache.GIT_CACHE_CORRUPT_MESSAGE):
    # parse url and temp dir from line

Even though the move might fail, we still want to print the path so we can at
least try to debug it.

+
 try:
   # pylint: disable=E0602
   WinErr = WindowsError
@@ -35,6 +37,8 @@ except NameError:
 class LockError(Exception):
   pass
 
+class RefsHeadsFailedToFetch(Exception):
+  pass
 
 class Lockfile(object):
   """Class to represent a cross-platform process-specific lockfile."""
@@ -248,7 +252,10 @@ class Mirror(object):
       self.RunGit(['config', '--add', 'remote.origin.fetch', refspec], cwd=cwd)
 
   def bootstrap_repo(self, directory):
-    """Bootstrap the repo from Google Stroage if possible."""
+    """Bootstrap the repo from Google Stroage if possible.
+
+    More apt-ly named bootstrap_repo_from_cloud_if_possible_else_do_nothing().
+    """
 
     python_fallback = False
     if sys.platform.startswith('win') and not self.FindExecutable('7z'):
@@ -309,72 +316,83 @@ class Mirror(object):
   def exists(self):
     return os.path.isfile(os.path.join(self.mirror_path, 'config'))
 
-  def populate(self, depth=None, shallow=False, bootstrap=False,
-               verbose=False, ignore_lock=False):
-    assert self.GetCachePath()
-    if shallow and not depth:
-      depth = 10000
-    gclient_utils.safe_makedirs(self.GetCachePath())
+  def _ensure_bootstrapped(self, depth, bootstrap, force=False):
+    tempdir = None
+    config_file = os.path.join(self.mirror_path, 'config')
+    pack_dir = os.path.join(self.mirror_path, 'objects', 'pack')
+    pack_files = []
+
+    if os.path.isdir(pack_dir):
+      pack_files = [f for f in os.listdir(pack_dir) if f.endswith('.pack')]
+
+    should_bootstrap = (force or
+                        not os.path.exists(config_file) or
+                        len(pack_files) > GC_AUTOPACKLIMIT)
+    if should_bootstrap:
+      tempdir = tempfile.mkdtemp(
+          prefix='_cache_tmp', suffix=self.basedir, dir=self.GetCachePath())
+      bootstrapped = not depth and bootstrap and self.bootstrap_repo(tempdir)
+      if bootstrapped:
+        # Bootstrap succeeded; delete previous cache, if any.
+        gclient_utils.rmtree(self.mirror_path)
+      elif not os.path.exists(config_file):
+        # Bootstrap failed, no previous cache; start with a bare git dir.
+        self.RunGit(['init', '--bare'], cwd=tempdir)
+      else:
+        # Bootstrap failed, previous cache exists; warn and continue.
+        logging.warn(
+            'Git cache has a lot of pack files (%d).  Tried to re-bootstrap '
+            'but failed.  Continuing with non-optimized repository.'
+            % len(pack_files))
+        gclient_utils.rmtree(tempdir)
+        tempdir = None
+    else:
+      if depth and os.path.exists(os.path.join(self.mirror_path, 'shallow')):
+        logging.warn(
+            'Shallow fetch requested, but repo cache already exists.')
+    return tempdir
 
+  def _fetch(self, rundir, verbose, depth):
+    self.config(rundir)
     v = []
+    d = []
     if verbose:
       v = ['-v', '--progress']
-
-    d = []
     if depth:
       d = ['--depth', str(depth)]
+    fetch_cmd = ['fetch'] + v + d + ['origin']
+    fetch_specs = subprocess.check_output(
+        [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'],
+        cwd=rundir).strip().splitlines()
+    for spec in fetch_specs:
+      try:
+        self.print('Fetching %s' % spec)
+        self.RunGit(fetch_cmd + [spec], cwd=rundir, retry=True)
+      except subprocess.CalledProcessError:
+        if spec == '+refs/heads/*:refs/heads/*':
+          raise RefsHeadsFailedToFetch
+        logging.warn('Fetch of %s failed' % spec)
 
+  def populate(self, depth=None, shallow=False, bootstrap=False,
+               verbose=False, ignore_lock=False):
+    assert self.GetCachePath()
+    if shallow and not depth:
+      depth = 10000
+    gclient_utils.safe_makedirs(self.GetCachePath())
 
     lockfile = Lockfile(self.mirror_path)
     if not ignore_lock:
       lockfile.lock()
 
     try:
-      # Setup from scratch if the repo is new or is in a bad state.
-      tempdir = None
-      config_file = os.path.join(self.mirror_path, 'config')
-      pack_dir = os.path.join(self.mirror_path, 'objects', 'pack')
-      pack_files = []
-      if os.path.isdir(pack_dir):
-        pack_files = [f for f in os.listdir(pack_dir) if f.endswith('.pack')]
-
-      should_bootstrap = (not os.path.exists(config_file) or
-                          len(pack_files) > GC_AUTOPACKLIMIT)
-      if should_bootstrap:
-        tempdir = tempfile.mkdtemp(
-            prefix='_cache_tmp', suffix=self.basedir, dir=self.GetCachePath())
-        bootstrapped = not depth and bootstrap and self.bootstrap_repo(tempdir)
-        if bootstrapped:
-          # Bootstrap succeeded; delete previous cache, if any.
-          gclient_utils.rmtree(self.mirror_path)
-        elif not os.path.exists(config_file):
-          # Bootstrap failed, no previous cache; start with a bare git dir.
-          self.RunGit(['init', '--bare'], cwd=tempdir)
-        else:
-          # Bootstrap failed, previous cache exists; warn and continue.
-          logging.warn(
-              'Git cache has a lot of pack files (%d).  Tried to re-bootstrap '
-              'but failed.  Continuing with non-optimized repository.'
-              % len(pack_files))
-          gclient_utils.rmtree(tempdir)
-          tempdir = None
-      else:
-        if depth and os.path.exists(os.path.join(self.mirror_path, 'shallow')):
-          logging.warn(
-              'Shallow fetch requested, but repo cache already exists.')
-        d = []
-
-      rundir = tempdir or self.mirror_path
-      self.config(rundir)
-      fetch_cmd = ['fetch'] + v + d + ['origin']
-      fetch_specs = subprocess.check_output(
-          [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'],
-          cwd=rundir).strip().splitlines()
-      for spec in fetch_specs:
-        try:
-          self.RunGit(fetch_cmd + [spec], cwd=rundir, retry=True)
-        except subprocess.CalledProcessError:
-          logging.warn('Fetch of %s failed' % spec)
+      tempdir = self._ensure_bootstrapped(depth, bootstrap)
+      try:

[szager1, 2014/06/26 20:25:36]

I don't think you really need nested 'try' here:
tempdir = None
try:
  tempdir = self._ensure_bootstrap(depth, bootstrap)
  rundir = tempdir or self.mirror_path
  self._fetch(rundir, verbose, depth)
except RefsHeadsFailedToFetch:
  gclient_utils.rmtree(rundir)
  tempdir = self._ensure_bootstrap(depth, bootstrap, force=True)
  assert tempdir
  self._fetch(tempdir, verbose, depth)
finally:
  if tempdir:
    os.rename(tempdir, self.mirror_path)
  if not ignore_lock:
    lockfile.unlock()

[Ryan Tseng, 2014/06/26 21:37:58]

Done.

+        self._fetch(tempdir or self.mirror_path, verbose, depth)
+      except RefsHeadsFailedToFetch:
+        # This is a major failure, we need to clean and force a bootstrap.
+        self.print(GIT_CACHE_CORRUPT_MESSAGE)
+        tempdir = self._ensure_bootstrapped(depth, bootstrap, force=True)
+        self._fetch(tempdir or self.mirror_path, verbose, depth)
       if tempdir:
         os.rename(tempdir, self.mirror_path)
     finally: