Add retries to file operations for Windows.

git_cache.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """A git command for managing a local cache of git repositories."""
 
 from __future__ import print_function
 import errno
 import logging
@@ skipping 23 matching lines @@
 except NameError:
   class WinErr(Exception):
     pass
 
 class LockError(Exception):
   pass
 
 class ClobberNeeded(Exception):
   pass
 
+
+def exponential_backoff_retry(fn, excs=(Exception,), name=None, count=10,
+                              sleep_time=0.25, printerr=None):
+  """Executes |fn| up to |count| times, backing off exponentially.
+
+  Args:
+    fn (callable): The function to execute. If this raises a handled
+        exception, the function will retry with exponential backoff.
+    excs (tuple): A tuple of Exception types to handle. If one of these is
+        raised by |fn|, a retry will be attempted. If |fn| raises an Exception
+        that is not in this list, it will immediately pass through. If |excs|
+        is empty, the Exception base class will be used.
+    name (str): Optional operation name to print in the retry string.
+    count (int): The number of times to try before allowing the exception to
+        pass through.
+    sleep_time (float): The initial number of seconds to sleep in between
+        retries. This will be doubled each retry.
+    printerr (callable): Function that will be called with the error string upon
+        failures. If None, |logging.warning| will be used.
+
+  Returns: The return value of the successful fn.
+  """
+  printerr = printerr or logging.warning
+  for i in xrange(count):
+    try:
+      return fn()
+    except excs as e:
+      if (i+1) >= count:
+        raise
+
+      printerr('Retrying %s in %.2f second(s) (%d / %d attempts): %s' % (
+          (name or 'operation'), sleep_time, (i+1), count, e))
+      time.sleep(sleep_time)
+      sleep_time *= 2
+
+
 class Lockfile(object):
   """Class to represent a cross-platform process-specific lockfile."""
 
   def __init__(self, path, timeout=0):
     self.path = os.path.abspath(path)
     self.timeout = timeout
     self.lockfile = self.path + ".lock"
     self.pid = os.getpid()
 
   def _read_pid(self):
@@ skipping 18 matching lines @@
     f.close()
 
   def _remove_lockfile(self):
     """Delete the lockfile. Complains (implicitly) if it doesn't exist.
 
     See gclient_utils.py:rmtree docstring for more explanation on the
     windows case.
     """
     if sys.platform == 'win32':
       lockfile = os.path.normcase(self.lockfile)
-      for _ in xrange(3):
+
+      def delete():
         exitcode = subprocess.call(['cmd.exe', '/c',
                                     'del', '/f', '/q', lockfile])
-        if exitcode == 0:
-          return
-        time.sleep(3)
-      raise LockError('Failed to remove lock: %s' % lockfile)
+        if exitcode != 0:
+          raise LockError('Failed to remove lock: %s' % (lockfile,))
+      exponential_backoff_retry(
+          delete,
+          excs=(LockError,),
+          name='del [%s]' % (lockfile,))
     else:
       os.remove(self.lockfile)
 
   def lock(self):
     """Acquire the lock.
 
     This will block with a deadline of self.timeout seconds.
     """
     elapsed = 0
     while True:
@@ skipping 75 matching lines @@
     self.url = url
     self.fetch_specs = set([self.parse_fetch_spec(ref) for ref in (refs or [])])
     self.basedir = self.UrlToCacheDir(url)
     self.mirror_path = os.path.join(self.GetCachePath(), self.basedir)
     if print_func:
       self.print = self.print_without_file
       self.print_func = print_func
     else:
       self.print = print
 
-  def print_without_file(self, message, **kwargs):
+  def print_without_file(self, message, **_kwargs):
     self.print_func(message)
 
   @property
   def bootstrap_bucket(self):
     if 'chrome-internal' in self.url:
       return 'chrome-git-cache'
     else:
       return 'chromium-git-cache'
 
   @classmethod
@@ skipping 28 matching lines @@
           cachepath = subprocess.check_output(
               [cls.git_exe, 'config', '--global', 'cache.cachepath']).strip()
         except subprocess.CalledProcessError:
           cachepath = None
         if not cachepath:
           raise RuntimeError(
               'No global cache.cachepath git configuration found.')
         setattr(cls, 'cachepath', cachepath)
       return getattr(cls, 'cachepath')
 
+  def Rename(self, src, dst):
+    # This is somehow racy on Windows.
+    # Catching OSError because WindowsError isn't portable and
+    # pylint complains.
+    exponential_backoff_retry(
+        lambda: os.rename(src, dst),
+        excs=(OSError,),
+        name='rename [%s] => [%s]' % (src, dst),
+        printerr=self.print)
+
   def RunGit(self, cmd, **kwargs):
     """Run git in a subprocess."""
     cwd = kwargs.setdefault('cwd', self.mirror_path)
     kwargs.setdefault('print_stdout', False)
     kwargs.setdefault('filter_fn', self.print)
     env = kwargs.get('env') or kwargs.setdefault('env', os.environ.copy())
     env.setdefault('GIT_ASKPASS', 'true')
     env.setdefault('SSH_ASKPASS', 'true')
     self.print('running "git %s" in "%s"' % (' '.join(cmd), cwd))
     gclient_utils.CheckCallAndFilter([self.git_exe] + cmd, **kwargs)
@@ skipping 74 matching lines @@
           with zipfile.ZipFile(filename, 'r') as f:
             f.printdir()
             f.extractall(directory)
         except Exception as e:
           self.print('Encountered error: %s' % str(e), file=sys.stderr)
           retcode = 1
         else:
           retcode = 0
     finally:
       # Clean up the downloaded zipfile.
-      gclient_utils.rm_file_or_tree(tempdir)
+      #
+      # This is somehow racy on Windows.
+      # Catching OSError because WindowsError isn't portable and
+      # pylint complains.
+      exponential_backoff_retry(
+          lambda: gclient_utils.rm_file_or_tree(tempdir),
+          excs=(OSError,),
+          name='rmtree [%s]' % (tempdir,),
+          printerr=self.print)
 
     if retcode:
       self.print(
           'Extracting bootstrap zipfile %s failed.\n'
           'Resuming normal operations.' % filename)
       return False
     return True
 
   def exists(self):
     return os.path.isfile(os.path.join(self.mirror_path, 'config'))
@@ skipping 96 matching lines @@
       # This is a major failure, we need to clean and force a bootstrap.
       gclient_utils.rmtree(rundir)
       self.print(GIT_CACHE_CORRUPT_MESSAGE)
       tempdir = self._ensure_bootstrapped(depth, bootstrap, force=True)
       assert tempdir
       self._fetch(tempdir or self.mirror_path, verbose, depth)
     finally:
       if tempdir:
         if os.path.exists(self.mirror_path):
           gclient_utils.rmtree(self.mirror_path)
-        os.rename(tempdir, self.mirror_path)
+        self.Rename(tempdir, self.mirror_path)
       if not ignore_lock:
         lockfile.unlock()
 
   def update_bootstrap(self, prune=False):
     # The files are named <git number>.zip
     gen_number = subprocess.check_output(
         [self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip()
     # Run Garbage Collect to compress packfile.
     self.RunGit(['gc', '--prune=all'])
     # Creating a temp file and then deleting it ensures we can use this name.
@@ skipping 267 matching lines @@
   dispatcher = subcommand.CommandDispatcher(__name__)
   return dispatcher.execute(OptionParser(), argv)
 
 
 if __name__ == '__main__':
   try:
     sys.exit(main(sys.argv[1:]))
   except KeyboardInterrupt:
     sys.stderr.write('interrupted\n')
     sys.exit(1)