Consolidate subprocess retry logic into gclient_utils.

gclient_scm.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Gclient-specific SCM-specific operations."""
 
 import collections
 import logging
 import os
 import posixpath
@@ skipping 263 matching lines @@
 
   def _FetchAndReset(self, revision, file_list, options):
     """Equivalent to git fetch; git reset."""
     quiet = []
     if not options.verbose:
       quiet = ['--quiet']
     self._UpdateBranchHeads(options, fetch=False)
 
     fetch_cmd = [
       '-c', 'core.deltaBaseCacheLimit=2g', 'fetch', 'origin', '--prune']
-    self._Run(fetch_cmd + quiet, options)
+    self._Run(fetch_cmd + quiet, options, retry=True)
     self._Run(['reset', '--hard', revision] + quiet, options)
     self.UpdateSubmoduleConfig()
     if file_list is not None:
       files = self._Capture(['ls-files']).splitlines()
       file_list.extend([os.path.join(self.checkout_path, f) for f in files])
 
   def update(self, options, args, file_list):
     """Runs git to update or transparently checkout the working copy.
 
     All updated files will be appended to file_list.
@@ skipping 500 matching lines @@
       if not os.path.exists(os.path.join(folder, 'config')):
         gclient_utils.rmtree(folder)
         cmd = ['clone'] + v + ['-c', 'core.deltaBaseCacheLimit=2g',
                                '--progress', '--mirror']
 
         if use_reference:
           cmd += ['--reference', os.path.abspath(self.checkout_path)]
 
         self._Run(cmd + [url, folder],
                   options, git_filter=True, filter_fn=filter_fn,
-                  cwd=self.cache_dir)
+                  cwd=self.cache_dir, retry=True)
       else:
         # For now, assert that host/path/to/repo.git is identical. We may want
         # to relax this restriction in the future to allow for smarter cache
         # repo update schemes (such as pulling the same repo, but from a
         # different host).
         existing_url = self._Capture(['config', 'remote.origin.url'],
                                      cwd=folder)
         assert self._NormalizeGitURL(existing_url) == self._NormalizeGitURL(url)
 
         if use_reference:
           with open(altfile, 'w') as f:
             f.write(os.path.abspath(checkout_objects))
 
         # Would normally use `git remote update`, but it doesn't support
         # --progress, so use fetch instead.
         self._Run(['fetch'] + v + ['--multiple', '--progress', '--all'],
-                  options, git_filter=True, filter_fn=filter_fn, cwd=folder)
+                  options, git_filter=True, filter_fn=filter_fn, cwd=folder,
+                  retry=True)
 
       # If the clone has an object dependency on the existing repo, break it
       # with repack and remove the linkage.
       if os.path.exists(altfile):
         self._Run(['repack', '-a'], options, cwd=folder)
         os.remove(altfile)
     return folder
 
   def _Clone(self, revision, url, options):
     """Clone a git repository from the given URL.
@@ skipping 18 matching lines @@
     clone_cmd.append(url)
     # If the parent directory does not exist, Git clone on Windows will not
     # create it, so we need to do it manually.
     parent_dir = os.path.dirname(self.checkout_path)
     gclient_utils.safe_makedirs(parent_dir)
     tmp_dir = tempfile.mkdtemp(
         prefix='_gclient_%s_' % os.path.basename(self.checkout_path),
         dir=parent_dir)
     try:
       clone_cmd.append(tmp_dir)
-      for i in xrange(3):
-        try:
-          self._Run(clone_cmd, options, cwd=self._root_dir, git_filter=True)
-          break
-        except subprocess2.CalledProcessError as e:
-          gclient_utils.rmtree(os.path.join(tmp_dir, '.git'))
-          if e.returncode != 128 or i == 2:
-            raise
-          print(str(e))
-          print('Retrying...')
+      self._Run(clone_cmd, options, cwd=self._root_dir, git_filter=True,
+                retry=True)
       gclient_utils.safe_makedirs(self.checkout_path)
       gclient_utils.safe_rename(os.path.join(tmp_dir, '.git'),
                                 os.path.join(self.checkout_path, '.git'))
     finally:
       if os.listdir(tmp_dir):
         print('\n_____ removing non-empty tmp dir %s' % tmp_dir)
       gclient_utils.rmtree(tmp_dir)
     if revision.startswith('refs/heads/'):
       self._Run(
           ['checkout', '--quiet', revision.replace('refs/heads/', '')], options)
@@ skipping 146 matching lines @@
 
   def _Capture(self, args, cwd=None):
     return subprocess2.check_output(
         ['git'] + args,
         stderr=subprocess2.VOID,
         cwd=cwd or self.checkout_path).strip()
 
   def _UpdateBranchHeads(self, options, fetch=False):
     """Adds, and optionally fetches, "branch-heads" refspecs if requested."""
     if hasattr(options, 'with_branch_heads') and options.with_branch_heads:
-      backoff_time = 5
-      for _ in range(3):
-        try:
-          config_cmd = ['config', 'remote.origin.fetch',
-                        '+refs/branch-heads/*:refs/remotes/branch-heads/*',
-                        '^\\+refs/branch-heads/\\*:.*$']
-          self._Run(config_cmd, options)
-          if fetch:
-            fetch_cmd = ['-c', 'core.deltaBaseCacheLimit=2g', 'fetch', 'origin']
-            if options.verbose:
-              fetch_cmd.append('--verbose')
-            self._Run(fetch_cmd, options)
-          break
-        except subprocess2.CalledProcessError, e:
-          print(str(e))
-          print('Retrying in %.1f seconds...' % backoff_time)
-          time.sleep(backoff_time)
-          backoff_time *= 1.3
+      config_cmd = ['config', 'remote.origin.fetch',
+                    '+refs/branch-heads/*:refs/remotes/branch-heads/*',
+                    '^\\+refs/branch-heads/\\*:.*$']
+      self._Run(config_cmd, options)
+      if fetch:
+        fetch_cmd = ['-c', 'core.deltaBaseCacheLimit=2g', 'fetch', 'origin']
+        if options.verbose:
+          fetch_cmd.append('--verbose')
+        self._Run(fetch_cmd, options, retry=True)
 
   def _Run(self, args, _options, git_filter=False, **kwargs):
     kwargs.setdefault('cwd', self.checkout_path)
     if git_filter:
       kwargs['filter_fn'] = GitFilter(kwargs.get('filter_fn'))
       kwargs.setdefault('print_stdout', False)
       # Don't prompt for passwords; just fail quickly and noisily.
       # By default, git will use an interactive terminal prompt when a username/
       # password is needed.  That shouldn't happen in the chromium workflow,
       # and if it does, then gclient may hide the prompt in the midst of a flood
@@ skipping 403 matching lines @@
       new_command.append('--force')
       if command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
         new_command.extend(('--accept', 'theirs-conflict'))
     elif options.manually_grab_svn_rev:
       new_command.append('--force')
       if command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
         new_command.extend(('--accept', 'postpone'))
     elif command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
       new_command.extend(('--accept', 'postpone'))
     return new_command