gclient: exponential backoff of SCM commands on known failure types....

gclient_scm.py

 # Copyright (c) 2010 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Gclient-specific SCM-specific operations."""
 
 import logging
 import os
 import posixpath
 import re
@@ skipping 238 matching lines @@
       if not upstream_branch or not upstream_branch.startswith('refs/remotes'):
         current_type = "hash"
         logging.debug("Current branch is not tracking an upstream (remote)"
                       " branch.")
       elif upstream_branch.startswith('refs/remotes'):
         current_type = "branch"
       else:
         raise gclient_utils.Error('Invalid Upstream: %s' % upstream_branch)
 
     # Update the remotes first so we have all the refs.
+    backoff_time = 5
     for _ in range(10):
       try:
         remote_output, remote_err = scm.GIT.Capture(
             ['remote'] + verbose + ['update'],
             self.checkout_path,
             print_error=False)
         break
       except gclient_utils.CheckCallError, e:
         # Hackish but at that point, git is known to work so just checking for
         # 502 in stderr should be fine.
         if '502' in e.stderr:
           options.stdout.write(str(e) + '\n')
-          options.stdout.write('Sleeping 15 seconds and retrying...\n')
-          time.sleep(15)
+          options.stdout.write('Sleeping %.1f seconds and retrying...\n' %
+                               backoff_time)
+          time.sleep(backoff_time)
+          backoff_time *= 1.3
           continue
         raise
 
     if verbose:
       options.stdout.write(remote_output.strip() + '\n')
       # git remote update prints to stderr when used with --verbose
       options.stdout.write(remote_err.strip() + '\n')
 
     # This is a big hammer, debatable if it should even be here...
     if options.force or options.reset:
@@ skipping 692 matching lines @@
 
     This method returns a new list to be used as a command."""
     new_command = command[:]
     if revision:
       new_command.extend(['--revision', str(revision).strip()])
     # --force was added to 'svn update' in svn 1.5.
     if ((options.force or options.manually_grab_svn_rev) and
         scm.SVN.AssertVersion("1.5")[0]):
       new_command.append('--force')
     return new_command