Switch bot_update to use exponential backoff on clone

recipe_modules/bot_update/resources/bot_update.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # TODO(hinoka): Use logging.
 
 import cStringIO
 import codecs
 import collections
@@ skipping 525 matching lines @@
   # Before we do anything, break all git_cache locks.
   if path.isdir(git_cache_dir):
     git('cache', 'unlock', '-vv', '--force', '--all',
         '--cache-dir', git_cache_dir)
     for item in os.listdir(git_cache_dir):
       filename = os.path.join(git_cache_dir, item)
       if item.endswith('.lock'):
         raise Exception('%s exists after cache unlock' % filename)
   first_solution = True
   for sln in solutions:
-    # This is so we can loop back and try again if we need to wait for the
-    # git mirrors to update from SVN.
+    # Just in case we're hitting a different git server than the one from
+    # which the target revision was polled, we retry some.
     done = False
-    tries_left = 60
+    deadline = time.time() + 60  # One minute (5 tries with exp. backoff).
+    tries = 0
     while not done:
       name = sln['name']
       url = sln['url']
       if url == CHROMIUM_SRC_URL or url + '.git' == CHROMIUM_SRC_URL:
         # Experiments show there's little to be gained from
         # a shallow clone of src.
         shallow = False
       sln_dir = path.join(build_dir, name)
       s = ['--shallow'] if shallow else []
       populate_cmd = (['cache', 'populate', '--ignore_locks', '-v',
@@ skipping 15 matching lines @@
           git('fetch', 'origin', cwd=sln_dir)
         for ref in refs:
           refspec = '%s:%s' % (ref, ref.lstrip('+'))
           git('fetch', 'origin', refspec, cwd=sln_dir)
 
         revision = get_target_revision(name, url, revisions) or 'HEAD'
         force_revision(sln_dir, revision)
         done = True
       except SubprocessFailed as e:
         # Exited abnormally, theres probably something wrong.
+        print 'Something failed: %s.' % str(e)
+
+        if time.time() > deadline:
+          overrun = time.time() - deadline
+          print 'Ran %s seconds past deadline. Aborting.' % overrun
+          raise
+
         # Lets wipe the checkout and try again.
-        tries_left -= 1
-        if tries_left > 0:
-          print 'Something failed: %s.' % str(e)
-          print 'waiting 5 seconds and trying again...'
-          time.sleep(5)
-        else:
-          raise
+        tries += 1
+        sleep_secs = 2**tries
+        print 'waiting %s seconds and trying again...' % sleep_secs
+        time.sleep(sleep_secs)
         remove(sln_dir)
 
     git('clean', '-dff', cwd=sln_dir)
 
     if first_solution:
       git_ref = git('log', '--format=%H', '--max-count=1',
                     cwd=sln_dir).strip()
     first_solution = False
   return git_ref
 
@@ skipping 587 matching lines @@
   except Exception:
     # Unexpected failure.
     emit_flag(options.flag_file)
     raise
   else:
     emit_flag(options.flag_file)
 
 
 if __name__ == '__main__':
   sys.exit(main())