Add exponential backoff to bot_update.

scripts/slave/bot_update.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # TODO(hinoka): Use logging.
 
 import cStringIO
 import codecs
 import collections
@@ skipping 397 matching lines @@
     result = result_fn(code, outval)
     if result in (FAIL, RETRY):
       print '===Failed in %.1f mins===' % elapsed_time
       print
     else:
       print '===Succeeded in %.1f mins===' % elapsed_time
       print
       return outval
     if result is FAIL:
       break
+    if attempt > 0:

[agable, 2015/07/24 02:37:22]

attempt is always > 0, it starts at 1. I think you want "if result == RETRY:"

+      sleep_time = 4 ** attempt

[agable, 2015/07/24 02:37:22]

jitter by +/- 20%?

+      print '===backing off, sleeping for %ds===' % sleep_time
+      time.sleep(sleep_time)
 
   raise SubprocessFailed('%s failed with code %d in %s after %d attempts.' %
                          (' '.join(args), code, cwd, attempt),
                          code, outval)
 
 
 def git(*args, **kwargs):  # pragma: no cover
   """Wrapper around call specifically for Git commands."""
   if args and args[0] == 'cache':
     # Rewrite "git cache" calls into "python git_cache.py".
@@ skipping 1191 matching lines @@
   except Exception:
     # Unexpected failure.
     emit_flag(options.flag_file)
     raise
   else:
     emit_flag(options.flag_file)
 
 
 if __name__ == '__main__':
   sys.exit(main())