Add exponential backoff to bot_update.

scripts/slave/bot_update.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # TODO(hinoka): Use logging.
 
 import cStringIO
 import codecs
 import collections
 import copy
 import ctypes
 import json
 import optparse
 import os
 import pprint
+import random
 import re
 import socket
 import subprocess
 import sys
 import tempfile
 import time
 import urllib2
 import urlparse
 import uuid
 
@@ skipping 381 matching lines @@
     result = result_fn(code, outval)
     if result in (FAIL, RETRY):
       print '===Failed in %.1f mins===' % elapsed_time
       print
     else:
       print '===Succeeded in %.1f mins===' % elapsed_time
       print
       return outval
     if result is FAIL:
       break
+    if result is RETRY:
+      sleep_backoff = 4 ** attempt
+      sleep_time = random.randint(sleep_backoff, int(sleep_backoff * 1.2))
+      print '===backing off, sleeping for %d secs===' % sleep_time
+      time.sleep(sleep_time)
 
   raise SubprocessFailed('%s failed with code %d in %s after %d attempts.' %
                          (' '.join(args), code, cwd, attempt),
                          code, outval)
 
 
 def git(*args, **kwargs):  # pragma: no cover
   """Wrapper around call specifically for Git commands."""
   if args and args[0] == 'cache':
     # Rewrite "git cache" calls into "python git_cache.py".
@@ skipping 1191 matching lines @@
   except Exception:
     # Unexpected failure.
     emit_flag(options.flag_file)
     raise
   else:
     emit_flag(options.flag_file)
 
 
 if __name__ == '__main__':
   sys.exit(main())