swarming: Back off requests on poll failure.

appengine/swarming/swarming_bot/bot_code/bot_main.py

 # Copyright 2013 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Swarming bot main process.
 
 This is the program that communicates with the Swarming server, ensures the code
 is always up to date and executes a child process to run tasks and upload
 results back.
 
@@ skipping 500 matching lines @@
 def poll_server(botobj, quit_bit):
   """Polls the server to run one loop.
 
   Returns True if executed some action, False if server asked the bot to sleep.
   """
   # Access to a protected member _XXX of a client class - pylint: disable=W0212
   start = time.time()
   resp = botobj.remote.url_read_json(
       '/swarming/api/v1/bot/poll', data=botobj._attributes)
   if not resp:
+    # Back off on failure.
+    time.sleep(max(1, min(60, botobj.state.get('sleep_streak', 10) * 2)))
     return False
   logging.debug('Server response:\n%s', resp)
 
   cmd = resp['cmd']
   if cmd == 'sleep':
     quit_bit.wait(resp['duration'])
     return False
 
   if cmd == 'terminate':
     quit_bit.set()
@@ skipping 339 matching lines @@
 
   error = None
   if len(args) != 0:
     error = 'Unexpected arguments: %s' % args
   try:
     return run_bot(error)
   finally:
     call_hook(bot.Bot(None, None, None, None, os.path.dirname(THIS_FILE), None),
               'on_bot_shutdown')
     logging.info('main() returning')