Fix evil retry loop on poll errors.

appengine/swarming/swarming_bot/bot_code/bot_main.py

 # Copyright 2013 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Swarming bot main process.
 
 This is the program that communicates with the Swarming server, ensures the code
 is always up to date and executes a child process to run tasks and upload
 results back.
 
@@ skipping 25 matching lines @@
 import bot_auth
 import common
 import file_refresher
 import remote_client
 import remote_client_errors
 import singleton
 from api import bot
 from api import os_utilities
 from api import platforms
 from utils import file_path
-from utils import net
 from utils import on_error
 from utils import subprocess42
 from utils import zip_package
 
 
 # Used to opportunistically set the error handler to notify the server when the
 # process exits due to an exception.
 _ERROR_HANDLER_WAS_REGISTERED = False
 
 
@@ skipping 487 matching lines @@
       try:
         botobj._update_dimensions(get_dimensions(botobj))
         botobj._update_state(get_state(botobj, consecutive_sleeps))
         did_something = poll_server(botobj, quit_bit, last_action)
         if did_something:
           last_action = time.time()
           consecutive_sleeps = 0
         else:
           consecutive_sleeps += 1
       except Exception as e:
-        logging.exception('poll_server failed')
+        logging.exception('poll_server failed in a completely unexpected way')

[Vadim Sh., 2016/12/21 00:27:16]

:) 
in theory, this should never happen.

         msg = '%s\n%s' % (e, traceback.format_exc()[-2048:])
         botobj.post_error(msg)
         consecutive_sleeps = 0
+        # Sleep a bit as a precaution to avoid hammering the server.
+        quit_bit.wait(10)
     logging.info('Quitting')
 
   # Tell the server we are going away.
   botobj.post_event('bot_shutdown', 'Signal was received')
   return 0
 
 
 def poll_server(botobj, quit_bit, last_action):
   """Polls the server to run one loop.
 
   Returns True if executed some action, False if server asked the bot to sleep.
   """
-  # Access to a protected member _XXX of a client class - pylint: disable=W0212
   start = time.time()
-  cmd, value = botobj.remote.poll(botobj._attributes)
-  if cmd == '':

[Vadim Sh., 2016/12/21 00:27:16]

on errors, cmd was None here. None != '' => no sleep :(

+  try:
+    cmd, value = botobj.remote.poll(botobj._attributes)
+  except remote_client_errors.PollError as e:
     # Back off on failure.
-    time.sleep(max(1, min(60, botobj.state.get('sleep_streak', 10) * 2)))
+    delay = max(1, min(60, botobj.state.get('sleep_streak', 10) * 2))
+    logging.warning('Poll failed (%s), sleeping %.1f sec', e, delay)
+    quit_bit.wait(delay)
     return False
   logging.debug('Server response:\n%s: %s', cmd, value)
 
   if cmd == 'sleep':
     # Value is duration
     call_hook(botobj, 'on_bot_idle', max(0, time.time() - last_action))
     quit_bit.wait(value)
     return False
 
   if cmd == 'terminate':
@@ skipping 363 matching lines @@
 
   error = None
   if len(args) != 0:
     error = 'Unexpected arguments: %s' % args
   try:
     return run_bot(error)
   finally:
     call_hook(
         bot.Bot(None, None, None, None, base_dir, None), 'on_bot_shutdown')
     logging.info('main() returning')