Only schedule termination task if an unconnected MP bot is dead

appengine/swarming/server/lease_management.py

 # Copyright 2016 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Lease management for machines leased from the Machine Provider.
 
 Keeps a list of machine types which should be leased from the Machine Provider
 and the list of machines of each type currently leased.
 
 Swarming integration with Machine Provider
@@ skipping 888 matching lines @@
           (event.ts - machine_lease.instruction_ts).total_seconds(),
           fields={
               'machine_type': machine_lease.machine_type.id(),
           },
       )
       return
 
   # The bot hasn't connected yet. If it's dead or missing, release the lease.
   # At this point we have sent the connection instruction so the bot could still
   # connect after we release the lease but before Machine Provider actually
-  # deletes the bot. Therefore we also schedule a termination task. If the bot
-  # connects, it will just shut itself down immediately.
-  task_scheduler.schedule_request(
-      task_request.create_termination_task(machine_lease.hostname, True),
-      None,
-      check_acls=False,
-  )
+  # deletes the bot. Therefore we also schedule a termination task if releasing
+  # the bot. That way, if the bot connects, it will just shut itself down.
   bot_info = bot_management.get_info_key(machine_lease.hostname).get()
   if not bot_info:
     logging.error(
         'BotInfo missing:\nKey: %s\nHostname: %s',
         machine_lease.key,
         machine_lease.hostname,
     )
+    task_scheduler.schedule_request(
+        task_request.create_termination_task(machine_lease.hostname, True),
+        None,
+        check_acls=False,
+    )
     if release(machine_lease):
       clear_lease_request(machine_lease.key, machine_lease.client_request_id)
     return
   if bot_info.is_dead(utils.utcnow()):
     logging.warning(
         'Bot failed to connect in time:\nKey: %s\nHostname: %s',
         machine_lease.key,
         machine_lease.hostname,
     )
+    task_scheduler.schedule_request(
+        task_request.create_termination_task(machine_lease.hostname, True),
+        None,
+        check_acls=False,
+    )
     if release(machine_lease):
       cleanup_bot(machine_lease)
 
 
 def cleanup_bot(machine_lease):
   """Cleans up entities after a bot is removed."""
   task_queues.cleanup_after_bot(machine_lease.hostname)
   bot_management.get_info_key(machine_lease.hostname).delete()
   clear_lease_request(machine_lease.key, machine_lease.client_request_id)
 
@@ skipping 358 matching lines @@
           target_fields=ts_mon_metrics.TARGET_FIELDS,
       )
       ts_mon_metrics.machine_types_actual_size.set(
           utilization.idle,
           fields={
               'busy': False,
               'machine_type': machine_type.key.id(),
           },
           target_fields=ts_mon_metrics.TARGET_FIELDS,
       )