Fixes and add smoke test: hard timeout on isolated task.

appengine/swarming/swarming_bot/bot_code/task_runner.py

 # Copyright 2013 The Swarming Authors. All rights reserved.
 # Use of this source code is governed by the Apache v2.0 license that can be
 # found in the LICENSE file.
 
 """Runs a Swarming task.
 
 Downloads all the necessary files to run the task, executes the command and
 streams results back to the Swarming server.
 
 The process exit code is 0 when the task was executed, even if the task itself
@@ skipping 327 matching lines @@
         u'must_signal_internal_failure': None,
         u'version': OUT_VERSION,
       }
 
     output_chunk_start = 0
     stdout = ''
     exit_code = None
     had_hard_timeout = False
     had_io_timeout = False
     must_signal_internal_failure = None
+    kill_sent = False
     timed_out = None
     try:
       calc = lambda: calc_yield_wait(
           task_details, start, last_io, timed_out, stdout)
       maxsize = lambda: MAX_CHUNK_SIZE - len(stdout)
       last_io = monotonic_time()
       for _, new_data in proc.yield_any(maxsize=maxsize, soft_timeout=calc):
         now = monotonic_time()
         if new_data:
           stdout += new_data
           last_io = now
 
         # Post update if necessary.
         if should_post_update(stdout, now, last_packet):
           last_packet = monotonic_time()
           params['cost_usd'] = (
               cost_usd_hour * (last_packet - task_start) / 60. / 60.)
           post_update(swarming_server, params, None, stdout, output_chunk_start)
           output_chunk_start += len(stdout)
           stdout = ''
 
         # Send signal on timeout if necessary. Both are failures, not
         # internal_failures.
         # Eventually kill but return 0 so bot_main.py doesn't cancel the task.
         if not timed_out:
           if now - last_io > task_details.io_timeout:
             had_io_timeout = True
-            logging.warning('I/O timeout')
-            try:
-              proc.terminate()
-            except OSError:
-              pass
+            logging.warning('I/O timeout; sending SIGTERM')
+            proc.terminate()
             timed_out = monotonic_time()
           elif now - start > task_details.hard_timeout:
             had_hard_timeout = True
-            logging.warning('Hard timeout')
-            try:
-              proc.terminate()
-            except OSError:
-              pass
+            logging.warning('Hard timeout; sending SIGTERM')
+            proc.terminate()
             timed_out = monotonic_time()
         else:
           # During grace period.
-          if now >= timed_out + task_details.grace_period:
+          if not kill_sent and now >= timed_out + task_details.grace_period:
             # Now kill for real. The user can distinguish between the following
             # states:
             # - signal but process exited within grace period,
             #   (hard_|io_)_timed_out will be set but the process exit code will
             #   be script provided.
             # - processed exited late, exit code will be -9 on posix.
-            try:
-              logging.warning('proc.kill() after grace')
-              proc.kill()
-            except OSError:
-              pass
+            logging.warning('Grace exhausted; sending SIGKILL')
+            proc.kill()
+            kill_sent = True
       logging.info('Waiting for proces exit')
       exit_code = proc.wait()
     except MustExit as e:
       # TODO(maruel): Do the send SIGTERM to child process and give it
       # task_details.grace_period to terminate.
       must_signal_internal_failure = (
           u'task_runner received signal %s' % e.signal)
       exit_code = kill_and_wait(proc, 'signal %d' % e.signal)
     except (IOError, OSError):
       # Something wrong happened, try to kill the child process.
@@ skipping 71 matching lines @@
   if options.start > now:
     options.start = now
 
   try:
     load_and_run(
         options.in_file, remote, options.cost_usd_hour, options.start,
         options.out_file)
     return 0
   finally:
     logging.info('quitting')