Significant refactoring of run_isolated.

appengine/swarming/swarming_bot/bot_code/task_runner.py

 # Copyright 2013 The Swarming Authors. All rights reserved.
 # Use of this source code is governed by the Apache v2.0 license that can be
 # found in the LICENSE file.
 
 """Runs a Swarming task.
 
 Downloads all the necessary files to run the task, executes the command and
 streams results back to the Swarming server.
 
 The process exit code is 0 when the task was executed, even if the task itself
@@ skipping 394 matching lines @@
 
     # This is the very last packet for this command. It if was an isolated task,
     # include the output reference to the archived .isolated file.
     now = monotonic_time()
     params['cost_usd'] = cost_usd_hour * (now - task_start) / 60. / 60.
     params['duration'] = now - start
     params['io_timeout'] = had_io_timeout
     params['hard_timeout'] = had_hard_timeout
     if isolated_result:
       try:
+        # See run_isolated.py for the format.
         with open(isolated_result, 'rb') as f:
-          params['outputs_ref'] = json.load(f)
+          run_isolated_result = json.load(f)
+        logging.debug('run_isolated:\n%s', run_isolated_result)
+        # TODO(maruel): Grab statistics (cache hit rate, data downloaded,
+        # mapping time, etc) from run_isolated and push them to the server.
+        params['outputs_ref'] = run_isolated_result['outputs_ref']
+        if run_isolated_result['internal_failure']:
+          must_signal_internal_failure = run_isolated_result['internal_failure']
+          logging.error('%s', must_signal_internal_failure)
+        elif exit_code:
+          # TODO(maruel): Grab stdout from run_isolated.
+          must_signal_internal_failure = (
+              'run_isolated internal failure %d' % exit_code)
+          logging.error('%s', must_signal_internal_failure)
+        exit_code = run_isolated_result['exit_code']
       except (IOError, OSError, ValueError) as e:
-        logging.error('Swallowing error: %s' % e)
+        logging.error('Swallowing error: %s', e)
+    # TODO(maruel): Send the internal failure here instead of sending it through
+    # bot_main, this causes a race condition.
     post_update(swarming_server, params, exit_code, stdout, output_chunk_start)
     return {
       u'exit_code': exit_code,
       u'hard_timeout': had_hard_timeout,
       u'io_timeout': had_io_timeout,
       u'must_signal_internal_failure': must_signal_internal_failure,
       u'version': OUT_VERSION,
     }
   finally:
     if isolated_result:
@@ skipping 27 matching lines @@
   if options.start > now:
     options.start = now
 
   try:
     load_and_run(
         options.in_file, remote, options.cost_usd_hour, options.start,
         options.out_file)
     return 0
   finally:
     logging.info('quitting')