[Findit] Record error if swarming task completes with exit_code as 2.

appengine/findit/waterfall/process_base_swarming_task_result_pipeline.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from collections import defaultdict
 import datetime
 import logging
 import time
 
 from common.http_client_appengine import HttpClientAppengine as HttpClient
@@ skipping 68 matching lines @@
     task_started = False
     task_completed = False
     tests_statuses = {}
     step_name_no_platform = None
 
     while not task_completed:
       # Keeps monitoring the swarming task, waits for it to complete.
       data = swarming_util.GetSwarmingTaskResultById(
           task_id, self.HTTP_CLIENT)
       task_state = data['state']
+      exit_code = (None if task_state != swarming_util.STATE_COMPLETED
+                   else data['exit_code'])

[lijeffrey, 2016/09/07 20:41:38]

how about the opposite for readability?

exit_code = data['exit_code'] if task_state == swarming_util.STATE_COMPLETED
else None

[chanli, 2016/09/07 22:37:11]

Done.

       step_name_no_platform = swarming_util.GetTagValue(
           data.get('tags', {}), 'ref_name')
       if task_state not in swarming_util.STATES_RUNNING:
         task_completed = True
         task = self._GetSwarmingTask(*call_args)
-        if task_state == swarming_util.STATE_COMPLETED:
+        if task_state == swarming_util.STATE_COMPLETED and int(exit_code) < 2:

[lijeffrey, 2016/09/07 20:41:38]

why 2? Is there something special about anything < 2? How about explicitly
checking for specific error codes defined in constants to make the code more
clear/robust?

[chanli, 2016/09/07 22:37:11]

exit_code == 0 means all tests passed; exit_code == 1 means some tests failed;
exit_code == 2 means the task stopped unexpectedly because of some error, where
we'll not get any failure logs for tests - in fact, no log for any tests at all.

           outputs_ref = data.get('outputs_ref')
           output_json = swarming_util.GetSwarmingTaskFailureLog(
               outputs_ref, self.HTTP_CLIENT)
           tests_statuses = self._CheckTestsRunStatuses(
               output_json, *call_args)
           task.status = analysis_status.COMPLETED
           task.tests_statuses = tests_statuses
         else:
           task.status = analysis_status.ERROR
-          logging.error('Swarming task stopped with status: %s' % (
-              task_state))
+          logging.error('Swarming task stopped with status: %s and'
+                        ' exit_code: %s' % (task_state, exit_code))
         priority_str = swarming_util.GetTagValue(
             data.get('tags', {}), 'priority')
         if priority_str:
           task.parameters['priority'] = int(priority_str)
         task.put()
       else:  # pragma: no cover
         if task_state == 'RUNNING' and not task_started:
           # swarming task just starts, update status.
           task_started = True
           task = self._GetSwarmingTask(*call_args)
           task.status = analysis_status.RUNNING
           task.put()
         time.sleep(server_query_interval_seconds)
       if time.time() > deadline:
         # Updates status as ERROR.
         task = self._GetSwarmingTask(*call_args)
         task.status = analysis_status.ERROR
         task.put()
         logging.error('Swarming task timed out after %d hours.' % timeout_hours)
         break  # Stops the loop and return.
     # Update swarming task metadate.
     task = self._GetSwarmingTask(*call_args)
     task.created_time = self._ConvertDateTime(data.get('created_ts'))
     task.started_time = self._ConvertDateTime(data.get('started_ts'))
     task.completed_time = self._ConvertDateTime(data.get('completed_ts'))
     task.put()
 
     return step_name, step_name_no_platform