[Findit] Record error if swarming task completes with exit_code as 2.

appengine/findit/waterfall/process_base_swarming_task_result_pipeline.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from collections import defaultdict
 import datetime
 import logging
 import time
 
 from common.http_client_appengine import HttpClientAppengine as HttpClient
 from common.pipeline_wrapper import BasePipeline
 from model import analysis_status
 from waterfall import swarming_util
 from waterfall import waterfall_config
 
 
+_ACCEPTED_EXIT_CODE = [0, 1]

[lijeffrey, 2016/09/08 03:41:00]

nit: _ACCEPTED_EXIT_CODES

on that note, how about moving these error codes explicit constants for each
type of error (ALL_TESTS_PASSED = 0, SOME_TESTS_FAILED = 1, etc) to a separate
module kind of like analysis_status.py, or even define them as constants here,
and include a dict that maps codes to messages? Then include the error message
in the log, not just status and code. 

[stgao, 2016/09/08 04:16:00]

+1
We also need comments for possible exit codes, and some explanation for them.

[chanli, 2016/09/08 21:16:56]

Done.

[chanli, 2016/09/08 21:16:56]

Since the name for each exit_code is self_explained, I didn't add comments here.

+
+
 class ProcessBaseSwarmingTaskResultPipeline(BasePipeline):
   """A pipeline for monitoring swarming task and processing task result.
 
   This pipeline waits for result for a swarming task and processes the result to
   generate a dict for statuses for each test run.
   """
 
   HTTP_CLIENT = HttpClient()
 
   def _CheckTestsRunStatuses(self, output_json):
@@ skipping 52 matching lines @@
     task_started = False
     task_completed = False
     tests_statuses = {}
     step_name_no_platform = None
 
     while not task_completed:
       # Keeps monitoring the swarming task, waits for it to complete.
       data = swarming_util.GetSwarmingTaskResultById(
           task_id, self.HTTP_CLIENT)
       task_state = data['state']
+      exit_code = (data['exit_code'] if
+                   task_state == swarming_util.STATE_COMPLETED else None)
       step_name_no_platform = swarming_util.GetTagValue(
           data.get('tags', {}), 'ref_name')
       if task_state not in swarming_util.STATES_RUNNING:
         task_completed = True
         task = self._GetSwarmingTask(*call_args)
-        if task_state == swarming_util.STATE_COMPLETED:
+        if (task_state == swarming_util.STATE_COMPLETED and
+                          int(exit_code) in _ACCEPTED_EXIT_CODE):
           outputs_ref = data.get('outputs_ref')
           output_json = swarming_util.GetSwarmingTaskFailureLog(
               outputs_ref, self.HTTP_CLIENT)
           tests_statuses = self._CheckTestsRunStatuses(
               output_json, *call_args)
           task.status = analysis_status.COMPLETED
           task.tests_statuses = tests_statuses
         else:
           task.status = analysis_status.ERROR
-          logging.error('Swarming task stopped with status: %s' % (
-              task_state))
+          logging.error('Swarming task stopped with status: %s and'
+                        ' exit_code: %s' % (task_state, exit_code))
         priority_str = swarming_util.GetTagValue(
             data.get('tags', {}), 'priority')
         if priority_str:
           task.parameters['priority'] = int(priority_str)
         task.put()
       else:  # pragma: no cover
         if task_state == 'RUNNING' and not task_started:
           # swarming task just starts, update status.
           task_started = True
           task = self._GetSwarmingTask(*call_args)
           task.status = analysis_status.RUNNING
           task.put()
         time.sleep(server_query_interval_seconds)
       if time.time() > deadline:
         # Updates status as ERROR.
         task = self._GetSwarmingTask(*call_args)
         task.status = analysis_status.ERROR
         task.put()
         logging.error('Swarming task timed out after %d hours.' % timeout_hours)
         break  # Stops the loop and return.
     # Update swarming task metadate.
     task = self._GetSwarmingTask(*call_args)
     task.created_time = self._ConvertDateTime(data.get('created_ts'))
     task.started_time = self._ConvertDateTime(data.get('started_ts'))
     task.completed_time = self._ConvertDateTime(data.get('completed_ts'))
     task.put()
 
     return step_name, step_name_no_platform