[Findit] Implement retry within swarming_util.py when making server calls

appengine/findit/waterfall/process_base_swarming_task_result_pipeline.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from collections import defaultdict
 import datetime
 import logging
 import time
 
 from common.http_client_appengine import HttpClientAppengine as HttpClient
@@ skipping 69 matching lines @@
     task_started = False
     task_completed = False
     step_name_no_platform = None
     task = self._GetSwarmingTask(*call_args)
 
     while not task_completed:
       data, error = swarming_util.GetSwarmingTaskResultById(
           task_id, self.HTTP_CLIENT)
 
       if error:
-        # An error occurred when trying to contact the swarming server.
-        task.status = analysis_status.ERROR
+        # An error occurred at some point when trying to retrieve data from
+        # the swarming server, even if eventually successful.
         task.error = error
         task.put()
-        break
+
+        if not data:
+          # Even after retry, no data was recieved.
+          task.status = analysis_status.ERROR
+          break
 
       task_state = data['state']
       exit_code = (data.get('exit_code') if
                    task_state == swarming_util.STATE_COMPLETED else None)
       step_name_no_platform = (
           step_name_no_platform or swarming_util.GetTagValue(
               data.get('tags', {}), 'ref_name'))
 
       if task_state not in swarming_util.STATES_RUNNING:
         task_completed = True
 
         if (task_state == swarming_util.STATE_COMPLETED and
             int(exit_code) != swarming_util.TASK_FAILED):
           outputs_ref = data.get('outputs_ref')
 
           # If swarming task aborted because of errors in request arguments,
           # it's possible that there is no outputs_ref.
           if not outputs_ref:
             task.status = analysis_status.ERROR
             task.error = {
-              'code': swarming_util.NO_TASK_OUTPUTS,
-              'message': 'outputs_ref is None'
+                'code': swarming_util.NO_TASK_OUTPUTS,
+                'message': 'outputs_ref is None'
             }
             task.put()
             break
 
           output_json, error = swarming_util.GetSwarmingTaskFailureLog(
               outputs_ref, self.HTTP_CLIENT)
 
+          task.status = analysis_status.COMPLETED
+
           if error:
-            task.status = analysis_status.ERROR
             task.error = error
-          else:
-            task.status = analysis_status.COMPLETED
+
+            if not output_json:
+              # Retry was ultimately unsuccessful.
+              task.status = analysis_status.ERROR
 
           tests_statuses = self._CheckTestsRunStatuses(output_json, *call_args)
           task.tests_statuses = tests_statuses
           task.put()
         else:
           if exit_code is not None:
             # Swarming task completed, but the task failed.
             code = int(exit_code)
             message = swarming_util.EXIT_CODE_DESCRIPTIONS[code]
           else:
@@ skipping 64 matching lines @@
       task_id (str): The task id to query the swarming server on the progresss
         of a swarming task.
 
     Returns:
       A dict of lists for reliable/flaky tests.
     """
     call_args = self._GetArgs(master_name, builder_name, build_number,
                               step_name, *args)
     step_name_no_platform = self._MonitorSwarmingTask(task_id, *call_args)
     return step_name, step_name_no_platform