[Findit] Implement retry within swarming_util.py when making server calls

appengine/findit/waterfall/test/process_base_swarming_task_result_pipeline_test.py

Index: appengine/findit/waterfall/test/process_base_swarming_task_result_pipeline_test.py
diff --git a/appengine/findit/waterfall/test/process_base_swarming_task_result_pipeline_test.py b/appengine/findit/waterfall/test/process_base_swarming_task_result_pipeline_test.py
index 3fd7a92c03bd1ca70bade39213ce8d78d7abf1bc..9cacc3a4f647f16c055dd7f4f415b5a9793fb8c1 100644
--- a/appengine/findit/waterfall/test/process_base_swarming_task_result_pipeline_test.py
+++ b/appengine/findit/waterfall/test/process_base_swarming_task_result_pipeline_test.py
@@ -147,9 +147,6 @@ class ProcessBaseSwarmingTaskResultPipelineTest(wf_testcase.WaterfallTestCase):
   def _MockedGetSwarmingTaskResultById(self, task_id, _):
     return _SWARMING_TASK_RESULTS[task_id], None
 
-  def _MockedGetSwarmingTaskFailureLog(self, *_):
-    return _SAMPLE_FAILURE_LOG, None
-
   def setUp(self):
     super(ProcessBaseSwarmingTaskResultPipelineTest, self).setUp()
     self.pipeline = ProcessBaseSwarmingTaskResultPipeline()
@@ -260,6 +257,42 @@ class ProcessBaseSwarmingTaskResultPipelineTest(wf_testcase.WaterfallTestCase):
     self.assertEqual(analysis_status.ERROR, task.status)
     self.assertEqual(task.error, {'code': 1, 'message': 'error'})
 
+  @mock.patch.object(swarming_util, 'GetSwarmingTaskResultById',
+                     return_value=(_SWARMING_TASK_RESULTS['task_id1'],
+                                   {'code': 1, 'message': 'error'}))
+  @mock.patch.object(swarming_util, 'GetSwarmingTaskFailureLog',
+                     return_value=(_SAMPLE_FAILURE_LOG, None))
+  def testMonitorSwarmingTaskGetSwarmingTaskResultIdErrorRecovered(self, *_):
+    task = WfSwarmingTask.Create(
+        self.master_name, self.builder_name,
+        self.build_number, self.step_name)
+    task.task_id = 'task_id1'
+    task.put()
+
+    pipeline = ProcessSwarmingTaskResultPipeline()
+    pipeline.run(
+        self.master_name, self.builder_name, self.build_number, self.step_name)
+
+    self.assertEqual(analysis_status.COMPLETED, task.status)
+    self.assertEqual(task.error, {'code': 1, 'message': 'error'})
+
+  @mock.patch.object(swarming_util, 'GetSwarmingTaskFailureLog',
+                     return_value=(_SAMPLE_FAILURE_LOG,
+                                   {'code': 1, 'message': 'error'}))
+  def testMonitorSwarmingTaskGetSwarmingTaskFailureLogErrorRecovered(self, _):
+    task = WfSwarmingTask.Create(
+        self.master_name, self.builder_name,
+        self.build_number, self.step_name)
+    task.task_id = 'task_id1'
+    task.put()
+
+    pipeline = ProcessSwarmingTaskResultPipeline()
+    pipeline.run(
+        self.master_name, self.builder_name, self.build_number, self.step_name)
+
+    self.assertEqual(analysis_status.COMPLETED, task.status)
+    self.assertEqual(task.error, {'code': 1, 'message': 'error'})
+
   @mock.patch.object(swarming_util, 'GetSwarmingTaskFailureLog',
                      return_value=(None, {'code': 1, 'message': 'error'}))
   def testMonitorSwarmingTaskGetSwarmingTaskFailureLogError(self, _):
@@ -296,11 +329,10 @@ class ProcessBaseSwarmingTaskResultPipelineTest(wf_testcase.WaterfallTestCase):
                 swarming_util.TASK_FAILED]
         })
 
-  def testProcessSwarmingTaskResultPipeline(self):
+  @mock.patch.object(swarming_util, 'GetSwarmingTaskFailureLog',
+                     return_value=(_SAMPLE_FAILURE_LOG, None))
+  def testProcessSwarmingTaskResultPipeline(self, _):
     # End to end test.
-    self.mock(swarming_util, 'GetSwarmingTaskFailureLog',
-              self._MockedGetSwarmingTaskFailureLog)
-
     task = WfSwarmingTask.Create(
         self.master_name, self.builder_name,
         self.build_number, self.step_name)