[Findit] Implement retry within swarming_util.py when making server calls

appengine/findit/waterfall/test/process_base_swarming_task_result_pipeline_test.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import datetime
 import mock
 
 from model import analysis_status
 from model.flake.flake_swarming_task import FlakeSwarmingTask
 from model.flake.master_flake_analysis import MasterFlakeAnalysis
@@ skipping 98 matching lines @@
         'completed_ts': '2016-02-10T18:33:09'
     },
     'task_id2': {
         'state': 'TIMED_OUT',
         'outputs_ref': None
     },
     'task_id3': {
         'state': 'COMPLETED',
         'exit_code': '2',  # Swarming task failed.
     },
-  'task_id4': {
-    'state': 'COMPLETED',
-    'exit_code': '1',
-  },
+    'task_id4': {
+        'state': 'COMPLETED',
+        'exit_code': '1',
+    },
 }
 
 
 _EXPECTED_TESTS_STATUS = {
     'TestSuite1.test1': {
         'total_run': 2,
         'SUCCESS': 2
     },
     'TestSuite1.test2': {
         'total_run': 4,
@@ skipping 11 matching lines @@
     'flaky_tests': ['TestSuite1.test2', 'TestSuite1.test1'],
     'reliable_tests': ['TestSuite1.test3']
 }
 
 
 class ProcessBaseSwarmingTaskResultPipelineTest(wf_testcase.WaterfallTestCase):
 
   def _MockedGetSwarmingTaskResultById(self, task_id, _):
     return _SWARMING_TASK_RESULTS[task_id], None
 
-  def _MockedGetSwarmingTaskFailureLog(self, *_):
-    return _SAMPLE_FAILURE_LOG, None
-
   def setUp(self):
     super(ProcessBaseSwarmingTaskResultPipelineTest, self).setUp()
     self.pipeline = ProcessBaseSwarmingTaskResultPipeline()
     self.master_name = 'm'
     self.builder_name = 'b'
     self.build_number = 121
     self.step_name = 'abc_tests on platform'
     self.test_name = 'test'
     self.mock(swarming_util, 'GetSwarmingTaskResultById',
               self._MockedGetSwarmingTaskResultById)
@@ skipping 90 matching lines @@
     task.task_id = 'task_id2'
     task.put()
 
     pipeline = ProcessSwarmingTaskResultPipeline()
     pipeline.run(
         self.master_name, self.builder_name, self.build_number, self.step_name)
 
     self.assertEqual(analysis_status.ERROR, task.status)
     self.assertEqual(task.error, {'code': 1, 'message': 'error'})
 
+  @mock.patch.object(swarming_util, 'GetSwarmingTaskResultById',
+                     return_value=(_SWARMING_TASK_RESULTS['task_id1'],
+                                   {'code': 1, 'message': 'error'}))
+  @mock.patch.object(swarming_util, 'GetSwarmingTaskFailureLog',
+                     return_value=(_SAMPLE_FAILURE_LOG, None))
+  def testMonitorSwarmingTaskGetSwarmingTaskResultIdErrorRecovered(self, *_):
+    task = WfSwarmingTask.Create(
+        self.master_name, self.builder_name,
+        self.build_number, self.step_name)
+    task.task_id = 'task_id1'
+    task.put()
+
+    pipeline = ProcessSwarmingTaskResultPipeline()
+    pipeline.run(
+        self.master_name, self.builder_name, self.build_number, self.step_name)
+
+    self.assertEqual(analysis_status.COMPLETED, task.status)
+    self.assertEqual(task.error, {'code': 1, 'message': 'error'})
+
+  @mock.patch.object(swarming_util, 'GetSwarmingTaskFailureLog',
+                     return_value=(_SAMPLE_FAILURE_LOG,
+                                   {'code': 1, 'message': 'error'}))
+  def testMonitorSwarmingTaskGetSwarmingTaskFailureLogErrorRecovered(self, _):
+    task = WfSwarmingTask.Create(
+        self.master_name, self.builder_name,
+        self.build_number, self.step_name)
+    task.task_id = 'task_id1'
+    task.put()
+
+    pipeline = ProcessSwarmingTaskResultPipeline()
+    pipeline.run(
+        self.master_name, self.builder_name, self.build_number, self.step_name)
+
+    self.assertEqual(analysis_status.COMPLETED, task.status)
+    self.assertEqual(task.error, {'code': 1, 'message': 'error'})
+
   @mock.patch.object(swarming_util, 'GetSwarmingTaskFailureLog',
                      return_value=(None, {'code': 1, 'message': 'error'}))
   def testMonitorSwarmingTaskGetSwarmingTaskFailureLogError(self, _):
     task = WfSwarmingTask.Create(
         self.master_name, self.builder_name,
         self.build_number, self.step_name)
     task.task_id = 'task_id1'
     task.put()
 
     pipeline = ProcessSwarmingTaskResultPipeline()
@@ skipping 35 matching lines @@
         self.master_name, self.builder_name, self.build_number, self.step_name)
 
     self.assertEqual(analysis_status.ERROR, task.status)
     self.assertEqual(
         task.error,
         {
             'code': swarming_util.NO_TASK_OUTPUTS,
             'message': 'outputs_ref is None'
         })
 
-  def testProcessSwarmingTaskResultPipeline(self):
+  @mock.patch.object(swarming_util, 'GetSwarmingTaskFailureLog',
+                     return_value=(_SAMPLE_FAILURE_LOG, None))
+  def testProcessSwarmingTaskResultPipeline(self, _):
     # End to end test.
-    self.mock(swarming_util, 'GetSwarmingTaskFailureLog',
-              self._MockedGetSwarmingTaskFailureLog)
-
     task = WfSwarmingTask.Create(
         self.master_name, self.builder_name,
         self.build_number, self.step_name)
     task.task_id = 'task_id1'
     task.put()
 
     pipeline = ProcessSwarmingTaskResultPipeline()
     step_name, task_info = pipeline.run(
         self.master_name, self.builder_name,
         self.build_number, self.step_name)
 
     self.assertEqual(self.step_name, step_name)
     self.assertEqual('abc_tests', task_info[0])
     self.assertEqual(
         _EXPECTED_CLASSIFIED_TESTS['reliable_tests'], task_info[1])
 
     task = WfSwarmingTask.Get(
         self.master_name, self.builder_name, self.build_number, self.step_name)
 
     self.assertEqual(analysis_status.COMPLETED, task.status)
     self.assertEqual(_EXPECTED_TESTS_STATUS, task.tests_statuses)
     self.assertEqual(
         _EXPECTED_CLASSIFIED_TESTS, task.classified_tests)
     self.assertEqual(datetime.datetime(2016, 2, 10, 18, 32, 6, 538220),
                      task.created_time)
     self.assertEqual(datetime.datetime(2016, 2, 10, 18, 32, 9, 90550),
                      task.started_time)
     self.assertEqual(datetime.datetime(2016, 2, 10, 18, 33, 9),
                      task.completed_time)