[Findit] Implement retry within swarming_util.py when making server calls

appengine/findit/waterfall/test/swarming_util_test.py

Index: appengine/findit/waterfall/test/swarming_util_test.py
diff --git a/appengine/findit/waterfall/test/swarming_util_test.py b/appengine/findit/waterfall/test/swarming_util_test.py
index 3ed66228af7fdb7738c05b7040339b68c541cc9b..d9283d381eb777ccc9ac8a03ddcff1f00883fb85 100644
--- a/appengine/findit/waterfall/test/swarming_util_test.py
+++ b/appengine/findit/waterfall/test/swarming_util_test.py
@@ -382,7 +382,7 @@ class SwarmingUtilTest(wf_testcase.WaterfallTestCase):
             'digest': 'isolatedhashunittests1',
             'namespace': 'default-gzip',
             'isolatedserver': waterfall_config.GetSwarmingSettings().get(
-              'isolated_server')
+                'isolated_server')
         }
     ]
     self.assertEqual(sorted(expected_data), sorted(data))
@@ -662,3 +662,23 @@ class SwarmingUtilTest(wf_testcase.WaterfallTestCase):
     content, error = swarming_util._SendRequestToServer('url', HttpClient())
     self.assertIsNone(content)
     self.assertEqual(error['code'], swarming_util.URLFETCH_DOWNLOAD_ERROR)
+
+  def testGetBackoffSeconds(self):
+    self.assertEqual(1, swarming_util._GetBackoffSeconds(1, 1, 1))
+    self.assertEqual(2, swarming_util._GetBackoffSeconds(1, 2, 100))
+    self.assertEqual(100, swarming_util._GetBackoffSeconds(1, 8, 100))
+
+  @mock.patch.object(
+      RetryHttpClient, 'Get', side_effect=ConnectionClosedError())
+  def testSendRequestToServerRetryTimeout(self, _):
+    override_swarming_settings = {
+        'should_retry_server': True,
+        'server_retry_timeout_hours': -1
+    }
+    self.UpdateUnitTestConfigSettings(
+        'swarming_settings', override_swarming_settings)
+    content, error = swarming_util._SendRequestToServer('url', HttpClient())
+    self.assertIsNone(content)
+    self.assertEqual(
+        error['code'], swarming_util.URLFETCH_CONNECTION_CLOSED_ERROR)
+    self.assertTrue(error['retry_timeout'])