[Findit] Implement retry within swarming_util.py when making server calls

appengine/findit/waterfall/test/swarming_util_test.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import collections
 import json
 import mock
 import os
 import urllib
 import zlib
@@ skipping 364 matching lines @@
         {
             'digest': 'isolatedhashunittests',
             'namespace': 'default-gzip',
             'isolatedserver': waterfall_config.GetSwarmingSettings().get(
                 'isolated_server')
         },
         {
             'digest': 'isolatedhashunittests1',
             'namespace': 'default-gzip',
             'isolatedserver': waterfall_config.GetSwarmingSettings().get(
-              'isolated_server')
+                'isolated_server')
         }
     ]
     self.assertEqual(sorted(expected_data), sorted(data))
 
   def testGetIsolatedDataForStepFailed(self):
     master_name = 'm'
     builder_name = 'download_failed'
     build_number = 223
     step_name = 's1'
 
@@ skipping 259 matching lines @@
     content, error = swarming_util._SendRequestToServer('url', HttpClient())
     self.assertIsNone(content)
     self.assertEqual(
         error['code'], swarming_util.URLFETCH_DEADLINE_EXCEEDED_ERROR)
 
   @mock.patch.object(RetryHttpClient, 'Get', side_effect=DownloadError())
   def testSendRequestToServerDownloadError(self, _):
     content, error = swarming_util._SendRequestToServer('url', HttpClient())
     self.assertIsNone(content)
     self.assertEqual(error['code'], swarming_util.URLFETCH_DOWNLOAD_ERROR)
+
+  def testGetBackoffSeconds(self):
+    self.assertEqual(1, swarming_util._GetBackoffSeconds(1, 1, 1))
+    self.assertEqual(2, swarming_util._GetBackoffSeconds(1, 2, 100))
+    self.assertEqual(100, swarming_util._GetBackoffSeconds(1, 8, 100))
+
+  @mock.patch.object(
+      RetryHttpClient, 'Get', side_effect=ConnectionClosedError())
+  def testSendRequestToServerRetryTimeout(self, _):
+    override_swarming_settings = {
+        'should_retry_server': True,
+        'server_retry_timeout_hours': -1
+    }
+    self.UpdateUnitTestConfigSettings(
+        'swarming_settings', override_swarming_settings)
+    content, error = swarming_util._SendRequestToServer('url', HttpClient())
+    self.assertIsNone(content)
+    self.assertEqual(
+        error['code'], swarming_util.URLFETCH_CONNECTION_CLOSED_ERROR)
+    self.assertTrue(error['retry_timeout'])