[Findit] Implement retry within swarming_util.py when making server calls

appengine/findit/handlers/test/config_test.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import datetime
 import json
 import re
 import webapp2
 import webtest
 
@@ skipping 50 matching lines @@
 _MOCK_SWARMING_SETTINGS = {
     'server_host': 'chromium-swarm.appspot.com',
     'default_request_priority': 150,
     'request_expiration_hours': 20,
     'server_query_interval_seconds': 60,
     'task_timeout_hours': 23,
     'isolated_server': 'https://isolateserver.appspot.com',
     'isolated_storage_url': 'isolateserver.storage.googleapis.com',
     'iterations_to_rerun': 10,
     'get_swarming_task_id_timeout_seconds': 5 * 60,  # 5 minutes.
-    'get_swarming_task_id_wait_seconds': 10
+    'get_swarming_task_id_wait_seconds': 10,
+    'server_retry_timeout_hours': 2,
+    'maximum_server_contact_retry_interval_seconds': 5 * 60,  # 5 minutes.
+    'should_retry_server': False,  # No retry for unit testing.
 }
 
 
 _MOCK_DOWNLOAD_BUILD_DATA_SETTINGS = {
     'download_interval_seconds': 10,
     'memcache_master_download_expiration_seconds': 3600,
     'use_chrome_build_extract': True
 }
 
 _MOCK_ACTION_SETTINGS = {
@@ skipping 537 matching lines @@
         'default_request_priority': 150,
         'request_expiration_hours': 20,
         'server_query_interval_seconds': 60,
         'task_timeout_hours': 23,
         'isolated_server': 'https://isolateserver.appspot.com',
         'isolated_storage_url': 'isolateserver.storage.googleapis.com',
         'iterations_to_rerun': 1,
         'get_swarming_task_id_timeout_seconds': 300,
         'get_swarming_task_id_wait_seconds': []  # Should be an int.
     }))
+    self.assertFalse(config._ValidateSwarmingSettings({
+        'server_host': 'chromium-swarm.appspot.com',
+        'default_request_priority': 150,
+        'request_expiration_hours': 20,
+        'server_query_interval_seconds': 60,
+        'task_timeout_hours': 23,
+        'isolated_server': 'https://isolateserver.appspot.com',
+        'isolated_storage_url': 'isolateserver.storage.googleapis.com',
+        'iterations_to_rerun': 1,
+        'get_swarming_task_id_timeout_seconds': 300,
+        'get_swarming_task_id_wait_secondds': 10,
+        'server_retry_timeout_hours': {}  # Should be an int.
+    }))
+    self.assertFalse(config._ValidateSwarmingSettings({
+        'server_host': 'chromium-swarm.appspot.com',
+        'default_request_priority': 150,
+        'request_expiration_hours': 20,
+        'server_query_interval_seconds': 60,
+        'task_timeout_hours': 23,
+        'isolated_server': 'https://isolateserver.appspot.com',
+        'isolated_storage_url': 'isolateserver.storage.googleapis.com',
+        'iterations_to_rerun': 1,
+        'get_swarming_task_id_timeout_seconds': 300,
+        'get_swarming_task_id_wait_secondds': 10,
+        'server_retry_timeout_hours': 1,
+        'maximum_server_contact_retry_interval_seconds': ''  # Should be an int.
+    }))
+    self.assertFalse(config._ValidateSwarmingSettings({
+        'server_host': 'chromium-swarm.appspot.com',
+        'default_request_priority': 150,
+        'request_expiration_hours': 20,
+        'server_query_interval_seconds': 60,
+        'task_timeout_hours': 23,
+        'isolated_server': 'https://isolateserver.appspot.com',
+        'isolated_storage_url': 'isolateserver.storage.googleapis.com',
+        'iterations_to_rerun': 1,
+        'get_swarming_task_id_timeout_seconds': 300,
+        'get_swarming_task_id_wait_secondds': 10,
+        'server_retry_timeout_hours': 1,
+        'maximum_server_contact_retry_interval_seconds': 2,
+        'should_retry_server': 3  # Should be a bool.
+    }))
     self.assertTrue(config._ValidateSwarmingSettings({
         'server_host': 'chromium-swarm.appspot.com',
         'default_request_priority': 150,
         'request_expiration_hours': 20,
         'server_query_interval_seconds': 60,
         'task_timeout_hours': 23,
         'isolated_server': 'https://isolateserver.appspot.com',
         'isolated_storage_url': 'isolateserver.storage.googleapis.com',
         'iterations_to_rerun': 10,
         'get_swarming_task_id_timeout_seconds': 300,
-        'get_swarming_task_id_wait_seconds': 10
+        'get_swarming_task_id_wait_seconds': 10,
+        'server_retry_timeout_hours': 1,
+        'maximum_server_contact_retry_interval_seconds': 1,
+        'should_retry_server': False,
     }))
 
   def testValidateDownloadBuildDataSettings(self):
     self.assertFalse(config._ValidateDownloadBuildDataSettings({}))
     self.assertFalse(config._ValidateDownloadBuildDataSettings({
         'download_interval_seconds': {},  # Should be an int.
         'memcache_master_download_expiration_seconds': 10,
         'use_chrome_build_extract': True
         }))
     self.assertFalse(config._ValidateDownloadBuildDataSettings({
@@ skipping 200 matching lines @@
         {
             'lower_flake_threshold': 0.02,
             'upper_flake_threshold': 0.98,
             'max_flake_in_a_row': 4,
             'max_stable_in_a_row': 4,
             'iterations_to_rerun': 100,
             'max_build_numbers_to_look_back': 1000,
             'use_nearby_neighbor': True,
             'update_monorail_bug': True,
         }))