[Findit] Using ts_mon to track swarming/isolated server outages

appengine/findit/waterfall/swarming_util.py

Index: appengine/findit/waterfall/swarming_util.py
diff --git a/appengine/findit/waterfall/swarming_util.py b/appengine/findit/waterfall/swarming_util.py
index 44e2e8fe0068d013ce02e52a771a642447c65480..d449e281d526251579ce539dab70dd912dc4ad95 100644
--- a/appengine/findit/waterfall/swarming_util.py
+++ b/appengine/findit/waterfall/swarming_util.py
@@ -17,6 +17,7 @@ from google.appengine.ext import ndb
 
 from common import auth_util
 from model.wf_step import WfStep
+from waterfall import monitoring
 from waterfall import waterfall_config
 from waterfall.swarming_task_request import SwarmingTaskRequest
 
@@ -82,6 +83,18 @@ def _GetBackoffSeconds(retry_backoff, tries, maximum_retry_interval):
   return min(retry_backoff * (2 ** (tries - 1)), maximum_retry_interval)
 
 
+def _OnConnectionFailed(url, exception):
+  swarming_settings = waterfall_config.GetSwarmingSettings()
+  swarming_server_host = swarming_settings.get('server_host')
+  isolated_server_host = swarming_settings.get('isolated_server')
+  exception_type_name = type(exception).__name__

[stgao, 2016/12/01 22:25:15]

Why not pass over the exception name from code below instead?

Usually __ means not expected to be used out of the class.

[lijeffrey, 2016/12/01 22:54:34]

Done.

+
+  if isolated_server_host in url:

[stgao, 2016/12/01 22:25:15]

Extract the host from the url, and make it a field instead.

[lijeffrey, 2016/12/01 22:54:34]

Done.

+    monitoring.isolated_server_failures.increment({'type': exception_type_name})
+  elif swarming_server_host in url:
+    monitoring.swarming_server_failures.increment({'type': exception_type_name})
+
+
 def _SendRequestToServer(url, http_client, post_data=None):
   """Sends GET/POST request to arbitrary url and returns response content.
 
@@ -96,8 +109,6 @@ def _SendRequestToServer(url, http_client, post_data=None):
     http_client (HttpClient): The httpclient object with which to make the
       server calls.
     post_data (dict): Data/params to send with the request, if any.
-    swarming_task (WfSwarmingTask, FlakeSwarmingTask): An optional swarming
-      task with which to capture errors.
 
   Returns:
     content (dict), error (dict): The content from the server and the last error
@@ -131,21 +142,25 @@ def _SendRequestToServer(url, http_client, post_data=None):
           'code': URLFETCH_CONNECTION_CLOSED_ERROR,
           'message': e.message
       }
+      _OnConnectionFailed(url, e)
     except DeadlineExceededError as e:
       error = {
           'code': URLFETCH_DEADLINE_EXCEEDED_ERROR,
           'message': e.message
       }
+      _OnConnectionFailed(url, e)
     except DownloadError as e:
       error = {
           'code': URLFETCH_DOWNLOAD_ERROR,
           'message': e.message
       }
+      _OnConnectionFailed(url, e)
     except Exception as e:  # pragma: no cover
       error = {
           'code': UNKNOWN,
           'message': e.message
       }
+      _OnConnectionFailed(url, e)
 
     if error or status_code != 200:
       # The retry upon 50x (501 excluded) is automatically handled in the