[Findit] Using ts_mon to track swarming/isolated server outages

appengine/findit/waterfall/swarming_util.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import base64
 from collections import defaultdict
 import json
 import logging
 import time
 import urllib
+from urlparse import urlparse
 import zlib
 
 from google.appengine.api.urlfetch_errors import DeadlineExceededError
 from google.appengine.api.urlfetch_errors import DownloadError
 from google.appengine.api.urlfetch_errors import ConnectionClosedError
 from google.appengine.ext import ndb
 
 from common import auth_util
 from model.wf_step import WfStep
+from waterfall import monitoring
 from waterfall import waterfall_config
 from waterfall.swarming_task_request import SwarmingTaskRequest
 
 
 # Swarming task states.
 STATES_RUNNING = ('RUNNING', 'PENDING')
 STATE_COMPLETED = 'COMPLETED'
 STATES_NOT_RUNNING = (
     'BOT_DIED', 'CANCELED', 'COMPLETED', 'EXPIRED', 'TIMED_OUT')
 
@@ skipping 45 matching lines @@
 
   Params:
     retry_backoff (int): The base backoff in seconds.
     tries (int): Indicates how many tries have been done.
     maximum_retry_interval (int): The upper limit in seconds of how long to wait
       between retries.
   """
   return min(retry_backoff * (2 ** (tries - 1)), maximum_retry_interval)
 
 
+def _OnConnectionFailed(url, exception_type):
+  host = urlparse(url).hostname
+  assert host
+
+  monitoring.http_errors.increment({'host': host, 'exception': exception_type})
+
+
 def _SendRequestToServer(url, http_client, post_data=None):
   """Sends GET/POST request to arbitrary url and returns response content.
 
   Because the Swarming and Isolated servers that _SendRequestToServer tries to
   contact are prone to outages, exceptions trying to reach them may occur thus
   this method should retry. We want to monitor and document these occurrences
   even if the request eventually succeeds after retrying, with the last error
   encountered being the one that is reported.
 
   Args:
     url (str): The url to send the request to.
     http_client (HttpClient): The httpclient object with which to make the
       server calls.
     post_data (dict): Data/params to send with the request, if any.
-    swarming_task (WfSwarmingTask, FlakeSwarmingTask): An optional swarming
-      task with which to capture errors.
 
   Returns:
     content (dict), error (dict): The content from the server and the last error
     encountered trying to retrieve it.
   """
   headers = {'Authorization': 'Bearer ' + auth_util.GetAuthToken()}
   swarming_settings = waterfall_config.GetSwarmingSettings()
   should_retry = swarming_settings.get('should_retry_server')
   timeout_seconds = (
       swarming_settings.get('server_retry_timeout_hours') * 60 * 60)
@@ skipping 13 matching lines @@
     try:
       if post_data:
         status_code, content = http_client.Post(url, post_data, headers=headers)
       else:
         status_code, content = http_client.Get(url, headers=headers)
     except ConnectionClosedError as e:
       error = {
           'code': URLFETCH_CONNECTION_CLOSED_ERROR,
           'message': e.message
       }
+      _OnConnectionFailed(url, 'ConnectionClosedError')
     except DeadlineExceededError as e:
       error = {
           'code': URLFETCH_DEADLINE_EXCEEDED_ERROR,
           'message': e.message
       }
+      _OnConnectionFailed(url, 'DeadlineExceededError')
     except DownloadError as e:
       error = {
           'code': URLFETCH_DOWNLOAD_ERROR,
           'message': e.message
       }
+      _OnConnectionFailed(url, 'DownloadError')
     except Exception as e:  # pragma: no cover
+      logging.error(
+          'An unknown exception occurred that need to be monitored: %s',
+          e.message)
       error = {
           'code': UNKNOWN,
           'message': e.message
       }
+      _OnConnectionFailed(url, 'Unknown Exception')
 
     if error or status_code != 200:
       # The retry upon 50x (501 excluded) is automatically handled in the
       # underlying http_client.
       # By default, it retries 5 times with exponential backoff.
       error = error or {
           'code': EXCEEDED_MAX_RETRIES_ERROR,
           'message': 'Max retries exceeded trying to reach %s' % url
       }
       logging.error(error['message'])
@@ skipping 373 matching lines @@
   for isolated_data in list_isolated_data:
     output_json, _ = _DownloadTestResults(isolated_data, http_client)
     if not output_json:
       # TODO(lijeffrey): Report/handle error returned from _DownloadTestResults.
       return None
     shard_results.append(output_json)
 
   if len(list_isolated_data) == 1:
     return shard_results[0]
   return _MergeSwarmingTestShards(shard_results)