[Findit] Implementing swarming task error detection

appengine/findit/waterfall/test/swarming_util_test.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import collections
 import json
+import mock
 import os
 import urllib
 import zlib
 
+from google.appengine.api.urlfetch_errors import (
+    DeadlineExceededError, DownloadError, ConnectionClosedError)
+
+from common.http_client_appengine import HttpClientAppengine as HttpClient
 from common.retry_http_client import RetryHttpClient
 from model.wf_config import FinditConfig
 from model.wf_step import WfStep
 from waterfall import swarming_util
 from waterfall import waterfall_config
 from waterfall.swarming_task_request import SwarmingTaskRequest
 from waterfall.test import wf_testcase
 
 
 class SwarmingHttpClient(RetryHttpClient):
@@ skipping 137 matching lines @@
     url = ('https://chromium-swarm.appspot.com/'
            '_ah/api/swarming/v1/task/%s/request' % task_id)
     self.logged_http_client.SetResponse(
         'get', url, json.dumps(task_request_json), 200)
 
     task_request = swarming_util.GetSwarmingTaskRequest(
         task_id, self.logged_http_client)
 
     self.assertEqual(task_request_json, task_request.Serialize())
 
+  @mock.patch.object(swarming_util, '_SendRequestToServer',
+                     return_value=(None, {'code': 1, 'message': 'error'}))
+  def testGetSwarmingTaskRequestError(self, _):
+    self.assertIsNone(
+        swarming_util.GetSwarmingTaskRequest('task_id1', HttpClient()))
+
   def testTriggerSwarmingTask(self):
     request = SwarmingTaskRequest()
     request.expiration_secs = 2
     request.name = 'name'
     request.parent_task_id = 'pti'
     request.priority = 1
     request.tags = ['tag']
     request.user = 'user'
     request.command = 'cmd'
     request.dimensions = [{'key': 'd', 'value': 'dv'}]
@@ skipping 22 matching lines @@
             'extra_args': ['--flag'],
             'grace_period_secs': 5,
             'idempotent': True,
             'inputs_ref': {'isolated': 'i'},
             'io_timeout_secs': 3,
         },
         'tags': ['tag', 'findit:1', 'project:Chromium', 'purpose:post-commit'],
         'user': 'user',
     }
 
-    task_id = swarming_util.TriggerSwarmingTask(
+    task_id, error = swarming_util.TriggerSwarmingTask(
         request, self.logged_http_client)
     self.assertEqual('1', task_id)
+    self.assertIsNone(error)
 
     method, data, _ = self.logged_http_client.GetRequest(url)
     self.assertEqual('post', method)
     self.assertEqual(expected_task_request_json, json.loads(data))
 
+  @mock.patch.object(swarming_util, '_SendRequestToServer',
+                     return_value=(None, {'code': 1, 'message': 'error'}))
+  def testTriggerSwarmingTaskError(self, _):
+    request = SwarmingTaskRequest()
+    task_id, error = swarming_util.TriggerSwarmingTask(
+        request, HttpClient())
+    self.assertIsNone(task_id)
+    self.assertIsNotNone(error)
+
   def testGetIsolatedDataForFailedBuild(self):
     master_name = 'm'
     builder_name = 'b'
     build_number = 223
     failed_steps = {
         'a_tests': {
             'current_failure': 2,
             'first_failure': 0
         },
         'unit_tests': {
@@ skipping 140 matching lines @@
         'isolatedserver': waterfall_config.GetSwarmingSettings().get(
             'isolated_server')
     }
     isolated_storage_url = waterfall_config.GetSwarmingSettings().get(
         'isolated_storage_url')
     self.http_client._SetResponseForPostRequest('shard1_isolated')
     self.http_client._SetResponseForPostRequest('shard1_url')
     self.http_client._SetResponseForGetRequestIsolated(
         'https://%s/default-gzip/shard1' % isolated_storage_url, 'shard1')
 
-    result = swarming_util._DownloadTestResults(
+    result, error = swarming_util._DownloadTestResults(
         isolated_data, self.http_client)
 
     expected_result = json.loads(zlib.decompress(
         self.http_client._GetData('isolated', 'shard1')))
     self.assertEqual(expected_result, result)
+    self.assertIsNone(error)
 
   def testDownloadTestResultsFailedForSecondHash(self):
     isolated_data = {
         'digest': 'not found',
         'namespace': 'default-gzip',
         'isolatedserver': waterfall_config.GetSwarmingSettings().get(
             'isolated_server')
     }
 
-    result = swarming_util._DownloadTestResults(
+    result, error = swarming_util._DownloadTestResults(
         isolated_data, self.http_client)
 
     self.assertIsNone(result)
+    self.assertIsNotNone(error)
 
   def testDownloadTestResultsFailedForParsingSecondHash(self):
     isolated_data = {
         'digest': 'not found',
         'namespace': 'default-gzip',
         'isolatedserver': waterfall_config.GetSwarmingSettings().get(
             'isolated_server')
     }
 
     self.http_client._SetResponseForPostRequest('not found')
-    result = swarming_util._DownloadTestResults(
+    result, error = swarming_util._DownloadTestResults(
         isolated_data, self.http_client)
 
     self.assertIsNone(result)
+    self.assertIsNone(error)
 
   def testDownloadTestResultsFailedForFileUrl(self):
     isolated_data = {
         'digest': 'shard1_isolated',
         'namespace': 'default-gzip',
         'isolatedserver': waterfall_config.GetSwarmingSettings().get(
             'isolated_server')
     }
     self.http_client._SetResponseForPostRequest('shard1_isolated')
-    result = swarming_util._DownloadTestResults(
+    result, error = swarming_util._DownloadTestResults(
         isolated_data, self.http_client)
 
     self.assertIsNone(result)
+    self.assertIsNotNone(error)
 
   def testDownloadTestResultsFailedForFile(self):
     isolated_data = {
         'digest': 'shard1_isolated',
         'namespace': 'default-gzip',
         'isolatedserver': waterfall_config.GetSwarmingSettings().get(
             'isolated_server')
     }
     self.http_client._SetResponseForPostRequest('shard1_isolated')
     self.http_client._SetResponseForPostRequest('shard1_url')
-    result = swarming_util._DownloadTestResults(
+    result, error = swarming_util._DownloadTestResults(
         isolated_data, self.http_client)
 
     self.assertIsNone(result)
+    self.assertIsNone(error)
 
   def testRetrieveShardedTestResultsFromIsolatedServer(self):
     isolated_data = [
         {
             'digest': 'shard1_isolated',
             'namespace': 'default-gzip',
             'isolatedserver': waterfall_config.GetSwarmingSettings().get(
                 'isolated_server')
         },
         {
@@ skipping 69 matching lines @@
     result = swarming_util.RetrieveShardedTestResultsFromIsolatedServer(
         isolated_data, self.http_client)
 
     self.assertIsNone(result)
 
   def testGetSwarmingTaskResultById(self):
     task_id = '2944afa502297110'
 
     self.http_client._SetResponseForGetRequestSwarmingResult(task_id)
 
-    data = swarming_util.GetSwarmingTaskResultById(
+    data, error = swarming_util.GetSwarmingTaskResultById(
         task_id, self.http_client)
 
     expected_outputs_ref = {
         'isolatedserver': waterfall_config.GetSwarmingSettings().get(
             'isolated_server'),
         'namespace': 'default-gzip',
         'isolated': 'shard1_isolated'
     }
 
     self.assertEqual('COMPLETED', data['state'])
     self.assertEqual(expected_outputs_ref, data['outputs_ref'])
+    self.assertIsNone(error)
+
+  @mock.patch.object(swarming_util, '_SendRequestToServer',
+                     return_value=(None, {'code': 1, 'message': 'error'}))
+  def testGetSwarmingTaskResultByIdError(self, _):
+    data, error = swarming_util.GetSwarmingTaskResultById(
+        'task_id', HttpClient())
+    self.assertEqual({}, data)
+    self.assertIsNotNone(error)
 
   def testGetSwarmingTaskFailureLog(self):
     outputs_ref = {
         'isolatedserver': waterfall_config.GetSwarmingSettings().get(
             'isolated_server'),
         'namespace': 'default-gzip',
         'isolated': 'shard1_isolated'
     }
 
     self.http_client._SetResponseForPostRequest('shard1_isolated')
     self.http_client._SetResponseForPostRequest('shard1_url')
     self.http_client._SetResponseForGetRequestIsolated(
         'https://%s/default-gzip/shard1' % (
             waterfall_config.GetSwarmingSettings().get('isolated_storage_url')),
         'shard1')
 
-    result = swarming_util.GetSwarmingTaskFailureLog(
+    result, error = swarming_util.GetSwarmingTaskFailureLog(
         outputs_ref, self.http_client)
 
     expected_result = json.loads(zlib.decompress(
         self.http_client._GetData('isolated', 'shard1')))
     self.assertEqual(expected_result, result)
+    self.assertIsNone(error)
 
   def testRetrieveOutputJsonFileGetDirectly(self):
     output_json_content = ('{"content": "eJyrVkpLzMwpLUotVrKKVgpJLS4xV'
                            'IrVUVAqS8zJTFGyUigpKk2tBQDr9wxZ"}')
 
     failure_log = swarming_util._RetrieveOutputJsonFile(
         output_json_content, self.http_client)
 
     expected_failure_log = {
         'failures': ['Test1'],
         'valid': True
     }
 
     self.assertEqual(expected_failure_log, failure_log)
 
   def testGetTagValueInvalidTag(self):
     tags = ['a:1', 'b:2']
     self.assertIsNone(swarming_util.GetTagValue(tags, 'c'))
 
   def testGenerateIsolatedDataOutputsrefNone(self):
     self.assertEqual({}, swarming_util._GenerateIsolatedData(None))
 
   def testFetchOutputJsonInfoFromIsolatedServerReturnNone(self):
     self.assertIsNone(swarming_util._FetchOutputJsonInfoFromIsolatedServer(
-        None, self.http_client))
+        None, self.http_client))
+
+  @mock.patch.object(
+      RetryHttpClient, 'Get', side_effect=ConnectionClosedError())
+  def testSendRequestToServerConnectionClosedError(self, _):
+    content, error = swarming_util._SendRequestToServer('url', HttpClient())
+    self.assertIsNone(content)
+    self.assertEqual(
+        error['code'], swarming_util.URLFETCH_CONNECTION_CLOSED_ERROR)
+
+  @mock.patch.object(
+      RetryHttpClient, 'Get', side_effect=DeadlineExceededError())
+  def testSendRequestToServerDeadlineExceededError(self, _):
+    content, error = swarming_util._SendRequestToServer('url', HttpClient())
+    self.assertIsNone(content)
+    self.assertEqual(
+        error['code'], swarming_util.URLFETCH_DEADLINE_EXCEEDED_ERROR)
+
+  @mock.patch.object(RetryHttpClient, 'Get', side_effect=DownloadError())
+  def testSendRequestToServerDownloadError(self, _):
+    content, error = swarming_util._SendRequestToServer('url', HttpClient())
+    self.assertIsNone(content)
+    self.assertEqual(error['code'], swarming_util.URLFETCH_DOWNLOAD_ERROR)