swarmbucket: allow overriding config

appengine/cr-buildbucket/swarming/test/swarming_test.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import base64
 import contextlib
 import datetime
 import json
+import logging
 
 from components import auth
 from components import config as config_component
 from components import net
 from components import utils
 from google.appengine.ext import ndb
 from testing_utils import testing
 from webob import exc
 import mock
 import webapp2
 
 from swarming import swarming
 from proto import project_config_pb2
 import config
 import errors
 import model
 
 
 def futuristic(result):
   f = ndb.Future()
   f.set_result(result)
   return f
 
 
 class SwarmingTest(testing.AppengineTestCase):
   def setUp(self):
     super(SwarmingTest, self).setUp()
     self.mock(utils, 'utcnow', lambda: datetime.datetime(2015, 11, 30))
+
+    self.json_response = None
+    def json_request_async(*_, **__):
+      if self.json_response is not None:
+        return futuristic(self.json_response)
+      self.fail('unexpected outbound request')  # pragma: no cover
+
+    self.mock(
+        net, 'json_request_async', mock.Mock(side_effect=json_request_async))
+
     self.bucket_cfg = project_config_pb2.Bucket(
       name='bucket',
       swarming=project_config_pb2.Swarming(
         hostname='chromium-swarm.appspot.com',
         url_format='https://example.com/{swarming_hostname}/{task_id}',
         common_swarming_tags=['commontag:yes'],
         common_dimensions=['cores:8', 'pool:default', 'cpu:x86-64'],
         common_recipe=project_config_pb2.Swarming.Recipe(
           repository='https://example.com/repo',
           name='recipe',
@@ skipping 93 matching lines @@
       {'changes': [{'author': {'email': 0}}]},
       {'changes': [{'author': {'email': ''}}]},
       {'changes': [{'author': {'email': 'a@example.com'}, 'repo_url': 0}]},
       {'bad key': 1},
       {'swarming': []},
       {'swarming': {'junk': 1}},
       {'swarming': {'recipe': []}},
       {'swarming': {'recipe': {'junk': 1}}},
       {'swarming': {'recipe': {'revision': 1}}},
       {'swarming': {'canary_template': 'yes'}},
+      {'swarming': {'override_cfg': 0}},
+      {'swarming': {'override_cfg': {'swarming': 0}}},
+      {
+        'swarming': {
+          'override_cfg': {
+            'swarming': {'hostname': 0},
+          },
+        },
+      },
+      {
+        'swarming': {
+          'override_cfg': {
+            'swarming': {'builders': [{}]},
+          },
+        },
+      },
+      {
+        'swarming': {
+          'override_cfg': {
+            'swarming': {'common_dimensions': ['wrong_dimension']},
+          },
+        },
+      },
+      {'swarming': {'override_cfg': {'x': 0}}},
     ]
     for p in bad:
+      logging.info('testing %s', p)
       p['builder_name'] = 'foo'
       with self.assertRaises(errors.InvalidInputError):
         swarming.validate_build_parameters(p['builder_name'], p)
 
   def test_execution_timeout(self):
     self.bucket_cfg.swarming.common_execution_timeout_secs = 120
     builder_cfg = project_config_pb2.Swarming.Builder(name='fast-builder')
 
     build = model.Build(
         bucket='bucket',
@@ skipping 27 matching lines @@
         'properties': {
           'a': 'b',
         },
         'changes': [{
           'author': {'email': 'bob@example.com'},
           'repo_url': 'https://chromium.googlesource.com/chromium/src',
         }]
       },
     )
 
-    self.mock(net, 'json_request_async', mock.Mock(return_value=futuristic({
+    self.json_response = {
       'task_id': 'deadbeef',
       'request': {
         'properties': {
           'dimensions': [
             {'key': 'cores', 'value': '8'},
             {'key': 'os', 'value': 'Linux'},
             {'key': 'pool', 'value': 'Chrome'},
           ],
         },
         'tags': [
           'builder:builder',
           'buildertag:yes',
           'commontag:yes',
           'master:master.bucket',
           'priority:108',
           'recipe_name:recipe',
           'recipe_repository:https://example.com/repo',
           'recipe_revision:badcoffee',
         ]
       }
-    })))
+    }
 
     swarming.create_task_async(build).get_result()
 
     # Test swarming request.
     self.assertEqual(
       net.json_request_async.call_args[0][0],
       'https://chromium-swarm.appspot.com/_ah/api/swarming/v1/tasks/new')
     actual_task_def = net.json_request_async.call_args[1]['payload']
     del actual_task_def['pubsub_auth_token']
     expected_task_def = {
@@ skipping 70 matching lines @@
   def test_create_task_async_canary_template(self):
     build = model.Build(
         bucket='bucket',
         parameters={
           'builder_name': 'builder',
           'swarming': {
             'canary_template': True,
           }        },
     )
 
-    self.mock(net, 'json_request_async', mock.Mock(return_value=futuristic({
+    self.json_response = {
       'task_id': 'deadbeef',
       'request': {
         'properties': {
           'dimensions': [
             {'key': 'cores', 'value': '8'},
             {'key': 'os', 'value': 'Linux'},
             {'key': 'pool', 'value': 'Chrome'},
           ],
         },
         'tags': [
           'builder:builder',
           'buildertag:yes',
           'commontag:yes',
           'master:master.bucket',
           'priority:108',
           'recipe_name:recipe',
           'recipe_repository:https://example.com/repo',
         ]
       }
-    })))
+    }
 
     swarming.create_task_async(build).get_result()
 
     # Test swarming request.
     self.assertEqual(
         net.json_request_async.call_args[0][0],
         'https://chromium-swarm.appspot.com/_ah/api/swarming/v1/tasks/new')
     actual_task_def = net.json_request_async.call_args[1]['payload']
     del actual_task_def['pubsub_auth_token']
     expected_task_def = {
@@ skipping 76 matching lines @@
 
     self.task_template_canary = None
     with self.assertRaises(errors.InvalidInputError):
       swarming.create_task_async(build).get_result()
 
   def test_create_task_async_no_canary_template_implicit(self):
     self.mock(swarming, 'should_use_canary_template', mock.Mock())
     swarming.should_use_canary_template.return_value = True
     self.task_template_canary = None
 
-    self.mock(net, 'json_request_async', mock.Mock(return_value=futuristic({
+    self.json_response = {
       'task_id': 'deadbeef',
       'request': {
         'properties': {
           'dimensions': [
             {'key': 'cores', 'value': '8'},
             {'key': 'os', 'value': 'Linux'},
             {'key': 'pool', 'value': 'Chrome'},
           ],
         },
         'tags': [
           'builder:builder',
           'buildertag:yes',
           'commontag:yes',
           'master:master.bucket',
           'priority:108',
           'recipe_name:recipe',
           'recipe_repository:https://example.com/repo',
           'recipe_revision:badcoffee',
         ]
       }
-    })))
+    }
 
     build = model.Build(
         bucket='bucket',
         parameters={'builder_name': 'builder'},
     )
     swarming.create_task_async(build).get_result()
 
     actual_task_def = net.json_request_async.call_args[1]['payload']
     self.assertIn('buildbucket_template_canary:false', actual_task_def['tags'])
 
+  def test_create_task_async_override_cfg(self):
+    build = model.Build(
+        bucket='bucket',
+        parameters={
+          'builder_name': 'builder',
+          'swarming': {
+            'override_builder_cfg': {
+              # Override cores dimension.
+              'dimensions': ['cores:16'],
+            },
+          }
+        },
+    )
+
+    self.json_response = {
+      'task_id': 'deadbeef',
+      'request': {
+        'properties': {
+          'dimensions': [
+            {'key': 'cores', 'value': '16'},
+            {'key': 'os', 'value': 'Linux'},
+            {'key': 'pool', 'value': 'Chrome'},
+          ],
+        },
+        'tags': [
+          'builder:builder',
+          'buildertag:yes',
+          'commontag:yes',
+          'master:master.bucket',
+          'priority:108',
+          'recipe_name:recipe',
+          'recipe_repository:https://example.com/repo',
+          'recipe_revision:badcoffee',
+        ]
+      }
+    }
+
+    swarming.create_task_async(build).get_result()
+
+    actual_task_def = net.json_request_async.call_args[1]['payload']
+    self.assertIn(
+        {'key': 'cores', 'value': '16'},
+        actual_task_def['properties']['dimensions'])
+
+  def test_create_task_async_override_cfg_malformed(self):
+    build = model.Build(
+        bucket='bucket',
+        parameters={
+          'builder_name': 'builder',
+          'swarming': {
+            'override_builder_cfg': [],
+          }
+        },
+    )
+    with self.assertRaises(errors.InvalidInputError):
+      swarming.create_task_async(build).get_result()
+
+    build = model.Build(
+        bucket='bucket',
+        parameters={
+          'builder_name': 'builder',
+          'swarming': {
+            'override_builder_cfg': {
+              'name': 'x',
+            },
+          }
+        },
+    )
+    with self.assertRaises(errors.InvalidInputError):
+      swarming.create_task_async(build).get_result()
+
+    build = model.Build(
+        bucket='bucket',
+        parameters={
+          'builder_name': 'builder',
+          'swarming': {
+            'override_builder_cfg': {
+              'blabla': 'x',
+            },
+          }
+        },
+    )
+    with self.assertRaises(errors.InvalidInputError):
+      swarming.create_task_async(build).get_result()
+
+    # Remove a required dimension.
+    build = model.Build(
+        bucket='bucket',
+        parameters={
+          'builder_name': 'builder',
+          'swarming': {
+            'override_builder_cfg': {
+              'dimensions': ['pool:'],
+            },
+          }
+        },
+    )
+    with self.assertRaises(errors.InvalidInputError):
+      swarming.create_task_async(build).get_result()
+
+
   def test_create_task_async_on_leased_build(self):
     build = model.Build(
       bucket='bucket',
       parameters={'builder_name': 'builder'},
       lease_key=12345,
     )
     with self.assertRaises(errors.InvalidInputError):
       swarming.create_task_async(build).get_result()
 
   def test_cancel_task(self):
-    self.mock(net, 'json_request_async', mock.Mock(return_value=futuristic({})))
+    self.json_response = {}
     build = model.Build(
       bucket='whatever',
       swarming_hostname='chromium-swarm.appspot.com',
       swarming_task_id='deadbeef',
     )
     swarming.cancel_task_async(build).get_result()
     net.json_request_async.assert_called_with(
       ('https://chromium-swarm.appspot.com/'
        '_ah/api/swarming/v1/task/deadbeef/cancel'),
       method='POST',
@@ skipping 303 matching lines @@
     self.assertEqual(build.status, model.BuildStatus.COMPLETED)
     self.assertEqual(build.result, model.BuildResult.FAILURE)
     self.assertEqual(build.failure_reason, model.FailureReason.INFRA_FAILURE)
     self.assertIsNotNone(build.result_details)
     self.assertIsNone(build.lease_key)
     self.assertIsNotNone(build.complete_time)
 
 
 def b64json(data):
   return base64.b64encode(json.dumps(data))