swarmbucket: allow overriding config

appengine/cr-buildbucket/swarming/swarming.py

-# Copyright 2014 The Chromium Authors. All rights reserved.
+# Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """This module integrates buildbucket with swarming.
 
 A bucket config may have "swarming" field that specifies how a builder
 is mapped to a recipe. If build is scheduled for a bucket/builder
 with swarming configuration, the integration overrides the default behavior.
 
 Prior adding Build to the datastore, a swarming task is created. The definition
@@ skipping 20 matching lines @@
 import json
 import logging
 import string
 
 from components import auth
 from components import config as component_config
 from components import decorators
 from components import net
 from components import utils
 from components.auth import tokens
+from components.config import validation
 from google.appengine.api import app_identity
 from google.appengine.ext import ndb
 import webapp2
 
 from proto import project_config_pb2
 from . import swarmingcfg as swarmingcfg_module
 import config
 import errors
 import model
 import notifications
+import protoutil
+
 
 PUBSUB_TOPIC = 'swarming'
 BUILDER_PARAMETER = 'builder_name'
 PARAM_PROPERTIES = 'properties'
 PARAM_SWARMING = 'swarming'
 PARAM_CHANGES = 'changes'
 DEFAULT_URL_FORMAT = 'https://{swarming_hostname}/user/task/{task_id}'
 
 
 ################################################################################
@@ skipping 68 matching lines @@
 
 def validate_build_parameters(builder_name, params):
   """Raises errors.InvalidInputError if build parameters are invalid."""
   params = copy.deepcopy(params)
 
   def bad(fmt, *args):
     raise errors.InvalidInputError(fmt % args)
 
   params.pop(BUILDER_PARAMETER)  # already validated
 
+
+  def assert_object(name, value):
+    if not isinstance(value, dict):
+      bad('%s parameter must be an object' % name)
+
   properties = params.pop(PARAM_PROPERTIES, None)
   if properties is not None:
-    if not isinstance(properties, dict):
-      bad('properties parameter must be an object')
+    assert_object('properties', properties)
     if properties.get('buildername', builder_name) != builder_name:
       bad('inconsistent builder name')
 
   swarming = params.pop(PARAM_SWARMING, None)
   if swarming is not None:
-    if not isinstance(swarming, dict):
-      bad('swarming parameter must be an object')
+    assert_object('swarming', swarming)
     swarming = copy.deepcopy(swarming)
     if 'recipe' in swarming:
+      logging.error(
+          'someone is still using deprecated swarming.recipe parameter')
       recipe = swarming.pop('recipe')
-      if not isinstance(recipe, dict):
-        bad('swarming.recipe parameter must be an object')
+      assert_object('swarming.recipe', recipe)
       if 'revision' in recipe:
         revision = recipe.pop('revision')
         if not isinstance(revision, basestring):
           bad('swarming.recipe.revision parameter must be a string')
       if recipe:
         bad('unrecognized keys in swarming.recipe: %r', recipe)
     canary_template = swarming.pop('canary_template', None)
     if canary_template not in (True, False, None):
       bad('swarming.canary_template parameter must true, false or null')
+
+    override_builder_cfg_data = swarming.pop('override_builder_cfg', None)
+    if override_builder_cfg_data is not None:
+      assert_object('swarming.override_builder_cfg', override_builder_cfg_data)
+      override_builder_cfg = project_config_pb2.Swarming.Builder()
+      try:
+        protoutil.merge_dict(
+            override_builder_cfg_data, override_builder_cfg)
+      except TypeError as ex:
+        bad('swarming.override_builder_cfg parameter: %s', ex)
+      if override_builder_cfg.name:
+        bad('swarming.override_builder_cfg cannot override builder name')
+      ctx = validation.Context.raise_on_error(
+          exc_type=errors.InvalidInputError,
+          prefix='swarming.override_builder_cfg parameter: ')
+      swarmingcfg_module.validate_builder_cfg(
+          override_builder_cfg, ctx, final=False)
+
     if swarming:
       bad('unrecognized keys in swarming param: %r', swarming.keys())
 
   changes = params.pop(PARAM_CHANGES, None)
   if changes is not None:
     if not isinstance(changes, list):
       bad('changes param must be an array')
     for c in changes:  # pragma: no branch
       if not isinstance(c, dict):
         bad('changes param must contain only objects')
@@ skipping 21 matching lines @@
   if percentage == 0:
     return False
   identity = {
     'bucket': build.bucket,
     'parametres': build.parameters,
   }
   digest = hashlib.sha1(json.dumps(identity, sort_keys=True)).digest()
   return digest[0] < 256 * percentage / 100
 
 
+def _prepare_builder_config(swarming_cfg, builder_cfg, swarming_param):
+  """Returns final version of builder config to use for |build|.
+
+  Expects arguments to be valid.
+  """
+  swarming_cfg = copy.deepcopy(swarming_cfg)
+  swarmingcfg_module.normalize_swarming_cfg(swarming_cfg)
+
+  # Apply defaults.
+  result = copy.deepcopy(swarming_cfg.builder_defaults)
+  swarmingcfg_module.merge_builder(result, builder_cfg)
+
+  # Apply overrides in the swarming parameter.
+  override_builder_cfg_data = swarming_param.get('override_builder_cfg', {})
+  if override_builder_cfg_data:
+    override_builder_cfg = project_config_pb2.Swarming.Builder()
+    protoutil.merge_dict(override_builder_cfg_data, result)
+    ctx = validation.Context.raise_on_error(
+        exc_type=errors.InvalidInputError,
+        prefix='swarming.override_buider_cfg parameter: ')
+    swarmingcfg_module.merge_builder(result, override_builder_cfg)
+    swarmingcfg_module.validate_builder_cfg(result, ctx)
+  return result
+
+
 @ndb.tasklet
 def create_task_def_async(project_id, swarming_cfg, builder_cfg, build):
   """Creates a swarming task definition for the |build|.
 
   Supports build properties that are supported by Buildbot-Buildbucket
   integration. See
   https://chromium.googlesource.com/chromium/tools/build/+/eff4ceb/scripts/master/buildbucket/README.md#Build-parameters
 
   Raises:
     errors.InvalidInputError if build.parameters are invalid.
   """
   params = build.parameters or {}
   validate_build_parameters(builder_cfg.name, params)
   swarming_param = params.get(PARAM_SWARMING) or {}
 
   # Use canary template?
   canary = swarming_param.get('canary_template')
   canary_required = bool(canary)
   if canary is None:
     canary = should_use_canary_template(
         build, swarming_cfg.task_template_canary_percentage)
 
-  # Normalize/merge configs.
-  swarming_cfg = copy.deepcopy(swarming_cfg)
-  swarmingcfg_module.normalize_swarming_cfg(swarming_cfg)
-
-  merged = copy.deepcopy(swarming_cfg.builder_defaults)
-  swarmingcfg_module.merge_builder(merged, builder_cfg)
-  builder_cfg = merged
+  builder_cfg = _prepare_builder_config(
+      swarming_cfg, builder_cfg, swarming_param)
 
   # Render task template.
   try:
     task_template, canary = yield get_task_template_async(
         canary, canary_required)
   except CanaryTemplateNotFound as ex:
     raise errors.InvalidInputError(ex.message)
   task_template_params = {
     'bucket': build.bucket,
     'builder': builder_cfg.name,
@@ skipping 474 matching lines @@
 def _extend_unique(target, items):
   for x in items:
     if x not in target:  # pragma: no branch
       target.append(x)
 
 
 class TaskToken(tokens.TokenKind):
   expiration_sec = 60 * 60 * 24  # 24 hours.
   secret_key = auth.SecretKey('swarming_task_token', scope='local')
   version = 1