swarmbucket: refactor builder defaults

appengine/cr-buildbucket/swarming/swarming.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """This module integrates buildbucket with swarming.
 
 A bucket config may have "swarming" field that specifies how a builder
 is mapped to a recipe. If build is scheduled for a bucket/builder
 with swarming configuration, the integration overrides the default behavior.
 
@@ skipping 26 matching lines @@
 from components import config as component_config
 from components import decorators
 from components import net
 from components import utils
 from components.auth import tokens
 from google.appengine.api import app_identity
 from google.appengine.ext import ndb
 import webapp2
 
 from proto import project_config_pb2
+from . import swarmingcfg as swarmingcfg_module
 import config
 import errors
 import model
 import notifications
 
 PUBSUB_TOPIC = 'swarming'
 BUILDER_PARAMETER = 'builder_name'
 PARAM_PROPERTIES = 'properties'
 PARAM_SWARMING = 'swarming'
 PARAM_CHANGES = 'changes'
@@ skipping 123 matching lines @@
       email = author.get('email')
       if not isinstance(email, basestring):
         bad('change author email must be a string')
       if not email:
         bad('change author email not specified')
 
   if params:
     bad('unrecognized params: %r', params.keys())
 
 
-def read_properties(recipe):
-  """Parses build properties from the recipe message."""
-  result = dict(p.split(':', 1) for p in recipe.properties)
-  for p in recipe.properties_j:
-    k, v = p.split(':', 1)
-    result[k] = json.loads(v)
-  return result
-
-
-def merge_recipe(r1, r2):
-  """Merges two Recipe messages. Values in r2 overwrite values in r1."""
-  if not r1:  # pragma: no branch
-    return r2  # pragma: no cover
-  props = read_properties(r1)
-  props.update(read_properties(r2))
-  recipe = project_config_pb2.Swarming.Recipe(
-    repository=r2.repository or r1.repository,
-    name=r2.name or r1.name,
-  )
-  return recipe, props
-
-
 def should_use_canary_template(build, percentage):  # pragma: no cover
   """Returns True if a canary template should be used for the |build|.
 
   This function is determinstic.
   """
   if percentage == 0:
     return False
   identity = {
     'bucket': build.bucket,
     'parametres': build.parameters,
@@ skipping 17 matching lines @@
   validate_build_parameters(builder_cfg.name, params)
   swarming_param = params.get(PARAM_SWARMING) or {}
 
   # Use canary template?
   canary = swarming_param.get('canary_template')
   canary_required = bool(canary)
   if canary is None:
     canary = should_use_canary_template(
         build, swarming_cfg.task_template_canary_percentage)
 
+  # Normalize/merge configs.
+  swarming_cfg = copy.deepcopy(swarming_cfg)
+  swarmingcfg_module.normalize_swarming_cfg(swarming_cfg)
+
+  merged = copy.deepcopy(swarming_cfg.builder_defaults)
+  swarmingcfg_module.merge_builder(merged, builder_cfg)
+  builder_cfg = merged
+
   # Render task template.
   try:
     task_template, canary = yield get_task_template_async(
         canary, canary_required)
   except CanaryTemplateNotFound as ex:
     raise errors.InvalidInputError(ex.message)
   task_template_params = {
     'bucket': build.bucket,
     'builder': builder_cfg.name,
     'project': project_id,
   }
 
-  is_recipe = (
-      builder_cfg.HasField('recipe') or swarming_cfg.HasField('common_recipe'))
+  is_recipe = builder_cfg.HasField('recipe')
   if is_recipe:  # pragma: no branch
-    recipe, build_properties = merge_recipe(
-        swarming_cfg.common_recipe, builder_cfg.recipe)
-    revision = swarming_param.get('recipe', {}).get('revision') or ''
+    build_properties = swarmingcfg_module.read_properties(builder_cfg.recipe)
+    recipe_revision = swarming_param.get('recipe', {}).get('revision') or ''
 
     build_properties['buildername'] = builder_cfg.name
 
     changes = params.get(PARAM_CHANGES)
     if changes:  # pragma: no branch
       # Buildbucket-Buildbot integration passes repo_url of the first change in
       # build parameter "changes" as "repository" attribute of SourceStamp.
       # https://chromium.googlesource.com/chromium/tools/build/+/2c6023d/scripts/master/buildbucket/changestore.py#140
       # Buildbot passes repository of the build source stamp as "repository"
       # build property. Recipes, in partiular bot_update recipe module, rely on
       # "repository" property and it is an almost sane property to support in
       # swarmbucket.
       repo_url = changes[0].get('repo_url')
       if repo_url:  # pragma: no branch
         build_properties['repository'] = repo_url
 
       # Buildbot-Buildbucket integration converts emails in changes to blamelist
       # property.
       emails = [c.get('author', {}).get('email') for c in changes]
       build_properties['blamelist'] = filter(None, emails)
 
     # Properties specified in build parameters must override any values derived
     # by swarmbucket.
     build_properties.update(build.parameters.get(PARAM_PROPERTIES) or {})
 
     task_template_params.update({
-      'repository': recipe.repository,
-      'revision': revision,
-      'recipe': recipe.name,
+      'repository': builder_cfg.recipe.repository,
+      'revision': recipe_revision,
+      'recipe': builder_cfg.recipe.name,
       'properties_json': json.dumps(build_properties, sort_keys=True),
     })
 
   task_template_params = {
     k: v or '' for k, v in task_template_params.iteritems()}
   task = format_obj(task_template, task_template_params)
 
   if builder_cfg.priority > 0:  # pragma: no branch
     task['priority'] = builder_cfg.priority
 
   build_tags = dict(t.split(':', 1) for t in build.tags)
   build_tags['builder'] = builder_cfg.name
   build.tags = sorted('%s:%s' % (k, v) for k, v in build_tags.iteritems())
 
   swarming_tags = task.setdefault('tags', [])
   _extend_unique(swarming_tags, [
     'buildbucket_hostname:%s' % app_identity.get_default_version_hostname(),
     'buildbucket_bucket:%s' % build.bucket,
     'buildbucket_template_canary:%s' % str(canary).lower(),
   ])
   if is_recipe:  # pragma: no branch
     _extend_unique(swarming_tags, [
-      'recipe_repository:%s' % recipe.repository,
-      'recipe_revision:%s' % revision,
-      'recipe_name:%s' % recipe.name,
+      'recipe_repository:%s' % builder_cfg.recipe.repository,
+      'recipe_revision:%s' % recipe_revision,
+      'recipe_name:%s' % builder_cfg.recipe.name,
     ])
-  _extend_unique(swarming_tags, swarming_cfg.common_swarming_tags)
   _extend_unique(swarming_tags, builder_cfg.swarming_tags)
   _extend_unique(swarming_tags, build.tags)
   swarming_tags.sort()
 
   task_properties = task.setdefault('properties', {})
-  task_properties['dimensions'] = _prepare_dimensions(
-    task_properties.get('dimensions', []),
-    swarming_cfg.common_dimensions,
-    builder_cfg.dimensions
-  )
+  task_properties['dimensions'] = _to_swarming_dimensions(
+    swarmingcfg_module.merge_dimensions(
+      builder_cfg.dimensions,
+      task_properties.get('dimensions', []),
+    ))
 
-  if swarming_cfg.common_execution_timeout_secs > 0:
-    task_properties['execution_timeout_secs'] = (
-        swarming_cfg.common_execution_timeout_secs)
   if builder_cfg.execution_timeout_secs > 0:
     task_properties['execution_timeout_secs'] = (
         builder_cfg.execution_timeout_secs)
 
   task['pubsub_topic'] = (
     'projects/%s/topics/%s' %
     (app_identity.get_application_id(), PUBSUB_TOPIC))
   task['pubsub_auth_token'] = TaskToken.generate()
   task['pubsub_userdata'] = json.dumps({
     'created_ts': utils.datetime_to_timestamp(utils.utcnow()),
     'swarming_hostname': swarming_cfg.hostname,
   }, sort_keys=True)
   raise ndb.Return(task)
 
 
-def _prepare_dimensions(global_dims, bucket_cfg_dims, builder_cfg_dims):
-  """Computes final task dimensions.
-
-  Configs must have valid format.
-  """
-  # Make them all lists of pairs.
-  global_dims = [(d['key'], d['value']) for d in global_dims]
-  parse_cfg_dim = lambda d: d.split(':', 1)
-  bucket_dims = map(parse_cfg_dim, bucket_cfg_dims)
-  builder_dims = map(parse_cfg_dim, builder_cfg_dims)
-
-  # Note: dimensions must be unique.
-  # Overwrite global dimensions by bucket-level dimensions,
-  # then by builder-level dimensions.
-  merged = {}
-  for ds in (global_dims, bucket_dims, builder_dims):
-    merged.update(ds)
-  return sorted({'key': k, 'value': v} for k, v in merged.iteritems() if v)
+def _to_swarming_dimensions(dims):
+  """Converts dimensions from buildbucket format to swarming format."""
+  return [
+    {'key': key, 'value': value}
+    for key, value in
+    (s.split(':', 1) for s in dims)
+  ]
 
 
 @ndb.tasklet
 def create_task_async(build):
   """Creates a swarming task for the build and mutates the build.
 
   May be called only if is_for_swarming(build) == True.
   """
   if build.lease_key:
     raise errors.InvalidInputError(
@@ skipping 369 matching lines @@
 def _extend_unique(target, items):
   for x in items:
     if x not in target:  # pragma: no branch
       target.append(x)
 
 
 class TaskToken(tokens.TokenKind):
   expiration_sec = 60 * 60 * 24  # 24 hours.
   secret_key = auth.SecretKey('swarming_task_token', scope='local')
   version = 1