CrOS: Update public waterfall to auto-configure.

masters/master.chromiumos/master.cfg

 # -*- python -*-
 # ex: set syntax=python:
 
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # This is the buildmaster config file for the 'chromeos' bot. It must
 # be installed as 'master.cfg' in your buildmaster's base directory
 # (although the filename can be changed with the --basedir option to
 # 'mktap buildbot master').
 
 # It has one job: define a dictionary named BuildmasterConfig. This
 # dictionary has a variety of keys to control different aspects of the
 # buildmaster. They are documented in docs/config.xhtml .
 
+import json
+import os
 import re
 
 from buildbot.changes.filter import ChangeFilter
 from buildbot.changes.pb import PBChangeSource
 from buildbot.scheduler import Periodic
 
 # These modules come from scripts/master, which must be in the PYTHONPATH.
 from master import master_utils
 from master import slaves_list
 from master.chromeos_manifest_scheduler import \
     ChromeOSManifestSingleBranchScheduler, ChromeOSManifestAnyBranchScheduler, \
     FilterNewSpec, CommentRespectingGitPoller
 from master.cros import builder_config
 from master.factory import chromeos_factory
 
 # These modules come from scripts/common, which must be in the PYTHONPATH.
+import chromiumos_board_config
 import config
 import master_site_config
+from master.cros import builder_config
+from common.cros_chromite import ChromiteTarget
 
 ActiveMaster = master_site_config.ChromiumOS
-
-TREE_GATE_KEEPER = ActiveMaster.is_production_host
-
-# Never enable auto_reboot for folks testing locally.
-DISABLE_AUTO_REBOOT = not ActiveMaster.is_production_host
+DRY_RUN = not ActiveMaster.is_production_host
 
 # This is the dictionary that the buildmaster pays attention to. We also use
 # a shorter alias to save typing.
 c = BuildmasterConfig = {}
 
-BUILDERS = []
+config.DatabaseSetup(c, require_dbconfig=ActiveMaster.is_production_host)
+
 
 # ----------------------------------------------------------------------------
 # BUILDER DEFINITIONS
 
 # The 'builders' list defines the Builders. Each one is configured with a
 # dictionary, using the following keys:
 #  name (required): the name used to describe this bilder
 #  slavename (required): which slave to use, must appear in c['slaves']
 #  builddir (required): which subdirectory to run the builder in
 #  factory (required): a BuildFactory to define how the build is run
 #  category (optional): it is not used in the normal 'buildbot' meaning. It is
 #                       used by JS generation to determine which steps it should
 #                       look for to close the tree.
 #
 
-# Cros helper functions to build builders and factories.
+# General source to add in cbuildbot types:
+def GenCBuild(bc, root_dir=None, auto_reboot=False, branch='master',
+              slave_root='/b', extra_categories=None):
+  """Generate a cbuild buildbot configuration
 
-def GetCBuildbotFactory(config, **kwargs):
-  """Returns cros buildbot factories."""
-  return chromeos_factory.CbuildbotFactory(
-      params=config, dry_run=not ActiveMaster.is_production_host,
-      buildroot='/b/cbuild/external_master', show_gclient_output=False,
-      **kwargs).get_factory()
+      Create a buildbot builder configuration and return a builder
+      dictionary associated with it.
 
+    Arguments:
+      bc: (builder_config.BuilderConfig) The config.
+      root_dir: Root of the directory where all work will take place.
+      name: Name as displayed in the waterfall, if None generate automatically.
+      auto_reboot: bool whether or not the target should reboot between builds
+      branch: The branch to set the builder up for, defaults to 'master'
+      slave_root: The root dir where the buildbot slave is installed.
+    Returns:
+      A builder dictionary assocaited with a factory
+  """
+  categories = ['1release full']
+  if bc.closer:
+    categories.append('closer')
+  else:
+    categories.append('info')
 
-TAGS = {}
+  build_dir_parts = [str(bc.config.name)]
+  if branch:
+    build_dir_parts.append(branch)
 
-class BuilderDefinition(object):
-  def __init__(self, name):
-    self.name = name
+  # All builders on the internal waterfall should use the same directory names
+  # as the trybot waterfall and any other internal waterfalls. The "_master"
+  # suffix is for historical reasons -- we use that suffix regardless of what
+  # branch we're building for.
+  if root_dir is None:
+    root_dir = 'external_master'
 
-  def tag(self, *tags):
-    for tag in tags:
-      TAGS.setdefault(tag, []).append(self.name)
-    return self
+  cbuild_root = os.path.join(slave_root, 'cbuild', root_dir)
 
+  kwargs = {}
+  # Give the SDK builder more time.
+  if bc.config.category == ChromiteTarget.SDK:
+    kwargs['max_time'] = 22*3600

[smut, 2015/04/10 21:17:17]

Consider a constant at the top. And does this really take 22 hours?

[dnj, 2015/04/10 21:42:45]

Good idea - I'm going to pull this out as a builder-type property instead.

And it can definitely take 14+ hours, don't know about 22. It's a pretty silly
builder.

 
-def AddBuilderDefinition(display_name, cbb_name, closer=True, auto_reboot=False,
-                         superTime=None, collapse=False):
-  """Adds a builder definition given by the args.
-
-  Args:
-    display_name: Name displayed on buildbot waterfall.
-    closer:  Do we close the tree based on this build's failure.
-    auto_reboot: Whether to reboot the bot after each run.
-  """
-  category = '1release full|info'
-  builder_definition = BuilderDefinition(display_name)
-  if closer:
-    category = '1release full|closer'
-    builder_definition.tag('closer')
-
-  # Discard suffixes such as " (chromium:12345)".
-  build_dir = re.sub(r'\s\(.*\)$', '', display_name)
-  build_dir = build_dir.replace(' ', '-')
-  if superTime:
-    factory = GetCBuildbotFactory(cbb_name, max_time=superTime)
-  else:
-    factory = GetCBuildbotFactory(cbb_name)
-
-  if DISABLE_AUTO_REBOOT:
-    auto_reboot = False
+  factory = chromeos_factory.CbuildbotFactory(
+      params=str(bc.config.name), branch=branch, buildroot=cbuild_root,
+      show_gclient_output=False, dry_run=DRY_RUN, **kwargs
+      ).get_factory()
 
   builder = {
-    'name': display_name,
-    'builddir': build_dir,
-    'factory': factory,
-    'category': category,
-    'auto_reboot': auto_reboot,
+      'name': str(bc.builder_name),
+      'builddir': '-'.join(build_dir_parts),
+      'factory': factory,
+      'category': '|'.join(categories),
+      'auto_reboot': auto_reboot,
   }
-  if collapse:
+  if bc.collapse:
     builder['collapseRequests'] = builder_config.AlwaysCollapseFunc
-  BUILDERS.append(builder)
-  return builder_definition
+  return builder
 
+# Associate the slaves to the builders.
+c['builders'] = []
 
-def _AddBoardTarget(board, target_type, **kw):
-  return AddBuilderDefinition('%s %s' % (board, target_type),
-                              '%s-%s' % (board, target_type), **kw)
-
-def AddPaladin(board, **kw):
-  kw['closer'] = False
-  return _AddBoardTarget(board, 'paladin', **kw).tag('paladin')
-
-def AddIncremental(board, **kw):
-  kw.setdefault('collapse', True)
-  return _AddBoardTarget(board, 'incremental', **kw).tag('default')
-
-def AddFull(board, **kw):
-  kw.setdefault('collapse', True)
-  return _AddBoardTarget(board, 'full', **kw).tag('default')
-
-def AddASAN(board, **kw):
-  kw.setdefault('collapse', True)
-  return AddBuilderDefinition(
-      '%s ASAN' % (board,),
-      '%s-asan' % (board,), **kw).tag('default')
-
-# Paladin Builders -- exception to closer rule below as they are very important
-# to watch.
-AddPaladin('x86-generic')
-AddPaladin('amd64-generic')
-AddPaladin('amd64-generic_freon')
-AddPaladin('gizmo')
-AddPaladin('mipsel-o32-generic')
-AddPaladin('arm-generic')
-AddPaladin('panther_embedded-minimal')
-
-AddIncremental('x86-generic').tag('x86')
-AddIncremental('amd64-generic').tag('x86')
-AddIncremental('daisy').tag('arm')
-
-# Full Builders
-AddFull('x86-generic').tag('x86')
-AddFull('amd64-generic').tag('x86')
-AddFull('daisy').tag('arm')
-AddFull('arm-generic').tag('arm')
-AddFull('mipsel-o32-generic')
-
-AddBuilderDefinition('chromiumos sdk', 'chromiumos-sdk',
-                     superTime=22*3600, collapse=True).tag('default')
-
-####### Non Closer build defs.
-
-# Miscellaneous builders.
-AddBuilderDefinition('refresh packages (chromium:412795)', 'refresh-packages',
-                     closer=False).tag('refresh')
-AddASAN('x86-generic')
-AddASAN('amd64-generic', closer=False)
-
-
-c['builders'] = BUILDERS
-
-
-config.DatabaseSetup(c, require_dbconfig=ActiveMaster.is_production_host)
+for bc in chromiumos_board_config.builder_configs.itervalues():
+  c['builders'].append(GenCBuild(bc))
 
 
 ####### CHANGESOURCES
 
 MANIFEST_VERSIONS_REPO = (
     'https://chromium.googlesource.com/chromiumos/manifest-versions')
 c['change_source'] = [PBChangeSource()]
 c['change_source'].append(CommentRespectingGitPoller(
     repourl=MANIFEST_VERSIONS_REPO,
     branch='master',
     workdir='/tmp/chromiumos-manifest-versions',
     pollinterval=10))
 
 
 ####### SCHEDULERS
 
+def GetBuilders(func):
+  return [b for b in chromiumos_board_config.builder_configs.itervalues()
+          if func(b)]
+
+
+def GetBuilderNames(func):
+  return [str(builder_config.builder_name)
+          for builder_config in GetBuilders(func)]
+
+
+def GetBuilderNamesForCategory(category):
+  return GetBuilderNames(lambda b: b.config.category == category)
+
+
 ## configure the Schedulers
-# XXX: Changes to builderNames must also be made in:# - slaves.cfg
+# XXX: Changes to builderNames must also be made in:
+# - slaves.cfg
 # - templates/announce.html
 # - And down below in the builder definitions as well
 # - and you probably need to restart any changed slaves as well as the master
 
 s_paladin = ChromeOSManifestSingleBranchScheduler(
   name='paladin',
   change_filter=FilterNewSpec(MANIFEST_VERSIONS_REPO, 'master-paladin'),
-  builderNames=TAGS['paladin'],
+  builderNames=GetBuilderNamesForCategory(ChromiteTarget.PALADIN),
   )
 
 s_refresh_packages = Periodic(
   name='refresh_pkgs_scheduler',
   periodicBuildTimer=24 * 60 * 60, # 1 day
-  builderNames=TAGS['refresh'],
+  builderNames=GetBuilderNamesForCategory(ChromiteTarget.REFRESH_PACKAGES),
   )
 
 # Default scheduler triggers when we see changes.
 repository_fn = lambda x: x != MANIFEST_VERSIONS_REPO
 s_chromeos_default = ChromeOSManifestAnyBranchScheduler(
   name='chromeos',
   change_filter=ChangeFilter(repository_fn=repository_fn, branch='master'),
-  builderNames=TAGS['default'],
+  builderNames=(
+      GetBuilderNamesForCategory(ChromiteTarget.INCREMENTAL) +
+      GetBuilderNamesForCategory(ChromiteTarget.FULL))
   )
 
 c['schedulers'] = [
     s_paladin, s_chromeos_default, s_refresh_packages,
 ]
 
 ####### BUILDSLAVES
 
 # the 'slaves' list defines the set of allowable buildslaves. Each element is a
 # tuple of bot-name and bot-password. These correspond to values given to the
 # buildslave's mktap invocation.
 
 # First, load the list from slaves.cfg.
 slaves = slaves_list.SlavesList('slaves.cfg', 'ChromiumOS')
-
-# Associate the slaves to the builders.
+if not slaves.GetSlaves():
+  raise ValueError("Failed to load slaves.")
 for builder in c['builders']:
   builder['slavenames'] = slaves.GetSlavesName(builder=builder['name'])
 
 # The 'slaves' list defines the set of allowable buildslaves. List all the
 # slaves registered to a builder. Remove dupes.
 c['slaves'] = master_utils.AutoSetupSlaves(c['builders'],
                                            config.Master.GetBotPassword())
 
 
 ####### STATUS TARGETS
 
 # Buildbot master url:
 # Must come before AutoSetupMaster().
 c['buildbotURL'] = ActiveMaster.buildbot_url
 
 # Adds common status and tools to this master.
 master_utils.AutoSetupMaster(c, ActiveMaster,
     templates=['./templates', '../master.chromium/templates'],
     order_console_by_time=True)
 
 ####### BUILDER LIST OUTPUT
-def write_js_reference(builder_groups):
-  """Generate a js file for the waterfall to include."""
 
+def write_js_json(varname, d):
+  """Generate a js file for the waterfall to include.
+
+  We do this by creating a Javascript fragment defining the variable, 'varname',
+  to be the result of parsing emitted JSON.
+  """
+  json_dump = json.dumps(d, indent=2, sort_keys=True)
+  data = 'var %s = %s;' % (varname, json_dump)
   with open('public_html/auto-builder.js', 'w') as f:
-    for name, builders in builder_groups:
-      f.write('var %s = [\n' % name)
-      f.write(',\n'.join(['   "builder=%s"' % b for b in builders]))
-      f.write('  ].join("&");\n')
+    f.write(data)
 
 # This gets called by the shim when we need to write the JS file(s).
 def WriteHTMLFragments():
-  write_js_reference([
-      ('closer', TAGS['closer']),
-      ('x86', TAGS['x86']),
-      ('arm', TAGS['arm']),
-      ('commit_queue', TAGS['paladin']),
-      ])
+  write_js_json('buildercfg', {
+    'closers': GetBuilderNames(lambda b: b.closer),
+    'paladin': GetBuilderNamesForCategory(ChromiteTarget.PALADIN),
+    'incremental': GetBuilderNamesForCategory(ChromiteTarget.INCREMENTAL),
+    'asan': GetBuilderNamesForCategory(ChromiteTarget.ASAN),
+    'full': GetBuilderNamesForCategory(ChromiteTarget.FULL),
+  })
 
 ####### TROOPER NAGGING
 if ActiveMaster.is_production_host:
   from master import chromium_notifier
   categories_steps = {
       'closer': [
           'update_scripts',
           'Clear and Clone chromite',
       ]
   }
 
   c['status'].append(chromium_notifier.ChromiumNotifier(
       fromaddr=ActiveMaster.from_address,
       categories_steps=categories_steps,
       relayhost=config.Master.smtp,
       status_header='%(steps)s failed on "%(builder)s"',
       subject='buildbot trooper alert on %(builder)s (%(projectName)s)',
       extraRecipients=['chrome-troopers@google.com'],
       sheriffs=['sheriff'],
       lookup=master_utils.FilterDomain(),
       use_getname=True))
 
 if not ActiveMaster.is_production_host:
+  # Save our slave pool state. This is populated when our 'slaves' variable
+  # gets generated.
+  chromiumos_board_config.slave_allocator.SaveState(list_unallocated=True)
+  slave_map = chromiumos_board_config.slave_allocator.GetSlaveMap()
+  if slave_map.unallocated:
+    log.msg("The following slaves were not allocated: %s" % (
+        sorted(slave_map.unallocated),))
+
+  # Dump the current configuration.
   master_utils.DumpSetup(c)
+
+  # Disable 'auto_reboot' on slaves for local testing.
+  for builder in c['builders']:
+    builder['auto_reboot'] = False