Update floating builder logic, add to "chromiumos"

masters/master.chromiumos/master.cfg

 # -*- python -*-
 # ex: set syntax=python:
 
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # This is the buildmaster config file for the 'chromeos' bot. It must
 # be installed as 'master.cfg' in your buildmaster's base directory
 # (although the filename can be changed with the --basedir option to
 # 'mktap buildbot master').
 
 # It has one job: define a dictionary named BuildmasterConfig. This
 # dictionary has a variety of keys to control different aspects of the
 # buildmaster. They are documented in docs/config.xhtml .
 
+import datetime
 import json
 import os
 import re
 
 from buildbot.changes.filter import ChangeFilter
 from buildbot.changes.pb import PBChangeSource
 from buildbot.scheduler import Periodic
 from twisted.python import log
 
 # These modules come from scripts/master, which must be in the PYTHONPATH.
+from master import floating_builder
 from master import master_utils
 from master import slaves_list
 from master.chromeos_manifest_scheduler import \
     ChromeOSManifestSingleBranchScheduler, ChromeOSManifestAnyBranchScheduler, \
     FilterNewSpec, CommentRespectingGitPoller
 from master.cros import builder_config
 from master.factory import annotator_factory, chromeos_factory
 
 # These modules come from scripts/common, which must be in the PYTHONPATH.
-import chromiumos_board_config
+import chromiumos_board_config as board_config
 import config
 import master_site_config
 from master.cros import builder_config
+from common import slave_alloc
 from common.cros_chromite import ChromiteTarget
 
 ActiveMaster = master_site_config.ChromiumOS
 DRY_RUN = not ActiveMaster.is_production_host
 
 # This is the dictionary that the buildmaster pays attention to. We also use
 # a shorter alias to save typing.
 c = BuildmasterConfig = {}
 
 config.DatabaseSetup(c)
@@ skipping 11 matching lines @@
 #  name (required): the name used to describe this bilder
 #  slavename (required): which slave to use, must appear in c['slaves']
 #  builddir (required): which subdirectory to run the builder in
 #  factory (required): a BuildFactory to define how the build is run
 #  category (optional): it is not used in the normal 'buildbot' meaning. It is
 #                       used by JS generation to determine which steps it should
 #                       look for to close the tree.
 #
 
 # General source to add in cbuildbot types:
-def GenCBuild(cfg):
+def GenCBuild(bc):
   """Generate a cbuild buildbot configuration
 
       Create a buildbot builder configuration and return a builder
       dictionary associated with it.
 
     Arguments:
-      cfg: (builder_config.BuilderConfig) The config.
+      bc: (builder_config.BuilderConfig) The config.
       root_dir: Root of the directory where all work will take place.
       name: Name as displayed in the waterfall, if None generate automatically.
       branch: The branch to set the builder up for, defaults to 'master'
     Returns:
       A builder dictionary assocaited with a factory
   """
   categories = ['1release full']
-  if cfg.closer:
+  if bc.closer:
     categories.append('closer')
   else:
     categories.append('info')
 
   # Give the SDK builder more time.
   factory_kwargs = {}
-  if cfg.timeout:
-    factory_kwargs['max_time'] = cfg.timeout
+  if bc.timeout:
+    factory_kwargs['max_time'] = bc.timeout
 
   properties = {
-      'cbb_config': cfg.config.name,
+      'cbb_config': bc.config.name,
   }
-  if cfg.cbb_variant:
-    properties['cbb_variant'] = cfg.cbb_variant
+  if bc.cbb_variant:
+    properties['cbb_variant'] = bc.cbb_variant
   builder = {
-      'name': str(cfg.builder_name),
-      'builddir': '%s-master' % (cfg.config.name,),
+      'name': str(bc.builder_name),
+      'builddir': '%s-master' % (bc.config.name,),
       'category': '|'.join(categories),
-      'auto_reboot': cfg.auto_reboot,
+      'auto_reboot': bc.auto_reboot,
       'factory': chromeos_factory.ChromiteRecipeFactory(
           factory_obj, 'cros/cbuildbot', **factory_kwargs),
       'properties': properties,
   }
 
-  if cfg.collapse:
+  if bc.collapse:
     builder['mergeRequests'] = builder_config.AlwaysCollapseFunc
   return builder
 
 # Associate the slaves to the builders.
 c['builders'] = []
 
-for cfg in chromiumos_board_config.builder_configs.itervalues():
+for cfg in board_config.builder_configs.itervalues():
   c['builders'].append(GenCBuild(cfg))
 
 
 ####### CHANGESOURCES
 
 MANIFEST_VERSIONS_REPO = (
     'https://chromium.googlesource.com/chromiumos/manifest-versions')
 c['change_source'] = [PBChangeSource()]
 c['change_source'].append(CommentRespectingGitPoller(
     repourl=MANIFEST_VERSIONS_REPO,
     branch='master',
     workdir='/tmp/chromiumos-manifest-versions',
     pollinterval=10))
 
 
 ####### SCHEDULERS
 
 def GetBuilders(func):
-  return [b for b in chromiumos_board_config.builder_configs.itervalues()
+  return [b for b in board_config.builder_configs.itervalues()
           if func(b)]
 
 
 def GetBuilderNames(func):
   return [str(builder_config.builder_name)
           for builder_config in GetBuilders(func)]
 
 
 def GetBuilderNamesForCategory(category):
   return GetBuilderNames(lambda b: b.config.category == category)
@@ skipping 36 matching lines @@
       GetBuilderNamesForCategory(ChromiteTarget.SDK)
   ),
 )
 
 c['schedulers'] = [
     s_paladin, s_pfq, s_chromeos_default, s_refresh_packages,
 ]
 
 ####### BUILDSLAVES
 
+# Returns 'True' if a builder is experimental.
+def cros_builder_experimental(name):
+  config = board_config.builder_name_map.get(name)
+  return config and config.is_experimental
+
 # the 'slaves' list defines the set of allowable buildslaves. Each element is a
 # tuple of bot-name and bot-password. These correspond to values given to the
 # buildslave's mktap invocation.
 
 # First, load the list from slaves.cfg.
 slaves = slaves_list.SlavesList('slaves.cfg', 'ChromiumOS')
-if not slaves.GetSlaves():
-  raise ValueError("Failed to load slaves.")
-for builder in c['builders']:
-  builder['slavenames'] = slaves.GetSlavesName(builder=builder['name'])
+slave_map = board_config.slave_allocator.GetSlaveMap()
+
+# Assert our slave integrity and build our floating slave list.
+#
+# We will loop through every builder, identify the primary and floating slaves
+# for that builder, and set its 'nextSlave' function to our floating builder.
+slave_class_map = slave_alloc.BuildClassMap(slave_map)
+for b in c['builders']:
+  builder_name = b['name']
+  b['slavenames'] = slaves.GetSlavesName(builder=builder_name)
+  assert b['slavenames'], 'No slaves allocated for [%s]' % (builder_name,)
+
+  bc = board_config.builder_configs[builder_name]
+  if bc.floating and not cros_builder_experimental(builder_name):
+    fs = floating_builder.FloatingSet()
+
+    for slave in b['slavenames']:
+      if slave in slave_class_map.get('floating', {}).get(bc.config.category, ()):
+        fs.AddFloating(slave)
+      else:
+        fs.AddPrimary(slave)
+    b['nextSlave'] = fs.NextSlaveFunc(datetime.timedelta(minutes=15))
+
+    # Enable verbose logging.
+    # TODO(dnj): Disable when reliable.
+    b['nextSlave'].verbose = True
+    log.msg('Assigning builder [%s] to `nextSlave` function: %s' % (
+        builder_name, b['nextSlave']))
 
 # The 'slaves' list defines the set of allowable buildslaves. List all the
 # slaves registered to a builder. Remove dupes.
 c['slaves'] = master_utils.AutoSetupSlaves(c['builders'],
                                            config.Master.GetBotPassword())
 
 
 ####### STATUS TARGETS
 
 # Buildbot master url:
 # Must come before AutoSetupMaster().
 c['buildbotURL'] = ActiveMaster.buildbot_url
 
-# Returns 'True' if a builder is experimental.
-def cros_builder_experimental(name):
-  config = chromiumos_board_config.builder_name_map.get(name)
-  return config and config.is_experimental
-
 # Adds common status and tools to this master.
 def cros_builder_doc(name):
-  config = chromiumos_board_config.builder_name_map.get(name)
+  config = board_config.builder_name_map.get(name)
   if config:
     doc = config.config.get('doc')
     if doc:
       return {'url': doc}
   return None
 
 web_template_globals = {
     'cros_builder_experimental': cros_builder_experimental,
     'cros_builder_doc': cros_builder_doc,
 }
@@ skipping 35 matching lines @@
   categories_steps = {
       'closer': [
           'update_scripts',
           'Clear and Clone chromite',
       ]
   }
 
 if not ActiveMaster.is_production_host:
   # Save our slave pool state. This is populated when our 'slaves' variable
   # gets generated.
-  chromiumos_board_config.slave_allocator.SaveState()
-  slave_map = chromiumos_board_config.slave_allocator.GetSlaveMap()
+  board_config.slave_allocator.SaveState()
   if slave_map.unallocated:
     log.msg("The following slaves were not allocated: %s" % (
         sorted(slave_map.unallocated),))
 
   # Disable 'auto_reboot' on slaves for local testing.
   for builder in c['builders']:
     builder['auto_reboot'] = False