Add ChromiumOS Trybot support.

masters/master.tryserver.chromiumos/master.cfg

Index: masters/master.tryserver.chromiumos/master.cfg
===================================================================
--- masters/master.tryserver.chromiumos/master.cfg	(revision 105558)
+++ masters/master.tryserver.chromiumos/master.cfg	(working copy)
@@ -5,188 +5,36 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-# This is the buildmaster config file for the 'chromeos' bot. It must
-# be installed as 'master.cfg' in your buildmaster's base directory
-# (although the filename can be changed with the --basedir option to
-# 'mktap buildbot master').
-
-# It has one job: define a dictionary named BuildmasterConfig. This
-# dictionary has a variety of keys to control different aspects of the
-# buildmaster. They are documented in docs/config.xhtml .
-
-from buildbot.changes import svnpoller
-from buildbot.changes.pb import PBChangeSource
-from buildbot.scheduler import Dependent
-from buildbot.scheduler import Periodic
-from buildbot.scheduler import Scheduler
-from buildbot.scheduler import Triggerable
-
 # These modules come from scripts/master, which must be in the PYTHONPATH.
-from master import build_utils
+import master.cros_builder_mapping as cros_builder_mapping
+# Perform reload so a 'make reconfig' picks up changes to cros_builder_mapping
+reload(cros_builder_mapping)
+
 from master import master_utils
 from master import slaves_list
+from master.cros_builders_pools import CrOSBuildersPools
+from master.cros_try_job_git import CrOSTryJobGit
 from master.factory import chromeos_factory
 
 # These modules come from scripts/common, which must be in the PYTHONPATH.
 import config
 
-ActiveMaster = config.Master.ChromiumOS
+ActiveMaster = config.Master.ChromiumOSTryServer
 
-TREE_GATE_KEEPER = ActiveMaster.is_production_host
-GOOD_REVISIONS = False
-
 # This is the dictionary that the buildmaster pays attention to. We also use
 # a shorter alias to save typing.
 c = BuildmasterConfig = {}
 
-# This is a dictionary that maps waterfall dashboard names to cbuildbot_configs.
-# In order to add a new dict, you must edit this first.
-_NAME_CONFIG_DICT = {
-  'x86 generic PFQ': 'x86-generic-pre-flight-queue',
-  'x86 generic commit queue': 'x86-generic-commit-queue',
-  'arm generic PFQ': 'arm-generic-bin',
-  'tegra2 PFQ': 'arm-tegra2-bin',
-  'x86 generic full': 'x86-generic-full',
-  'arm generic full': 'arm-generic-full',
-  'tegra2 full': 'arm-tegra2-full',
-  'tegra2 seaboard full': 'arm-tegra2-seaboard-full',
-  'x86 pineview full': 'x86-pineview-full',
-  'x86 generic chrome PFQ': 'x86-generic-chrome-pre-flight-queue',
-  'arm generic chrome PFQ': 'arm-generic-chrome-pre-flight-queue',
-  'tegra2 chrome PFQ': 'arm-tegra2-chrome-pre-flight-queue',
-  'chromiumos sdk': 'chromiumos-sdk',
-  'refresh packages': 'refresh-packages',
-  'x86 generic ASAN': 'x86-generic-asan',
-}
-
 ####### CHANGESOURCES
-def ChromeTreeFileSplitter(path):
-  """split_file for chrome tree changes; return None for uninteresting paths."""
-  # List of projects we are interested in. The project names must exactly
-  # match paths in the Subversion repository, relative to the 'path' URL
-  # argument. build_utils.SplitPath() will use them as branch names to
-  # kick off the Schedulers for different projects.
-  projects = ['releases']
-  return build_utils.SplitPath(projects, path)
 
+c['change_source'] = []
 
-chromium_rev = 'http://src.chromium.org/viewvc/chrome?view=rev&revision=%s'
+####### BUILDERS
 
-chrome_trunk_poller = svnpoller.SVNPoller(
-    svnurl='svn://svn.chromium.org/chrome/',
-    split_file=ChromeTreeFileSplitter,
-    pollinterval=300,
-    revlinktmpl=chromium_rev)
+c['builders'] = []
+pools = CrOSBuildersPools(default_pool_name='ChromeOS',
+                          builder_mapping=cros_builder_mapping.CONFIG_NAME_DICT)
 
-pb_source = PBChangeSource()
-
-c['change_source'] = [pb_source, chrome_trunk_poller]
-
-_TOOLCHAIN_TARGETS = ['amd64-generic-full']
-_TOOLCHAIN_TARGETS_CLOSER = ['arm-tegra2-seaboard-toolchain',
-                             'x86-generic-gcc_46',
-                             'arm-tegra2-seaboard-gcc_46']
-
-####### SCHEDULERS
-
-## configure the Schedulers
-
-# Pre-flight scheduler for changes in Chromium OS.  All other slaves are either
-# triggered or depend on this scheduler.
-# XXX: Changes to builderNames must also be made in:# - slaves.cfg
-# - public_html/announce.html
-# - And down below in the builder definitions as well
-# - and you probably need to restart any changed slaves as well as the master
-s_chromiumos_sdk = Scheduler(
-  name='sdk_scheduler',
-  # Renamed 'src' to 'master' for git_buildbot.py
-  branch='master',
-  treeStableTimer=0,
-  builderNames=[
-     'chromiumos sdk',
-  ])
-
-s_refresh_packages = Periodic(
-  name='refresh_pkgs_scheduler',
-  branch='master',
-  periodicBuildTimer=24*60*60, # 1 day
-  builderNames=[
-    'refresh packages',
-  ])
-
-s_cq = Periodic(
-  name='commit_queue_scheduler',
-  branch='master',
-  periodicBuildTimer=5*60, # 5 minutes
-  builderNames=[
-    'x86 generic commit queue',
-  ])
-
-s_asan = Periodic(
-  name='x86_generic_asan_scheduler',
-  branch='master',
-  periodicBuildTimer=5*60, # 5 minutes
-  builderNames=[
-    'x86 generic ASAN',
-  ])
-
-s_pre_flight_queue = Scheduler(
-  name='pre_flight_queue',
-  # Renamed 'src' to 'master' for git_buildbot.py
-  branch='master',
-  treeStableTimer=0,
-  builderNames=[
-     'x86 generic PFQ',
-  ])
-
-# Pre-flight scheduler for deps changes of Chromium for Chromium OS.
-s_chrome_pre_flight_queue = Scheduler(
-  name='chrome_pre_flight_queue',
-  branch='releases',
-  treeStableTimer=0,
-  builderNames=[
-     'x86 generic chrome PFQ',
-  ])
-
-# Triggerable for chrome-pre-flight-queue-slaves.  All chrome PFQ's should be
-# triggered from master chrome pfq.
-s_chrome_pre_flight_queue_slaves = Triggerable(
-  name='chrome_pre_flight_queue_slaves',
-  builderNames=[
-     'arm generic chrome PFQ',
-     'tegra2 chrome PFQ',
-  ])
-
-# Triggerable for other pre-flight-queue slaves. We use triggers so that
-# the other slaves are guaranteed to use the same sourcestamp as the main.
-s_pre_flight_queue_slaves = Triggerable(
-  name='pre_flight_queue_slaves',
-  builderNames=[
-     'arm generic PFQ',
-     'tegra2 PFQ',
-  ])
-
-# Only trigger fulls when we have a newly revved build.
-s_chromeos_full = Dependent(
-  name='chromeos',
-  upstream=s_pre_flight_queue,
-  builderNames=[
-     'arm generic full',
-     'tegra2 full',
-     'tegra2 seaboard full',
-     'x86 generic full',
-     'x86 pineview full',
-  ] + _TOOLCHAIN_TARGETS)
-
-c['schedulers'] = [
-    s_pre_flight_queue, s_pre_flight_queue_slaves,
-    s_chrome_pre_flight_queue, s_chrome_pre_flight_queue_slaves,
-    s_chromeos_full, s_chromiumos_sdk,
-    s_refresh_packages, s_cq, s_asan,
-]
-
-BUILDERS = []
-
 # ----------------------------------------------------------------------------
 # BUILDER DEFINITIONS
 
@@ -203,51 +51,33 @@
 
 # Cros helper functions to build builders and factories.
 
-def GetCBuildbotFactory(config, is_master=False):
+def GetCBuildbotFactory(config):
   """Returns cros buildbot factories."""
   return chromeos_factory.CbuildbotFactory(
-      params=config, dry_run=not ActiveMaster.is_production_host,
-      crostools_repo=None,
-      is_master=is_master).get_factory()
+      params=config, trybot=True, crostools_repo=None).get_factory()
 
-def GetChromeCBuildbotFactory(config, is_master=False):
-  """Returns buildbot factories specifically for Chrome pfqs."""
-  return chromeos_factory.ChromeCbuildbotFactory(
-      params=config, chrome_rev_stages=[],
-      dry_run=not ActiveMaster.is_production_host,
-      crostools_repo=None,
-      is_master=is_master).get_factory()
 
-def AddBuilderDefinition(display_name, closer=True, chrome_factory=False,
-                         is_master=False):
+def AddBuilderDefinition(display_name):
   """Adds a builder definition given by the args.
 
   Args:
     display_name: Name displayed on buildbot waterfall.
-    closer:  Do we close the tree based on this build's failure.
-    chrome_factory: Use the chrome factory rather than normal factory.
-    is_master:  One set per pfq type.
   """
   def GetConfig():
-    return _NAME_CONFIG_DICT.get(display_name, display_name)
+    return cros_builder_mapping.NAME_CONFIG_DICT.get(display_name, display_name)
 
-  category = '1release full|info'
-  if closer: category = '1release full|closer'
-  build_dir = display_name.replace(' ', '-')
-  if chrome_factory:
-    factory = GetChromeCBuildbotFactory(GetConfig(), is_master)
-  else:
-    factory = GetCBuildbotFactory(GetConfig(), is_master)
-
-  BUILDERS.append({
+  c['builders'].append({
     'name': display_name,
-    'builddir': build_dir,
-    'factory': factory,
-    'category': category,
+    'builddir': display_name.replace(' ', '-'),
+    'factory': GetCBuildbotFactory(GetConfig()),
+    'category': '1release full|info',
   })
 
+  pools['ChromeOS'].append(display_name)
+
+
 # Pre Flight Queues
-AddBuilderDefinition('x86 generic PFQ', is_master=True)
+AddBuilderDefinition('x86 generic PFQ')
 AddBuilderDefinition('arm generic PFQ')
 AddBuilderDefinition('tegra2 PFQ')
 
@@ -258,29 +88,8 @@
 AddBuilderDefinition('tegra2 seaboard full')
 AddBuilderDefinition('x86 pineview full')
 
-# Chrome pre-flight builds.
-AddBuilderDefinition('x86 generic chrome PFQ', chrome_factory=True,
-                      is_master=True)
-AddBuilderDefinition('arm generic chrome PFQ', chrome_factory=True,)
-AddBuilderDefinition('tegra2 chrome PFQ', chrome_factory=True,)
-
 AddBuilderDefinition('chromiumos sdk',)
 
-for builder in _TOOLCHAIN_TARGETS_CLOSER:
-  AddBuilderDefinition(builder, closer=True)
-
-####### Non Closer build defs.
-
-AddBuilderDefinition('x86 generic commit queue', closer=False)
-AddBuilderDefinition('refresh packages', closer=False)
-AddBuilderDefinition('x86 generic ASAN', closer=False)
-
-for builder in _TOOLCHAIN_TARGETS:
-  AddBuilderDefinition(builder, closer=False)
-
-c['builders'] = BUILDERS
-
-
 ####### BUILDSLAVES
 
 # the 'slaves' list defines the set of allowable buildslaves. Each element is a
@@ -288,7 +97,7 @@
 # buildslave's mktap invocation.
 
 # First, load the list from slaves.cfg.
-slaves = slaves_list.SlavesList('slaves.cfg', 'ChromiumOS')
+slaves = slaves_list.SlavesList('slaves.cfg', 'ChromiumOSTryServer')
 
 # Associate the slaves to the builders.
 for builder in c['builders']:
@@ -301,92 +110,41 @@
 c['slaves'] = master_utils.AutoSetupSlaves(c['builders'],
                                            config.Master.GetBotPassword())
 
+####### SCHEDULERS
 
+c['schedulers'] = []
+
+c['schedulers'].append(CrOSTryJobGit(
+    name='cros_try_job_git',
+    pools=pools,
+    repo_url=ActiveMaster.repo_url
+    ))
+
 ####### STATUS TARGETS
 
 # Adds common status and tools to this master.
 master_utils.AutoSetupMaster(c, ActiveMaster, order_console_by_time=True)
 
-# Add more.
+# Add a dumb MailNotifier first so it will be used for BuildSlave with
+# notify_on_missing set when they go missing.
+from buildbot.status import mail
+c['status'].append(mail.MailNotifier(
+    fromaddr=ActiveMaster.from_address,
+    builders=[],
+    relayhost=config.Master.smtp,
+    lookup=master_utils.UsersAreEmails()))
 
-if TREE_GATE_KEEPER:
-  from master import gatekeeper
-  # This is the list of the builder categories and the corresponding critical
-  # steps. If one critical step fails, gatekeeper will close the tree
-  # automatically.
-  # Note: don't include 'update scripts' since we can't do much about it when
-  # it's failing and the tree is still technically fine.
-  categories_steps = {
-      'closer': [
-          'update', 'configure build', 'setup mirrors', 'update chroot',
-          'setup board', 'build packages', 'run tests',
-          'build image', 'create qemu image', 'run smoke bvt', 'archived build',
-          'cbuildbot', 'cbuildbot_master', 'cbuildbot_chrome_latest_release',
-      ],
-      'info': [
-      ],
-      '': [
-      ],
-  }
-  exclusions = {}
-  # TODO: convert GateKeeper to 'use_getname=True'
-  # 1) set names for all steps in categories_steps
-  # 2) update categories_steps to use names insteads of description
-  # 3) change forgiving_steps to:
-  #     forgiving_steps = ['update_scripts', 'update', 'archive_build']
-  # 4) add 'use_getname=True' to the GateKeeper call
-  forgiving_steps = ['update scripts', 'update', 'archived build']
-  c['status'].append(gatekeeper.GateKeeper(
-      fromaddr=ActiveMaster.from_address,
-      categories_steps=categories_steps,
-      exclusions=exclusions,
-      relayhost=config.Master.smtp,
-      subject='buildbot %(result)s in %(projectName)s on %(builder)s, '
-              'revision %(revision)s',
-      extraRecipients=ActiveMaster.tree_closing_notification_recipients,
-      lookup=master_utils.FilterDomain(),
-      forgiving_steps=forgiving_steps,
-      tree_status_url=ActiveMaster.tree_status_url,
-      check_revisions=False))
+# Try job result emails.
+from master.try_mail_notifier import TryMailNotifier
+c['status'].append(TryMailNotifier(
+    fromaddr=ActiveMaster.from_address,
+    subject="try %(result)s for %(reason)s on %(builder)s",
+    mode='all',
+    relayhost=config.Master.smtp,
+    lookup=master_utils.UsersAreEmails()))
 
-if GOOD_REVISIONS:
-  # TODO: update for ChromeOS
-  from master import goodrevisions
-  # This is the list of builders with their respective list of critical steps
-  # that all need to succeed to mark a revision as successful. A single failure
-  # in any of the steps of any of the builders will mark the revision as failed.
-  good_revision_steps = {
-      'XP Tests (dbg)(2)': ['unit_tests'],
-      'Mac10.5 Tests (dbg)(2)': ['unit_tests'],
-      'Chromium Linux': ['unit_tests'],
-    }
-  c['status'].append(goodrevisions.GoodRevisions(
-      good_revision_steps=good_revision_steps))
 
-
-####### TROOPER NAGGING
-if ActiveMaster.is_production_host:
-  from master import chromium_notifier
-  categories_steps = {
-      'closer': [
-          'CleanUp', 'LKGMCandidateSync', 'Sync', 'Uprev', 'UploadPrebuilts',
-          'Archive', 'PublishUprevChanges'
-      ]
-  }
-
-  c['status'].append(chromium_notifier.ChromiumNotifier(
-      fromaddr=ActiveMaster.from_address,
-      categories_steps=categories_steps,
-      relayhost=config.Master.smtp,
-      status_header='%(steps)s failed on "%(builder)s"',
-      subject='buildbot trooper alert on %(builder)s (%(projectName)s)',
-      extraRecipients=['troopers@chromium.org'],
-      send_to_sheriffs=True,
-      lookup=master_utils.FilterDomain(),
-      use_getname=True))
-
-
 ####### PROJECT IDENTITY
 
 # Buildbot master url:
-c['buildbotURL'] = 'http://build.chromium.org/buildbot/chromiumos/'
+c['buildbotURL'] = ''