Add buildbot state machine and master_manager tool.

infra/services/master_lifecycle/buildbot_state.py

Index: infra/services/master_lifecycle/buildbot_state.py
diff --git a/infra/services/master_lifecycle/buildbot_state.py b/infra/services/master_lifecycle/buildbot_state.py
new file mode 100644
index 0000000000000000000000000000000000000000..0664c85c91bb13d4538e9c8e73ed6c43ca997339
--- /dev/null
+++ b/infra/services/master_lifecycle/buildbot_state.py
@@ -0,0 +1,139 @@
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""A state machine to determine and act on a buildbot master's state."""
+
+
+from infra.libs.state_machine import pattern_match
+from infra.libs.time_functions import timestamp
+from infra.libs.buildbot import master
+
+
+STATES = {
+  'buildbot': [
+    'offline',
+    'booting',
+    'started_serving_accept',
+    'started_serving_not_accept',
+    'started_serving_not_accept_drained',
+    'started_not_serving',
+  ],
+  'desired_buildbot_state': [

[agable, 2015/04/27 20:23:08]

As noted in the comments below, the desired states don't make much sense. It
seems like there should be a common set of states that a buildbot can be in
(offline, starting, running, draining, drained, error/other), and that should be
the *same* as the set of possible future desired states. It doesn't make sense
to me to have disjoint sets of "current" states and "desired" states.

[ghost stip (do not use), 2015/04/29 01:30:33]

After online discussion, adjusted to three dimensions: buildbot,
desired_buildbot_state, and desired_transition_time_utc.

+    'offline',
+    'future',

[agable, 2015/04/27 20:23:08]

I have no earthly idea what "desired buildbot state: future" could possibly
mean.

+    'needs_reboot',

[agable, 2015/04/27 20:23:08]

Desired state: needs reboot? I would like this buildbot to require a reboot.
What?

+    'up_to_date',
+  ],
+}
+
+
+def collect_evidence(master_directory, connection_timeout=30):
+  """Collects evidence from the OS for late state determination."""
+  evidence = {}
+  evidence['now'] = timestamp.utcnow_ts()
+  evidence['last_boot'] = master.get_last_boot(master_directory)
+  evidence['last_no_new_builds'] = master.get_last_no_new_builds(
+      master_directory)
+  evidence['buildbot_is_running'] = master.buildbot_is_running(master_directory)
+
+  if evidence['buildbot_is_running']:
+    evidence['accepting_builds'] = master.get_accepting_builds(
+        master_directory, timeout=connection_timeout)
+
+  return evidence
+
+
+def construct_pattern_matcher(
+    boot_timeout_sec=5 * 60, drain_timeout_sec=5 * 60):
+  # There is a bug in pylint which triggers false positives on decorated
+  # decorators with arguments: http://goo.gl/Ln6uyn
+  # pylint: disable=no-value-for-parameter
+  matchlist = pattern_match.MatchList(STATES)
+
+  @matchlist.add_match(
+      buildbot='started_serving_accept',
+      desired_buildbot_state='up_to_date')
+  @matchlist.add_match(
+      buildbot='started_serving_accept',
+      desired_buildbot_state='future')
+  @matchlist.add_match(
+      buildbot='offline',
+      exclusions={'desired_buildbot_state': ['up_to_date', 'needs_reboot']})
+  @matchlist.add_match(
+      buildbot='booting',
+      exclusions={'desired_buildbot_state': ['offline']})
+  @matchlist.add_match(
+      buildbot='started_serving_not_accept')
+  def _do_nothing():
+    return []
+
+  @matchlist.add_match(
+      buildbot='started_serving_not_accept_drained',
+      exclusions={'desired_buildbot_state': ['offline']})
+  @matchlist.add_match(
+      buildbot='started_not_serving',
+      exclusions={'desired_buildbot_state': ['offline']})
+  def _make_restart():
+    return [
+        master.GclientSync, master.MakeStop, master.MakeWait, master.MakeStart]
+
+  @matchlist.add_match(
+      buildbot='started_serving_accept',
+      desired_buildbot_state='needs_reboot')
+  @matchlist.add_match(
+      buildbot='started_serving_accept',
+      desired_buildbot_state='offline')
+  def _make_no_new_builds():
+    return [master.MakeNoNewBuilds]
+
+  @matchlist.add_match(
+      buildbot='offline',
+      desired_buildbot_state='up_to_date')
+  @matchlist.add_match(
+      buildbot='offline',
+      desired_buildbot_state='needs_reboot')
+  def _make_start():
+    return [master.GclientSync, master.MakeStart]
+
+  @matchlist.add_match(
+      buildbot='started_not_serving',
+      desired_buildbot_state='offline')
+  @matchlist.add_match(
+      buildbot='booting',
+      desired_buildbot_state='offline')
+  @matchlist.add_match(
+      buildbot='started_serving_not_accept_drained',
+      desired_buildbot_state='offline')
+  def _make_stop():
+    return [master.MakeStop]
+
+  @matchlist.add_detector('buildbot')
+  def _check_buildbot_state(data):
+    if not data['buildbot_is_running']:
+      return 'offline'
+    if data['accepting_builds'] is None:
+      if data['last_boot'] > (data['now'] - boot_timeout_sec):
+        return 'booting'
+      return 'started_not_serving'
+    if data['accepting_builds']:
+      return 'started_serving_accept'
+    if data['last_no_new_builds'] > (data['now'] - drain_timeout_sec):
+      return 'started_serving_not_accept'
+    return 'started_serving_not_accept_drained'
+
+  @matchlist.add_detector('desired_buildbot_state')
+  def _check_desired_state(data):
+    if not data['desired_buildbot_state']['desired_state']:
+      return 'offline'
+    if data['desired_buildbot_state']['desired_state'] > data['now']:
+      return 'future'
+    if not data['last_boot']:
+      return 'needs_reboot'
+    if data['desired_buildbot_state']['desired_state'] >= data['last_boot']:
+      return 'needs_reboot'
+    return 'up_to_date'
+
+  assert matchlist.is_correct
+  return matchlist