Add buildbot state machine and master_manager tool.

infra/services/master_lifecycle/buildbot_state.py

Index: infra/services/master_lifecycle/buildbot_state.py
diff --git a/infra/services/master_lifecycle/buildbot_state.py b/infra/services/master_lifecycle/buildbot_state.py
new file mode 100644
index 0000000000000000000000000000000000000000..8a2f4f56b5daabab2550a556eb52294de2b80642
--- /dev/null
+++ b/infra/services/master_lifecycle/buildbot_state.py
@@ -0,0 +1,160 @@
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""A state machine to determine and act on a buildbot master's state."""
+
+
+from infra.libs.state_machine import pattern_match
+from infra.libs.time_functions import timestamp
+from infra.libs.buildbot import master
+
+
+STATES = {

[agable, 2015/04/30 21:23:30]

Love it, this set of states is great.

+  'buildbot': [
+    'offline',
+    'starting',
+    'running',
+    'draining',
+    'drained',
+    'crashed',
+  ],
+  'desired_buildbot_state': [
+    'offline',
+    'running',
+    'drained',
+  ],
+  'desired_transition_time': [
+    'ready_to_fire',
+    'hold_steady',
+  ],
+}
+
+
+def collect_evidence(master_directory, connection_timeout=30):
+  """Collects evidence from the OS for late state determination."""
+  evidence = {}
+  evidence['now'] = timestamp.utcnow_ts()
+  evidence['last_boot'] = master.get_last_boot(master_directory)
+  evidence['last_no_new_builds'] = master.get_last_no_new_builds(
+      master_directory)
+  evidence['buildbot_is_running'] = master.buildbot_is_running(master_directory)
+
+  if evidence['buildbot_is_running']:
+    evidence['accepting_builds'] = master.get_accepting_builds(
+        master_directory, timeout=connection_timeout)
+
+  return evidence
+
+
+def construct_pattern_matcher(
+    boot_timeout_sec=5 * 60, drain_timeout_sec=5 * 60):
+  # There is a bug in pylint which triggers false positives on decorated
+  # decorators with arguments: http://goo.gl/Ln6uyn
+  # pylint: disable=no-value-for-parameter
+  matchlist = pattern_match.MatchList(STATES)
+
+  @matchlist.add_match(
+      buildbot='running',
+      desired_buildbot_state='running',
+      desired_transition_time='hold_steady')
+  @matchlist.add_match(
+      buildbot='drained',
+      desired_buildbot_state='drained',
+      desired_transition_time='hold_steady')
+  @matchlist.add_match(
+      buildbot='offline',
+      desired_buildbot_state='offline')
+  @matchlist.add_match(
+      buildbot='starting',
+      exclusions={'desired_buildbot_state': ['offline']})
+  @matchlist.add_match(
+      buildbot='draining')
+  def _do_nothing():
+    return []
+
+  @matchlist.add_match(
+      buildbot='drained',
+      desired_buildbot_state='running')
+  @matchlist.add_match(
+      buildbot='drained',
+      desired_buildbot_state='drained',
+      desired_transition_time='ready_to_fire')
+  @matchlist.add_match(
+      buildbot='crashed',
+      exclusions={'desired_buildbot_state': ['offline']})
+  def _make_restart():
+    return [
+        master.GclientSync, master.MakeStop, master.MakeWait, master.MakeStart]
+
+  @matchlist.add_match(
+      buildbot='running',
+      desired_buildbot_state='running',
+      desired_transition_time='ready_to_fire')
+  @matchlist.add_match(
+      buildbot='running',
+      desired_buildbot_state='offline')
+  @matchlist.add_match(
+      buildbot='running',
+      desired_buildbot_state='drained')
+  def _make_no_new_builds():
+    return [master.MakeNoNewBuilds]
+
+  @matchlist.add_match(
+      buildbot='offline',
+      exclusions={
+        'desired_buildbot_state': ['offline'],
+      })
+  def _make_start():
+    return [master.GclientSync, master.MakeStart]
+
+  @matchlist.add_match(
+      buildbot='crashed',
+      desired_buildbot_state='offline')
+  @matchlist.add_match(
+      buildbot='starting',
+      desired_buildbot_state='offline')
+  @matchlist.add_match(
+      buildbot='drained',
+      desired_buildbot_state='offline')
+  def _make_stop():
+    return [master.MakeStop]
+
+  @matchlist.add_detector('buildbot')
+  def _check_buildbot_state(data):
+    if not data['buildbot_is_running']:
+      return 'offline'
+    if data['accepting_builds'] is None:
+      if data['last_boot'] > (data['now'] - boot_timeout_sec):
+        return 'starting'
+      return 'crashed'
+    if data['accepting_builds']:
+      return 'running'
+    if data['last_no_new_builds'] > (data['now'] - drain_timeout_sec):
+      return 'draining'
+    return 'drained'
+
+  @matchlist.add_detector('desired_buildbot_state')
+  def _check_desired_state(data):
+    desired_state = data['desired_buildbot_state']['desired_state']
+    if desired_state in STATES['desired_buildbot_state']:
+      return desired_state
+
+    raise ValueError('%s is not a valid desired_buildbot_state' % desired_state)
+
+  @matchlist.add_detector('desired_transition_time')
+  def _check_transition_time(data):
+    transition_time = data['desired_buildbot_state']['transition_time_utc']
+    if transition_time > data['now']:
+      # If we specify a date in the future and request 'running', the state
+      # machine will continually reboot buildbot until that time is reached.
+      raise ValueError(
+          'specifying a date in the future creates ambiguity about now')
+    if transition_time >= data['last_boot']:
+      return 'ready_to_fire'
+    return 'hold_steady'
+
+
+  assert matchlist.is_correct
+  return matchlist