Add buildbot state machine and master_manager tool.

infra/services/master_lifecycle/buildbot_state.py

Index: infra/services/master_lifecycle/buildbot_state.py
diff --git a/infra/services/master_lifecycle/buildbot_state.py b/infra/services/master_lifecycle/buildbot_state.py
new file mode 100644
index 0000000000000000000000000000000000000000..dba2412445d019b70658607474c69cda77568147
--- /dev/null
+++ b/infra/services/master_lifecycle/buildbot_state.py
@@ -0,0 +1,167 @@
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""A state machine to determine and act on a buildbot master's state."""
+
+
+from infra.libs.state_machine import pattern_match
+from infra.libs.time_functions import timestamp
+from infra.libs.buildbot import master
+
+
+STATES = {
+  'buildbot': [
+    'offline',
+    'starting',
+    'running',
+    'draining',
+    'drained',
+    'crashed',
+  ],
+  'desired_buildbot_state': [
+    'offline',
+    'running',
+    'drained',
+  ],
+  'desired_transition_time_utc': [
+    'transition_hasnt_happened',
+    'transition_happened',
+    'future_transition',

[iannucci, 2015/04/29 01:55:12]

ready, done, future

+  ],
+}
+
+
+def collect_evidence(master_directory, connection_timeout=30):
+  """Collects evidence from the OS for late state determination."""
+  evidence = {}
+  evidence['now'] = timestamp.utcnow_ts()
+  evidence['last_boot'] = master.get_last_boot(master_directory)
+  evidence['last_no_new_builds'] = master.get_last_no_new_builds(
+      master_directory)
+  evidence['buildbot_is_running'] = master.buildbot_is_running(master_directory)
+
+  if evidence['buildbot_is_running']:
+    evidence['accepting_builds'] = master.get_accepting_builds(
+        master_directory, timeout=connection_timeout)
+
+  return evidence
+
+
+def construct_pattern_matcher(
+    boot_timeout_sec=5 * 60, drain_timeout_sec=5 * 60):
+  # There is a bug in pylint which triggers false positives on decorated
+  # decorators with arguments: http://goo.gl/Ln6uyn
+  # pylint: disable=no-value-for-parameter
+  matchlist = pattern_match.MatchList(STATES)
+
+  @matchlist.add_match(

[iannucci, 2015/04/29 01:55:12]

what about a

@matchlist.add_match(
  desired_transition_time="future")

?

+      buildbot='running',
+      desired_buildbot_state='running',
+      exclusions={'desired_transition_time_utc': ['transition_hasnt_happened']})

[iannucci, 2015/04/29 01:55:12]

what about

  desired_transition_time_utc=matchlist.NOT('transition_hasnt_happened')

? That would read a lot more cleanly than the exclusions thing

+  @matchlist.add_match(
+      buildbot='running',
+      desired_transition_time_utc='future_transition')
+  @matchlist.add_match(
+      buildbot='drained',
+      desired_buildbot_state='drained',
+      exclusions={'desired_transition_time_utc': ['transition_hasnt_happened']})
+  @matchlist.add_match(
+      buildbot='offline',
+      desired_buildbot_state='offline')
+  @matchlist.add_match(
+      buildbot='offline',
+      desired_transition_time_utc='future_transition')
+  @matchlist.add_match(
+      buildbot='starting',
+      exclusions={'desired_buildbot_state': ['offline']})
+  @matchlist.add_match(
+      buildbot='draining')
+  def _do_nothing():
+    return []
+
+  @matchlist.add_match(
+      buildbot='drained',
+      desired_buildbot_state='running')
+  @matchlist.add_match(
+      buildbot='drained',
+      desired_buildbot_state='drained',
+      desired_transition_time_utc='transition_hasnt_happened')
+  @matchlist.add_match(
+      buildbot='crashed',
+      exclusions={'desired_buildbot_state': ['offline']})
+  def _make_restart():
+    return [
+        master.GclientSync, master.MakeStop, master.MakeWait, master.MakeStart]
+
+  @matchlist.add_match(
+      buildbot='running',
+      desired_buildbot_state='running',
+      desired_transition_time_utc='transition_hasnt_happened')
+  @matchlist.add_match(
+      buildbot='running',
+      desired_buildbot_state='offline',
+      exclusions={'desired_transition_time_utc': ['future_transition']})
+  @matchlist.add_match(
+      buildbot='running',
+      desired_buildbot_state='drained',
+      exclusions={'desired_transition_time_utc': ['future_transition']})
+  def _make_no_new_builds():
+    return [master.MakeNoNewBuilds]
+
+  @matchlist.add_match(
+      buildbot='offline',
+      exclusions={
+        'desired_buildbot_state': ['offline'],
+        'desired_transition_time_utc': ['future_transition'],
+      })
+  def _make_start():
+    return [master.GclientSync, master.MakeStart]
+
+  @matchlist.add_match(
+      buildbot='crashed',
+      desired_buildbot_state='offline')
+  @matchlist.add_match(
+      buildbot='starting',
+      desired_buildbot_state='offline')
+  @matchlist.add_match(
+      buildbot='drained',
+      desired_buildbot_state='offline')
+  def _make_stop():
+    return [master.MakeStop]
+
+  @matchlist.add_detector('buildbot')
+  def _check_buildbot_state(data):
+    if not data['buildbot_is_running']:
+      return 'offline'
+    if data['accepting_builds'] is None:

[iannucci, 2015/04/29 01:55:12]

later: let's invert this and call it 'draining_builds' since that's really more
like what it indicates.

+      if data['last_boot'] > (data['now'] - boot_timeout_sec):
+        return 'starting'
+      return 'crashed'
+    if data['accepting_builds']:
+      return 'running'
+    if data['last_no_new_builds'] > (data['now'] - drain_timeout_sec):
+      return 'draining'
+    return 'drained'
+
+  @matchlist.add_detector('desired_buildbot_state')
+  def _check_desired_state(data):
+    desired_state = data['desired_buildbot_state']['desired_state']
+    if desired_state in STATES['desired_buildbot_state']:
+      return desired_state
+
+    raise ValueError('%s is not a valid desired_buildbot_state' % desired_state)
+
+  @matchlist.add_detector('desired_transition_time_utc')
+  def _check_transition_time(data):
+    transition_time = data['desired_buildbot_state']['transition_time_utc']
+    if transition_time > data['now']:
+      return 'future_transition'
+    if transition_time >= data['last_boot']:
+      return 'transition_hasnt_happened'
+    return 'transition_happened'
+
+
+  assert matchlist.is_correct
+  return matchlist