Add buildbot state machine and master_manager tool.

infra/tools/master_manager/__main__.py

+#!/usr/bin/python
+# Copyright 2015 Google Inc. All Rights Reserved.
+# pylint: disable=F0401
+
+"""Start, restart and shut down masters as needed."""
+
+import argparse
+from functools import partial
+import json
+import os
+import subprocess
+import sys
+
+from infra.services.master_lifecycle import buildbot_state
+from infra.libs.buildbot import master
+from infra.libs.service_utils import daemon
+from infra.libs.service_utils import outer_loop
+
+
+def parse_args():  # pragma: no cover

[ghost stip (do not use), 2015/04/24 07:06:55]

this file basically has no unittests. I'm not sure what the value in adding
tests to it would be, but I can if you think it necessary

+  parser = argparse.ArgumentParser(
+      description='Manage the state of a buildbot master.')
+  parser.add_argument('directory', nargs='?',
+      help='location of the master to manage')
+  parser.add_argument('--desired-state-file', default='desired_state.json',
+      help='location of the master to manage')

[agable, 2015/04/27 20:23:08]

Um

[ghost stip (do not use), 2015/04/29 01:30:33]

Acknowledged.

+  parser.add_argument('--list-all-states', action='store_true',
+      help='list all states with their actions and exit')
+  parser.add_argument('--enable-gclient-sync', action='store_true',
+      help='perform a gclient sync before each master start')

[agable, 2015/04/27 20:23:08]

Before each? Before any? At least once during the whole process?

[ghost stip (do not use), 2015/04/29 01:30:33]

Done.

+  parser.add_argument('--emergency-file',
+      default='.stop_master_lifecycle',
+      help='perform a gclient sync before each master start')

[agable, 2015/04/27 20:23:08]

Um

[agable, 2015/04/27 20:23:08]

Um

[ghost stip (do not use), 2015/04/29 01:30:33]

Acknowledged.

+  parser.add_argument('--prod', action='store_true',
+      help='actually run commands instead of printing them.')

[agable, 2015/04/27 20:23:08]

Note loudly in the --help that this script runs in dry-run mode *by default*

[ghost stip (do not use), 2015/04/29 01:30:33]

Done.

+  parser.add_argument('--loop', action='store_true',
+      help='repeatedly run the state machine. will not terminate unless killed')
+  parser.add_argument('--loop-sleep-secs', type=int, default=5,
+      help='how many seconds to wait between loop runs. default %(default)s')
+  parser.add_argument('--connection-timeout', type=int, default=30,
+      help='how many seconds to wait for a master http request before timing '
+           'out.')
+  outer_loop.add_argparse_options(parser)
+  args = parser.parse_args()
+
+  if not args.list_all_states and not args.directory:
+    parser.error('A master directory must be specified.')
+  return args
+
+
+def run_state_machine_pass(

[agable, 2015/04/27 20:23:08]

Seems like this function could be tested with a fake checkout (like the other
CL), a mock of the accepting_builds json endpoint, and a bunch of testing
desired_state.json files.

+    matchlist, abs_master_directory, emergency_file, desired_state_file,
+    enable_gclient_sync, prod, connection_timeout):  # pragma: no cover
+  if os.path.exists(os.path.join(abs_master_directory, emergency_file)):
+    print '%s detected in %s, aborting!' % (
+        emergency_file, abs_master_directory)
+    return 1
+
+  desired_state_file = os.path.abspath(desired_state_file)
+  master_directory = os.path.basename(abs_master_directory)
+  evidence = buildbot_state.collect_evidence(
+      abs_master_directory, connection_timeout=connection_timeout)
+  with open(desired_state_file) as f:
+    evidence['desired_buildbot_state'] = json.load(f).get(master_directory)
+  if not evidence['desired_buildbot_state']:
+    raise KeyError('Couldn\'t get evidence for master %s from %s.' %
+        (master_directory, desired_state_file))
+
+  state, action_name, action_items = matchlist.execution_list(evidence)
+  execution_list = list(
+      master.convert_action_items_to_cli(
+      action_items, abs_master_directory,
+      enable_gclient=enable_gclient_sync))
+  print 'current state', state

[agable, 2015/04/27 20:23:08]

Use infra.libs.logs

[ghost stip (do not use), 2015/04/29 01:30:33]

Done.

+  print 'performing action', action_name
+
+  if execution_list:
+    if prod:
+      print 'production run, executing:'
+    else:
+      print 'dry run, not executing:'
+    for cmd in execution_list:
+      print '*  %s (in %s)' % (cmd['cmd'], cmd['cwd'])
+      if prod:
+        with daemon.flock(cmd['lockfile']) as acquired:
+          if acquired:
+            subprocess.check_call([str(x) for x in cmd['cmd']], cwd=cmd['cwd'])
+          else:
+            print '  lock could not be acquired, no action taken.'
+  else:
+    print 'no action to be taken.'
+  return 0
+
+
+def main():  # pragma: no cover
+  args = parse_args()
+  matchlist = buildbot_state.construct_pattern_matcher()
+
+  if args.list_all_states:
+    matchlist.print_all_states()
+    return 0
+
+  abs_master_directory = os.path.abspath(args.directory)
+
+  state_machine = partial(run_state_machine_pass,
+        matchlist, abs_master_directory, args.emergency_file,
+        args.desired_state_file, args.enable_gclient_sync, args.prod,
+        args.connection_timeout)
+
+  if args.loop:
+    loop_opts = outer_loop.process_argparse_options(args)
+    outer_loop.loop(
+        state_machine, lambda: args.loop_sleep_secs, **loop_opts)
+  else:
+    return state_machine()
+
+  return 0
+
+
+if __name__ == '__main__':  # pragma: no cover
+  sys.exit(main())