Add libs to get information about or manipulate a buildbot master.

infra/libs/buildbot/master.py

+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+

[agable, 2015/04/27 20:01:45]

Since the only file in this directory is called master.py, not buildbot.py, it
suggests that this library will end up containing other pieces. As such, I'd
suggest populating __init__.py with the pieces of the library that you'd like to
expose publicly as the API.

[ghost stip (do not use), 2015/04/27 23:42:18]

Done.

+"""Get information about a buildbot master and manipulate it."""
+
+
+import errno
+import os
+import json
+from dateutil.parser import parse

[agable, 2015/04/27 20:01:45]

nit: "from" import statements come in a separate block after all the regular
imports

[ghost stip (do not use), 2015/04/27 23:42:18]

Done.

+import re
+import requests
+import simplejson
+import subprocess
+import sys
+
+from infra.libs.time_functions import timestamp
+
+######## Getting information about the master.
+
+
+# Derived from http://stackoverflow.com/a/6940314.

[agable, 2015/04/27 20:01:45]

Include in the comment here or in the docstring that sending signal 0 is
equivalent to an "are you alive" ping so that people don't have to follow the
link just for that.

[ghost stip (do not use), 2015/04/27 23:42:17]

Done.

+def _pid_is_alive(pid):  # pragma: no cover
+  """Determine if the given pid is still running."""
+  if sys.platform.startswith('win'):
+    raise NotImplementedError
+  if pid <= 0:
+    return False
+  try:
+    os.kill(pid, 0)
+  except OSError as err:
+    if err.errno == errno.ESRCH:
+      return False
+    elif err.errno == errno.EPERM:
+      return True
+    else:
+      raise
+  else:
+    return True
+
+
+def buildbot_is_running(directory):
+  """Determine if the twisted.pid in a directory is running."""
+  pidfile = os.path.join(directory, 'twistd.pid')
+  if not os.path.exists(pidfile):
+    return False
+  with open(pidfile) as f:
+    pid = int(f.read().strip())
+  return _pid_is_alive(pid)
+
+
+def _get_last_action(directory, action):
+  """Get the last time the given action was run in the actions.log."""
+  actions_log_file = os.path.join(directory, 'actions.log')
+  if not os.path.exists(actions_log_file):
+    return None
+  last_line = None
+  with open(actions_log_file) as f:
+    for line in f:

[agable, 2015/04/27 20:01:45]

Read the file backwards and search for the first line that looks like an action.
Less state, and results in processing fewer lines (though they'll still all get
read into memory).

[ghost stip (do not use), 2015/04/27 23:42:18]

it's way cleaner now. nice

+      if action in line:
+        last_line = line
+  if not last_line:
+    return None
+
+  unparsed_date = last_line.split(action)[0][len('**'):]
+  return timestamp.utctimestamp(parse(unparsed_date))
+
+
+def get_last_boot(directory):
+  """Determine the last time the master was started."""
+  return max(
+      _get_last_action(directory, 'make restart'),
+      _get_last_action(directory, 'make start'))
+
+
+def get_last_no_new_builds(directory):
+  """Determine the last time a *current* make no-new-builds was called.
+  
+  If a 'make start' or 'make restart' was issued after the no-new-builds, it is
+  not valid (since the make start or make restart nullified the no-new-builds).
+  """
+  last_boot = get_last_boot(directory)
+  last_no_new_builds = _get_last_action(directory, 'make no-new-builds')
+  if not last_no_new_builds:
+    return None
+  if last_boot > last_no_new_builds:
+    return None
+  return last_no_new_builds
+
+
+def _get_mastermap_data(directory):
+  """Get mastermap JSON from a master directory."""
+
+  build_dir = os.path.join(directory, os.pardir, os.pardir, os.pardir, 'build')
+  runit = os.path.join(build_dir, 'scripts', 'tools', 'runit.py')
+  build_internal_dir = os.path.join(build_dir, os.pardir, 'build_internal')
+
+  script_path = os.path.join(build_dir, 'scripts', 'tools', 'mastermap.py')
+  if os.path.exists(build_internal_dir):

[agable, 2015/04/27 20:01:45]

I'm a little leery about infra.git knowing about build_internal, since it isn't
in the deps file. But if we do actually roll out the Infra Repo Deployment Plan,
then this will be kosher. And it's better than build knowing about
build_internal anyway.

[ghost stip (do not use), 2015/04/27 23:42:18]

yeah. I actually with mastermap.py would just automagically use internal data if
it saw it, but this was the best I could come up with

+    script_path = os.path.join(
+         build_internal_dir, 'scripts', 'tools', 'mastermap_internal.py')
+
+  script_data = json.loads(subprocess.check_output(
+    [runit, script_path, '-f', 'json']))
+
+  short_dirname = os.path.basename(directory)
+  matches = [x for x in script_data if x.get('dirname') == short_dirname]
+  assert len(matches) < 2

[agable, 2015/04/27 20:01:45]

Add an error message, plain asserts in libraries make me sad.

[ghost stip (do not use), 2015/04/27 23:42:18]

Done.

+  if matches:
+    return matches[0]
+  return None
+
+
+def _get_master_web_port(directory):
+  """Determine the web port of the master running in the given directory."""
+  mastermap_data = _get_mastermap_data(directory)
+  if mastermap_data:
+    return mastermap_data['port']
+  return None
+
+
+def get_accepting_builds(directory, timeout=30):
+  """Determine whether the master is accepting new builds or not."""

[agable, 2015/04/27 20:01:45]

Note in big huge words that this only works for masters running locally. Someone
might expect to be able to call this on their local machine and find out in
master.chromium.mac is up.

[ghost stip (do not use), 2015/04/27 23:42:18]

Done.

+  port = _get_master_web_port(directory)
+  if port is not None:
+    try:
+      res = requests.get(
+          'http://localhost:%d/json/accepting_builds' % port,
+          timeout=timeout)
+      if res.status_code == 200:
+        try:
+          return res.json().get('accepting_builds')
+        except simplejson.scanner.JSONDecodeError:
+          pass
+    except requests.exceptions.Timeout:
+      pass
+  return None
+
+
+######## Performing actions on the master.
+
+
+GclientSync, MakeStop, MakeWait, MakeStart, MakeNoNewBuilds = range(5)
+
+
+def convert_action_items_to_cli(
+    action_items, directory, enable_gclient=False):
+
+  def cmd_dict(cmd, lockfile_prefix):
+    # Using the same lockfile prefix for two actions will make them mutually
+    # exclusive. So setting 'make' for all the make commands tries to prevent
+    # two make actions happening at once in the same master directory.
+    lockfile = '%s_%s' % (
+        lockfile_prefix,
+        re.sub('[^\w]', '_', directory.lower()))
+    return {
+        'cwd': directory,
+        'cmd': cmd,
+        'lockfile': lockfile,
+    }
+
+  for action_item in action_items:
+    if action_item == GclientSync:
+      if enable_gclient:
+        yield cmd_dict(
+            ['gclient', 'sync', '--reset', '--force', '--auto_rebase'],
+            'gclient')
+    elif action_item == MakeStop:
+      yield cmd_dict(['make', 'stop'], 'make')
+    elif action_item == MakeWait:
+      yield cmd_dict(['make', 'wait'], 'make')
+    elif action_item == MakeStart:
+      yield cmd_dict(['make', 'start'], 'make')
+    elif action_item == MakeNoNewBuilds:
+      yield cmd_dict(['make', 'no-new-builds'], 'make')
+    else:
+      raise ValueError('Invalid action item %s' % action_item)