Initial implementation of annotator library.

scripts/common/annotator.py

+#!/usr/bin/env python
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Contains generating and parsing systems of the Chromium Buildbot Annotator.
+
+When executed as a script, this reads step name / command pairs from a file and
+executes those lines while annotating the output. The input is either
+
+step_name command arg1 arg2...
+step_name2 command2 arg1 arg2...
+
+or if --json is specified:
+{"name": "step_name", "cmd": ["command", "arg1", "arg2"]}
+{"name": "step_name2", "cmd": ["command2", "arg1"]}

[iannucci, 2013/02/28 09:56:03]

Why not just do json? It's easier to parse and then we don't need to support 2
formats.

+
+"""
+
+import json
+import optparse
+import re
+import shlex
+import sys
+import traceback
+
+from common import chromium_utils
+
+
+class AdvancedAnnotationStream(object):
+  """Holds individual annotation generating functions.
+
+  Most callers should use StructuredAnnotationStream to simplify coding and
+  avoid errors. For the rare cases where StructuredAnnotationStream is
+  insufficient (parallel step execution), the indidividual functions are exposed
+  here.
+
+  """
+
+  def __init__(self, stream=sys.stdout):
+    self.stream = stream
+
+  def emit(self, line):
+    print >> self.stream, line
+
+  def seed_step(self, step):
+    self.emit('@@@SEED_STEP %s@@@' % step)

[iannucci, 2013/02/28 09:56:03]

These methods can all be generated from the same list of tuples as below (see
next comment)

+
+  def step_cursor(self, step):
+    self.emit('@@@STEP_CURSOR %s@@@' % step)
+
+  def build_step(self, line, step):
+    self.emit('@@@BUILD_STEP %s@@@' % step)  # deprecated, use SEED_STEP
+
+  def step_started(self):
+    self.emit('@@@STEP_STARTED@@@')
+
+  def step_closed(self):
+    self.emit('@@@STEP_CLOSED@@@')
+
+  def step_warnings(self):
+    self.emit('@@@STEP_WARNINGS@@@')
+
+  def step_failure(self):
+    self.emit('@@@STEP_FAILURE@@@')
+
+  def step_exception(self):
+    self.emit('@@@STEP_EXCEPTION@@@')
+
+  def halt_on_failure(self):
+    self.emit('@@@HALT_ON_FAILURE@@@')
+
+  def honor_zero_return_code(self):
+    self.emit('@@@HONOR_ZERO_RETURN_CODE@@@')
+
+  def step_clear(self):
+    self.emit('@@@STEP_CLEAR@@@')
+
+  def step_summary_clear(self):
+    self.emit('@@@STEP_SUMMARY_CLEAR@@@')
+
+  def step_text(self, text):
+    self.emit('@@@STEP_TEXT@%s@@@' % text)
+
+  def step_summary_text(self, text):
+    self.emit('@@@STEP_SUMMARY_TEXT@%s@@@' % text)
+
+  def write_log_line(self, logname, line):
+    self.emit('@@@STEP_LOG_LINE@%s@%s@@@' % (logname, line.rstrip('\n')))
+
+  def log_end(self, logname):
+    self.emit('@@@STEP_LOG_END@%s@@@' % logname)
+
+  def log_end_perf(self, logname, perf):
+    self.emit('@@@STEP_LOG_END_PERF@%s@%s@@@' % (logname, perf))
+
+  def write_log_lines(self, logname, lines, perf=None):
+    for line in lines:
+      self.write_log_line(logname, line)
+    if perf:
+      self.log_end_perf(logname, perf)
+    else:
+      self.log_end(logname)
+
+
+class StructuredAnnotationStream(AdvancedAnnotationStream):
+  """Provides an interface to handle an annotated build.
+
+  StructuredAnnotationStream handles most of the step setup and closure calls
+  for you. All you have to do is execute your code within the steps and set any
+  failures or warnings that come up. You may optionally provide a list of steps
+  to seed before execution.
+
+  Usage:
+
+  stream = StructuredAnnotationStream()
+  with stream.step('compile') as s:
+    # do something
+    if error:
+      s.step_failure()
+  with stream.step('test') as s:
+    # do something
+    if warnings:
+      s.step_warnings()
+  """
+
+  def __init__(self, seed_steps=None, stream=sys.stdout):
+    super(StructuredAnnotationStream, self).__init__(stream=stream)
+    seed_steps = seed_steps or []
+    self.seed_steps = seed_steps
+
+    for step in seed_steps:
+      self.seed_step(step)
+
+    self.current_step = ''
+
+  def __enter__(self):
+    return self
+
+  def __exit__(self, exc_type, exc_value, tb):
+    if exc_type:
+      trace = traceback.format_exception(exc_type, exc_value, tb)
+      unflattened = [l.split('\n') for l in trace]
+      flattened = [item for sublist in unflattened for item in sublist]
+      self.write_log_lines('exception', filter(None, flattened))
+      self.step_exception()
+    self.step_closed()
+    self.current_step = ''
+    return not exc_type
+
+  def step(self, name):

[iannucci, 2013/02/28 09:56:03]

I'm not sure I like using self as both the container AND the context object.
Could there be an intermediate class which is returned from this function which
models the context object (i.e. perhaps a StructuredAnnotationStep).

[Mike Stip (use stip instead), 2013/02/28 20:35:22]

Done.

+    """Provide a context with which to execute a step."""
+    if self.current_step:
+      raise Exception('Can\'t start step %s while in step %s.' % (
+          name, self.current_step))
+    if name in self.seed_steps:
+      # Seek ahead linearly, skipping steps that weren't emitted in order.
+      # chromium_step.AnnotatedCommands uses the last in case of duplicated
+      # step names, so we do the same here.
+      idx = len(self.seed_steps) - self.seed_steps[::-1].index(name)
+      self.seed_steps = self.seed_steps[idx:]
+    else:
+      self.seed_step(name)
+
+    self.step_cursor(name)
+    self.step_started()
+    self.current_step = name
+    return self
+
+
+class Match:
+  """Holds annotator line parsing functions."""
+
+  def __init__(self):
+    raise Exception('Don\'t instantiate the Match class!')
+
+  @staticmethod
+  def _parse_line(regex, line):
+    m = re.match(regex, line)
+    if m:
+      return list(m.groups())
+    else:
+      return []
+
+  @staticmethod
+  def log_line(line):

[iannucci, 2013/02/28 09:56:03]

It seems to me like all of these methods should be generated from a list of
tuples like:

method_descriptions = [
  ('log_line', '^@@@STEP_LOG_LINE@(.*)@(.*)@@@'),
  ...
]

for name, match in method_descriptions:
  @staticmethod
  def method(line):
    return Match._parse_line(match, line)
  method.__name__ = name
  setattr(Match, name, method)

[Mike Stip (use stip instead), 2013/02/28 20:35:22]

I like this idea, but let's save it for a future CL.

[agable, 2013/02/28 21:32:08]

While a fun trick, that doesn't gain you any performance and makes the code
actually less readable, I think.

On 2013/02/28 09:56:03, iannucci wrote:

+    return Match._parse_line('^@@@STEP_LOG_LINE@(.*)@(.*)@@@', line)
+
+  @staticmethod
+  def log_end(line):
+    return Match._parse_line('^@@@STEP_LOG_END@(.*)@@@', line)
+
+  @staticmethod
+  def log_end_perf(line):
+    return Match._parse_line('^@@@STEP_LOG_END_PERF@(.*)@(.*)@@@', line)
+
+  @staticmethod
+  def step_link(line):
+    m = Match._parse_line('^@@@STEP_LINK@(.*)@(.*)@@@', line)
+    if not m:
+      return Match._parse_line('^@@@link@(.*)@(.*)@@@', line)  # deprecated
+    else:
+      return m
+
+  @staticmethod
+  def step_started(line):
+    return line.startswith('@@@STEP_STARTED@@@')
+
+  @staticmethod
+  def step_closed(line):
+    return line.startswith('@@@STEP_CLOSED@@@')
+
+  @staticmethod
+  def step_warnings(line):
+    return (line.startswith('@@@STEP_WARNINGS@@@') or
+            line.startswith('@@@BUILD_WARNINGS@@@'))  # deprecated
+
+  @staticmethod
+  def step_failure(line):
+    return (line.startswith('@@@STEP_FAILURE@@@') or
+            line.startswith('@@@BUILD_FAILED@@@'))  # deprecated
+
+  @staticmethod
+  def step_exception(line):
+    return (line.startswith('@@@STEP_EXCEPTION@@@') or
+            line.startswith('@@@BUILD_EXCEPTION@@@'))  # deprecated
+
+  @staticmethod
+  def halt_on_failure(line):
+    return line.startswith('@@@HALT_ON_FAILURE@@@')
+
+  @staticmethod
+  def honor_zero_return_code(line):
+    return line.startswith('@@@HONOR_ZERO_RETURN_CODE@@@')
+
+  @staticmethod
+  def step_clear(line):
+    return line.startswith('@@@STEP_CLEAR@@@')
+
+  @staticmethod
+  def step_summary_clear(line):
+    return line.startswith('@@@STEP_SUMMARY_CLEAR@@@')
+
+  @staticmethod
+  def step_text(line):
+    return Match._parse_line('^@@@STEP_TEXT@(.*)@@@', line)
+
+  @staticmethod
+  def step_summary_text(line):
+    return Match._parse_line('^@@@STEP_SUMMARY_TEXT@(.*)@@@', line)
+
+  @staticmethod
+  def seed_step(line):
+    return Match._parse_line('^@@@SEED_STEP (.*)@@@', line)
+
+  @staticmethod
+  def step_cursor(line):
+    return Match._parse_line('^@@@STEP_CURSOR (.*)@@@', line)
+
+  @staticmethod
+  def build_step(line):
+    return Match._parse_line('^@@@BUILD_STEP (.*)@@@', line)
+
+
+def main():
+  usage = '%s <command list file>' % sys.argv[0]
+  parser = optparse.OptionParser(usage=usage)
+  parser.add_option('-j', '--json', action='store_true',
+                    help='Specify that each line is a json blob encoding '
+                         'the step.')
+  options, args = parser.parse_args()
+  if not args:
+    parser.error('Must specify an input filename!')
+
+  steps = []
+  with open(args[0], 'rb') as f:
+    for line in f:
+      if options.json:
+        steps.append(json.loads(line))
+        if ('cmd' not in steps[-1] or
+            'name' not in steps[-1]):
+          print 'line \'%s\' is invalid' % line.rstrip()
+          return 1
+      else:
+        step = {}
+        chunks = line.split(' ')
+        step['name'] = chunks[0]
+        if not step['name']:
+          print 'line \'%s\' is invalid' % line.rstrip()
+          return 1
+        command = line[len(step['name'])+1:]
+        if not command:
+          print 'line \'%s\' is invalid' % line.rstrip()
+          return 1
+        step['cmd'] = shlex.split(command)
+        steps.append(step)
+
+  stepnames = [s['name'] for s in steps]
+  stream = StructuredAnnotationStream(seed_steps = stepnames)
+  for step in steps:
+    with stream.step(step['name']) as s:
+      ret = chromium_utils.RunCommand(step['cmd'])
+      if ret != 0:
+        s.step_failure()

[iannucci, 2013/02/28 09:56:03]

Should ret be interpreted to mean 88 == warning (in slave_utils)?

+        return 1

[iannucci, 2013/02/28 09:56:03]

Do we need a way to express commands which should always run? (like cleanup
scripts or stack trace collections?)

[Mike Stip (use stip instead), 2013/02/28 20:35:22]

Done.

+
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main())