mastermon: collect step results count

infra/services/mastermon/pollers.py

 # Copyright (c) 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import collections
 import copy
 import json
 import logging
 import os
 import time
@@ skipping 143 matching lines @@
   """Poll a file instead of an endpoint.
 
   The base_url in __init__ must be the file path.  The file is assumed
   to contain JSON objects, one per line.
 
   The poller will first rotate the file (rename), read it, and delete
   the file.  The writer of the file is expected to create a new file if
   it was rotated or deleted.
   """
   endpoint = 'FILE'
-  field_keys = ('builder', 'slave', 'result', 'project_id', 'subproject_tag')
+  build_field_keys = ('builder', 'slave', 'result',
+                      'project_id', 'subproject_tag')
+  step_field_keys = ('builder', 'slave', 'step_result',
+                     'project_id', 'subproject_tag')
+
+  ### These metrics are sent when a build finishes.
   result_count = ts_mon.CounterMetric('buildbot/master/builders/results/count',
       description='Number of items consumed from ts_mon.log by mastermon')
   # A custom bucketer with 12% resolution in the range of 1..10**5,
   # better suited for build cycle times.
   bucketer = ts_mon.GeometricBucketer(
       growth_factor=10**0.05, num_finite_buckets=100)
   cycle_times = ts_mon.CumulativeDistributionMetric(
       'buildbot/master/builders/builds/durations', bucketer=bucketer,
       description='Durations (in seconds) that slaves spent actively doing '
                   'work towards builds for each builder')
   pending_times = ts_mon.CumulativeDistributionMetric(
       'buildbot/master/builders/builds/pending_durations', bucketer=bucketer,
       description='Durations (in seconds) that the master spent waiting for '
                   'slaves to become available for each builder')
   total_times = ts_mon.CumulativeDistributionMetric(
       'buildbot/master/builders/builds/total_durations', bucketer=bucketer,
       description='Total duration (in seconds) that builds took to complete '
                   'for each builder')
 
   pre_test_times = ts_mon.CumulativeDistributionMetric(
       'buildbot/master/builders/builds/pre_test_durations', bucketer=bucketer,
       description='Durations (in seconds) that builds spent before their '
                   '"before_tests" step')
 
+  ### This metric is sent when a step finishes.
+  step_results_count = ts_mon.CounterMetric(
+    'buildbot/master/builders/steps/results/count',
+    description='Count of step results, per builder')
+
   def poll(self):
     LOGGER.info('Collecting results from %s', self._url)
 
     if not os.path.isfile(self._url):
       LOGGER.info('No file found, assuming no data: %s', self._url)
       return True
 
     try:
       rotated_name = rotated_filename(self._url)
       # Remove the previous rotated file. We keep it on disk after
       # processing for debugging.
       safe_remove(rotated_name)
       os.rename(self._url, rotated_name)
       with open(rotated_name, 'r') as f:
         for line in f:  # pragma: no branch
           self.handle_response(json.loads(line))
     except (ValueError, OSError, IOError) as e:
       LOGGER.error('Could not collect or send results from %s: %s',
                    self._url, e)
 
     # Never return False - we don't know if master is down.
     return True
 
   def handle_response(self, data):
-    fields = self.fields({k: data.get(k, 'unknown') for k in self.field_keys})
-    self.result_count.increment(fields)
-    if 'duration_s' in data:
-      self.cycle_times.add(data['duration_s'], fields)
-    if 'pending_s' in data:
-      self.pending_times.add(data['pending_s'], fields)
-    if 'total_s' in data:
-      self.total_times.add(data['total_s'], fields)
-    if 'pre_test_time_s' in data:
-      self.pre_test_times.add(data['pre_test_time_s'], fields)
+    # We handle two cases here: whether the data was generated when a build
+    # finished or when a step finished. We use the content of the json dict to
+    # tell the difference.
+
+    if 'step_result' in data:  # generated when a step finishes
+      fields = self.fields({k: data.get(k, 'unknown')
+                            for k in self.step_field_keys})
+      self.step_results_count.increment(fields=fields)
+
+    else:  # otherwise it's generated after a build finishes
+      fields = self.fields({k: data.get(k, 'unknown')
+                            for k in self.build_field_keys})
+      self.result_count.increment(fields)
+      if 'duration_s' in data:
+        self.cycle_times.add(data['duration_s'], fields)
+      if 'pending_s' in data:
+        self.pending_times.add(data['pending_s'], fields)
+      if 'total_s' in data:
+        self.total_times.add(data['total_s'], fields)
+      if 'pre_test_time_s' in data:
+        self.pre_test_times.add(data['pre_test_time_s'], fields)