Allow expectations for different graphs within tests....

log_parser/process_log.py

 #!/usr/bin/python
 # Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Defines various log processors used by buildbot steps.
 
 Current approach is to set an instance of log processor in
 the ProcessLogTestStep implementation and it will call process()
 method upon completion with full data from process stdio.
@@ skipping 19 matching lines @@
 
 # perf_expectations.json holds performance expectations.  It is a
 # JSON-formatted file with this format:
 #
 #   {PERFID: {
 #      RESULTTYPE: {"delta": DELTA, "var": VAR},
 #      RESULTTYPE: {"delta": DELTA, "var": VAR},
 #      ...
 #    },
 #    ...,
-#    "loaded": true
+#    "load": true
 #   }
 #
 # PERFID (string):
 #   Perf mapping identifier of the form "build-perf-name/test-name",
 #   (see factory/chromium_commands.py).
 #
 # RESULTTYPE (string):
 #   A particular trace within a test (ie. t, vm_rss_f_r).
 #
 # DELTA (integer):
@@ skipping 60 matching lines @@
     self._report_link = report_link
     if output_dir is None:
       output_dir = os.getcwd()
     elif output_dir.startswith('~'):
       output_dir = os.path.expanduser(output_dir)
     self._output_dir = output_dir
     self._matches = {}
 
     # Performance regression/speedup alerts.
     self._perf_name = perf_name
-    self._actual_performance = None
-    self._expected_performance = None
-    self._result_types = []
+    self._actual_performance = {}
+    self._expected_performance = {}
     self._perf_regress = []
     self._var_regress = []
     self._perf_improve = []
     self._var_improve = []
 
     # The revision isn't known until the Process() call.
     self._revision = -1
 
+  def LoadPerformanceExpectationsGroup(self, perf_data):
+    """All keys in perf_expectations have 4 components:
+      slave/test/graph/result
+
+    LoadPerformanceExpectationsGroup finds all keys that match the initial
+    portion of the string ("slave/test") and adds the graph and result
+    portions to the expected performance structure.
+    """
+
+    for perf_key in perf_data.keys():
+      m = re.match(r"^" + self._perf_name + "/(\w+)/(\w+)$", perf_key)
+      if not m:
+        continue
+
+      graph_name = m.group(1)
+      result_type = m.group(2)
+
+      self._expected_performance.setdefault(graph_name, {})
+      self._expected_performance[graph_name][result_type] = perf_data[perf_key]
+
   def LoadPerformanceExpectations(self):
     self._expected = {}
     try:
       perf_file = open(PERF_EXPECTATIONS, 'r')
     except IOError, e:
       raise
 
-    perf_list = []
+    perf_data = []
     if perf_file:
       try:
-        perf_list = simplejson.load(perf_file)
+        perf_data = simplejson.load(perf_file)
       except ValueError:
         perf_file.seek(0)
         logging.error("Error parsing %s: '%s'" % (PERF_EXPECTATIONS,
                       perf_file.read().strip()))
       perf_file.close()
 
     # Find this perf/test entry
-    if perf_list and perf_list.has_key(self._perf_name):
-        self._expected_performance = perf_list[self._perf_name]
+    if perf_data and perf_data.has_key('load') and perf_data['load']:
+      self.LoadPerformanceExpectationsGroup(perf_data)
+    else:
+      logging.error("not loading perf expectations, perf_data is disabled")
+
+  def TrackActualPerformance(self, graph_name=None, result_type=None,
+                             value=None, stddev=None):
+    """Set actual performance data when we come across useful values.
+
+    result_type will be of the form "RESULTTYPE" or "RESULTTYPE_ref".
+    A trace with _ref in its name refers to a reference build.
+
+    Common result types for page cyclers: t, vm_rss_f_r, IO_b_b, etc.
+    A test's result types vary between test types.  Currently, a test
+    only needs to output the appropriate text format to embed a new
+    result type.
+    """
+
+    self._actual_performance.setdefault(graph_name, {})
+    m = re.match(r"^(\w+)_ref$", result_type)
+    if m:  # a test on the reference build
+      result_type = m.group(1)
+
+    self._actual_performance[graph_name].setdefault(result_type, {})
+    actual = self._actual_performance[graph_name][result_type]
+
+    if m:  # a test on the reference build
+      actual['ref'] = value
+    else:  # a test on the current build
+      actual['test'] = value
+      actual['var'] = stddev
+
+    # If we have both the current and ref results, compute the delta for this
+    # result type.
+    if 'test' in actual and 'ref' in actual:
+      actual['delta'] = actual['test'] - actual['ref']
 
   def PerformanceChangesAsText(self):
     text = []
 
     if len(self._perf_regress) > 0:
       text.append("PERF_REGRESS: " + ', '.join(self._perf_regress))
 
     if len(self._var_regress) > 0:
       text.append("VAR_REGRESS: " + ', '.join(self._var_regress))
 
     if len(self._perf_improve) > 0:
       text.append("PERF_IMPROVE: " + ', '.join(self._perf_improve))
 
     if len(self._var_improve) > 0:
       text.append("VAR_IMPROVE: " + ', '.join(self._var_improve))
 
     return text
 
+  def ComparePerformance(self, graph_name, result_type):
+    # Skip graphs and result types we don't expect values for.
+    if (not graph_name in self._expected_performance or
+        not result_type in self._expected_performance[graph_name]):
+      return
+
+    expected = self._expected_performance[graph_name][result_type]
+    actual = self._actual_performance[graph_name][result_type]
+    graph_result = graph_name + '/' + result_type
+
+    # Skip result types that didn't calculate a delta.
+    if not 'delta' in actual:
+      return
+
+    # Exception when missing delta/var in expectations.
+    if not 'delta' in expected or not 'var' in expected:

[M-A Ruel, 2009/09/03 23:43:41]

nit: Why do you care at all if you want to throw an exception anyway?

+      missing = []
+      if not 'delta' in expected:
+        missing.append('delta')
+      if not 'var' in expected:
+        missing.append('var')
+      msg = "%s/%s expectations missing %s" % (self._perf_name, graph_result,
+                                               ', '.join(missing))
+      raise Exception(msg)
+
+    # Set the high and low performance and variance tolerances.  The actual
+    # delta and variance needs to be within 50% above and below this range to
+    # keep the performance test green.  If the results fall above or below
+    # this range, the test will go red (signaling a regression) or orange
+    # (signaling a speedup).
+    high_perf = (expected['delta'] + 1.5*expected['var'])
+    low_perf = (expected['delta'] - 1.5*expected['var'])
+    high_var = 1.5*expected['var']
+    low_var = 0.5*expected['var']
+
+    if actual['delta'] > high_perf:
+      self._perf_regress.append(graph_result)
+    elif actual['delta'] < low_perf:
+      self._perf_improve.append(graph_result)
+
+    if actual['var'] > high_var:
+      self._var_regress.append(graph_result)
+    elif actual['var'] < low_var:
+      self._var_improve.append(graph_result)
+
   def PerformanceChanges(self):
     # Load performance expectations for this test.
     self.LoadPerformanceExpectations()
 
-    # Return if no performance expectations or results were found.
-    if not self._expected_performance or not self._actual_performance:
-      return []
-
     # Compare actual and expected results.
-    for type in self._result_types:
-      if not (self._expected_performance.has_key(type) and
-              self._actual_performance.has_key(type)):
-        # Skip result types we don't know about.
-        continue
-
-      expect = self._expected_performance[type]
-      actual = self._actual_performance[type]
-
-      # Set the high and low performance and variance tolerances.  The actual
-      # delta and variance needs to be within 50% above and below this range to
-      # keep the performance test green.  If the results fall above or below
-      # this range, the test will go red (signaling a regression) or orange
-      # (signaling a speedup).
-      high_perf = (expect['delta'] + 1.5*expect['var'])
-      low_perf = (expect['delta'] - 1.5*expect['var'])
-      high_var = 1.5*expect['var']
-      low_var = 0.5*expect['var']
-
-      if actual['delta'] > high_perf:
-        self._perf_regress.append(type)
-      elif actual['delta'] < low_perf:
-        self._perf_improve.append(type)
-
-      if actual['var'] > high_var:
-        self._var_regress.append(type)
-      elif actual['var'] < low_var:
-        self._var_improve.append(type)
+    for graph_name in self._actual_performance:
+      for result_type in self._actual_performance[graph_name]:
+        self.ComparePerformance(graph_name, result_type)
 
     return self.PerformanceChangesAsText()
 
   def evaluateCommand(self, cmd):
     if len(self._perf_regress) > 0 or len(self._var_regress) > 0:
       return builder.FAILURE
 
     if len(self._perf_improve) > 0 or len(self._var_improve) > 0:
       return builder.WARNINGS
 
@@ skipping 296 matching lines @@
       else:
         try:
           trace.value = float(trace.value)
         except ValueError:
           logging.warning("Bad test output: '%s'" % trace.value.strip())
           return
 
       graph.traces[trace_name] = trace
       self._graphs[graph_name] = graph
 
-      # Set actual performance data when we come across useful values.
-      #
-      # trace_name will be of the form "RESULTTYPE" or "RESULTTYPE_ref".
-      # A trace with _ref in its name refers to a reference build.
-      #
-      # Common result types for page cyclers: t, vm_rss_f_r, IO_b_b, etc.
-      # A test's result types vary between test types.  Currently, a test
-      # only needs to output the appropriate text format to embed a new
-      # result type.
-
-      m = re.match(r"^(\w+)_ref$", trace_name)
-      if m:
-        is_ref_build = True
-        result_type = m.group(1)
-      else:
-        is_ref_build = False
-        result_type = trace_name
-
-      if not self._actual_performance:
-        self._actual_performance = {}
-
-      if not self._actual_performance.has_key(result_type):
-        self._actual_performance[result_type] = {}
-
-      actual = self._actual_performance[result_type]
-      if is_ref_build:
-        actual['ref'] = trace.value
-      else:
-        actual['test'] = trace.value
-        actual['var'] = trace.stddev
-
-      # If we have both the test and ref results, compute the delta for this
-      # result type.
-      if actual.has_key('test') and actual.has_key('ref'):
-        self._result_types.append(result_type)
-        actual['delta'] = actual['test'] - actual['ref']
+      # Store values in actual performance.
+      self.TrackActualPerformance(graph_name=graph_name, result_type=trace_name,
+                                  value=trace.value, stddev=trace.stddev)
 
   def _CalculateStatistics(self, value_list, trace_name):
     """Returns a tuple (mean, standard deviation) from a list of values.
 
     This method may be overridden by subclasses wanting a different standard
     deviation calcuation (or some other sort of error value entirely).
 
     Args:
       value_list: the list of values to use in the calculation
       trace_name: the trace that produced the data (not used in the base
@@ skipping 182 matching lines @@
            FormatFloat(mean),
            FormatFloat(stddev),
            self._JoinWithSpacesAndNewLine(times)))
 
     filename = os.path.join(self._output_dir,
                             '%s_%s.dat' % (self._revision, trace_name))
     file = open(filename, 'w')
     file.write(''.join(file_data))
     file.close()
     os.chmod(filename, READABLE_FILE_PERMISSIONS)