[test] Refactoring - Make perf suite definitions stateless regarding measurements.

tools/run_perf.py

Index: tools/run_perf.py
diff --git a/tools/run_perf.py b/tools/run_perf.py
index c04e4e77c3c2e70ecf26d46bb7658c396db25a8d..b994fdde925f74874544fbde79c2090da1cffce5 100755
--- a/tools/run_perf.py
+++ b/tools/run_perf.py
@@ -169,6 +169,146 @@ class Results(object):
     return str(self.ToDict())
 
 
+class Measurement(object):
+  """Represents a series of results of one trace.

[Michael Achenbach, 2015/07/08 10:03:19]

Basically copy from line 259.

+
+  The results are from repetitive runs of the same executable. They are
+  gathered by repeated calls to ConsumeOutput.
+  """
+  def __init__(self, graphs, units, results_regexp, stddev_regexp):
+    self.name = graphs[-1]
+    self.graphs = graphs
+    self.units = units
+    self.results_regexp = results_regexp
+    self.stddev_regexp = stddev_regexp
+    self.results = []
+    self.errors = []
+    self.stddev = ""
+
+  def ConsumeOutput(self, stdout):
+    try:
+      result = re.search(self.results_regexp, stdout, re.M).group(1)
+      self.results.append(str(float(result)))
+    except ValueError:
+      self.errors.append("Regexp \"%s\" returned a non-numeric for test %s."
+                         % (self.results_regexp, self.name))
+    except:
+      self.errors.append("Regexp \"%s\" didn't match for test %s."
+                         % (self.results_regexp, self.name))
+
+    try:
+      if self.stddev_regexp and self.stddev:
+        self.errors.append("Test %s should only run once since a stddev "
+                           "is provided by the test." % self.name)
+      if self.stddev_regexp:
+        self.stddev = re.search(self.stddev_regexp, stdout, re.M).group(1)
+    except:
+      self.errors.append("Regexp \"%s\" didn't match for test %s."
+                         % (self.stddev_regexp, self.name))
+
+  def GetResults(self):
+    return Results([{
+      "graphs": self.graphs,
+      "units": self.units,
+      "results": self.results,
+      "stddev": self.stddev,
+    }], self.errors)
+
+
+def AccumulateResults(graph_names, traces, iter_output, calc_total):
+  """Iterates over the output of multiple benchmark reruns and accumulates

[Michael Achenbach, 2015/07/08 10:03:19]

Basically copy from line 320.

+  results for a configured list of traces.
+
+  Args:
+    graph_names: List of names that configure the base path of the traces. E.g.
+                 ['v8', 'Octane'].
+    traces: List of "Trace" instances. Each trace defines how to perform a
+            measurement.
+    iter_output: Iterator over the standard output of each test run.
+    calc_total: Boolean flag to speficy the calculation of a summary trace.
+  Returns: A "Results" object.
+  """
+  measurements = [trace.CreateMeasurement() for trace in traces]
+  for stdout in iter_output():
+    for measurement in measurements:
+      measurement.ConsumeOutput(stdout)
+
+  res = reduce(lambda r, t: r + t.GetResults(), measurements, Results())
+
+  if not res.traces or not calc_total:
+    return res
+
+  # Assume all traces have the same structure.
+  if len(set(map(lambda t: len(t["results"]), res.traces))) != 1:
+    res.errors.append("Not all traces have the same number of results.")
+    return res
+
+  # Calculate the geometric means for all traces. Above we made sure that
+  # there is at least one trace and that the number of results is the same
+  # for each trace.
+  n_results = len(res.traces[0]["results"])
+  total_results = [GeometricMean(t["results"][i] for t in res.traces)
+                   for i in range(0, n_results)]
+  res.traces.append({
+    "graphs": graph_names + ["Total"],
+    "units": res.traces[0]["units"],
+    "results": total_results,
+    "stddev": "",
+  })
+  return res
+
+
+def AccumulateGenericResults(graph_names, suite_units, iter_output):
+  """Iterates over the output of multiple benchmark reruns and accumulates

[Michael Achenbach, 2015/07/08 10:03:19]

Basically copy from line 365.

+  generic results.
+
+  Args:
+    graph_names: List of names that configure the base path of the traces. E.g.
+                 ['v8', 'Octane'].
+    suite_units: Measurement default units as defined by the benchmark suite.
+    iter_output: Iterator over the standard output of each test run.
+  Returns: A "Results" object.
+  """
+  traces = OrderedDict()
+  for stdout in iter_output():
+    for line in stdout.strip().splitlines():
+      match = GENERIC_RESULTS_RE.match(line)
+      if match:
+        stddev = ""
+        graph = match.group(1)
+        trace = match.group(2)
+        body = match.group(3)
+        units = match.group(4)
+        match_stddev = RESULT_STDDEV_RE.match(body)
+        match_list = RESULT_LIST_RE.match(body)
+        errors = []
+        if match_stddev:
+          result, stddev = map(str.strip, match_stddev.group(1).split(","))
+          results = [result]
+        elif match_list:
+          results = map(str.strip, match_list.group(1).split(","))
+        else:
+          results = [body.strip()]
+
+        try:
+          results = map(lambda r: str(float(r)), results)
+        except ValueError:
+          results = []
+          errors = ["Found non-numeric in %s" %
+                    "/".join(graph_names + [graph, trace])]
+
+        trace_result = traces.setdefault(trace, Results([{
+          "graphs": graph_names + [graph, trace],
+          "units": (units or suite_units).strip(),
+          "results": [],
+          "stddev": "",
+        }], errors))
+        trace_result.traces[0]["results"].extend(results)
+        trace_result.traces[0]["stddev"] = stddev
+
+  return reduce(lambda r, t: r + t, traces.itervalues(), Results())
+
+
 class Node(object):
   """Represents a node in the suite tree structure."""
   def __init__(self, *args):
@@ -249,45 +389,18 @@ class Graph(Node):
 
 
 class Trace(Graph):
-  """Represents a leaf in the suite tree structure.
-
-  Handles collection of measurements.
-  """
+  """Represents a leaf in the suite tree structure."""
   def __init__(self, suite, parent, arch):
     super(Trace, self).__init__(suite, parent, arch)
     assert self.results_regexp
-    self.results = []
-    self.errors = []
-    self.stddev = ""
-
-  def ConsumeOutput(self, stdout):
-    try:
-      result = re.search(self.results_regexp, stdout, re.M).group(1)
-      self.results.append(str(float(result)))
-    except ValueError:
-      self.errors.append("Regexp \"%s\" returned a non-numeric for test %s."
-                         % (self.results_regexp, self.graphs[-1]))
-    except:
-      self.errors.append("Regexp \"%s\" didn't match for test %s."
-                         % (self.results_regexp, self.graphs[-1]))
 
-    try:
-      if self.stddev_regexp and self.stddev:
-        self.errors.append("Test %s should only run once since a stddev "
-                           "is provided by the test." % self.graphs[-1])
-      if self.stddev_regexp:
-        self.stddev = re.search(self.stddev_regexp, stdout, re.M).group(1)
-    except:
-      self.errors.append("Regexp \"%s\" didn't match for test %s."
-                         % (self.stddev_regexp, self.graphs[-1]))
-
-  def GetResults(self):
-    return Results([{
-      "graphs": self.graphs,
-      "units": self.units,
-      "results": self.results,
-      "stddev": self.stddev,
-    }], self.errors)
+  def CreateMeasurement(self):
+    return Measurement(
+        self.graphs,
+        self.units,
+        self.results_regexp,
+        self.stddev_regexp,
+    )
 
 
 class Runnable(Graph):
@@ -317,32 +430,8 @@ class Runnable(Graph):
 
   def Run(self, runner):
     """Iterates over several runs and handles the output for all traces."""
-    for stdout in runner():
-      for trace in self._children:
-        trace.ConsumeOutput(stdout)
-    res = reduce(lambda r, t: r + t.GetResults(), self._children, Results())
-
-    if not res.traces or not self.total:
-      return res
-
-    # Assume all traces have the same structure.
-    if len(set(map(lambda t: len(t["results"]), res.traces))) != 1:
-      res.errors.append("Not all traces have the same number of results.")
-      return res
-
-    # Calculate the geometric means for all traces. Above we made sure that
-    # there is at least one trace and that the number of results is the same
-    # for each trace.
-    n_results = len(res.traces[0]["results"])
-    total_results = [GeometricMean(t["results"][i] for t in res.traces)
-                     for i in range(0, n_results)]
-    res.traces.append({
-      "graphs": self.graphs + ["Total"],
-      "units": res.traces[0]["units"],
-      "results": total_results,
-      "stddev": "",
-    })
-    return res
+    return AccumulateResults(self.graphs, self._children, runner, self.total)
+
 
 class RunnableTrace(Trace, Runnable):
   """Represents a runnable suite definition that is a leaf."""
@@ -351,9 +440,10 @@ class RunnableTrace(Trace, Runnable):
 
   def Run(self, runner):
     """Iterates over several runs and handles the output."""
+    measurement = self.CreateMeasurement()
     for stdout in runner():
-      self.ConsumeOutput(stdout)
-    return self.GetResults()
+      measurement.ConsumeOutput(stdout)
+    return measurement.GetResults()
 
 
 class RunnableGeneric(Runnable):
@@ -362,45 +452,7 @@ class RunnableGeneric(Runnable):
     super(RunnableGeneric, self).__init__(suite, parent, arch)
 
   def Run(self, runner):
-    """Iterates over several runs and handles the output."""
-    traces = OrderedDict()
-    for stdout in runner():
-      for line in stdout.strip().splitlines():
-        match = GENERIC_RESULTS_RE.match(line)
-        if match:
-          stddev = ""
-          graph = match.group(1)
-          trace = match.group(2)
-          body = match.group(3)
-          units = match.group(4)
-          match_stddev = RESULT_STDDEV_RE.match(body)
-          match_list = RESULT_LIST_RE.match(body)
-          errors = []
-          if match_stddev:
-            result, stddev = map(str.strip, match_stddev.group(1).split(","))
-            results = [result]
-          elif match_list:
-            results = map(str.strip, match_list.group(1).split(","))
-          else:
-            results = [body.strip()]
-
-          try:
-            results = map(lambda r: str(float(r)), results)
-          except ValueError:
-            results = []
-            errors = ["Found non-numeric in %s" %
-                      "/".join(self.graphs + [graph, trace])]
-
-          trace_result = traces.setdefault(trace, Results([{
-            "graphs": self.graphs + [graph, trace],
-            "units": (units or self.units).strip(),
-            "results": [],
-            "stddev": "",
-          }], errors))
-          trace_result.traces[0]["results"].extend(results)
-          trace_result.traces[0]["stddev"] = stddev
-
-    return reduce(lambda r, t: r + t, traces.itervalues(), Results())
+    return AccumulateGenericResults(self.graphs, self.units, runner)
 
 
 def MakeGraph(suite, arch, parent):