Teach benchmark runner to understand generic traces.

tools/run_benchmarks.py

 #!/usr/bin/env python
 # Copyright 2014 the V8 project authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """
 Performance runner for d8.
 
 Call e.g. with tools/run-benchmarks.py --arch ia32 some_suite.json
 
@@ skipping 95 matching lines @@
                    "android_ia32",
                    "arm",
                    "ia32",
                    "mips",
                    "mipsel",
                    "nacl_ia32",
                    "nacl_x64",
                    "x64",
                    "arm64"]
 
+GENERIC_RESULTS_RE = re.compile(
+    r"^Trace\(([^\)]+)\), Result\(([^\)]+)\), StdDev\(([^\)]+)\)$")
 
 class Results(object):
   """Place holder for result traces."""
   def __init__(self, traces=None, errors=None):
     self.traces = traces or []
     self.errors = errors or []
 
   def ToDict(self):
     return {"traces": self.traces, "errors": self.errors}
 
@@ skipping 116 matching lines @@
       "results": self.results,
       "stddev": self.stddev,
     }], self.errors)
 
 
 class Runnable(Graph):
   """Represents a runnable benchmark suite definition (i.e. has a main file).
   """
   @property
   def main(self):
-    return self._suite["main"]
+    return self._suite.get("main", "")
 
   def ChangeCWD(self, suite_path):
     """Changes the cwd to to path defined in the current graph.
 
     The benchmarks are supposed to be relative to the suite configuration.
     """
     suite_dir = os.path.abspath(os.path.dirname(suite_path))
     bench_dir = os.path.normpath(os.path.join(*self.path))
     os.chdir(os.path.join(suite_dir, bench_dir))
 
@@ skipping 19 matching lines @@
   def __init__(self, suite, parent, arch):
     super(RunnableTrace, self).__init__(suite, parent, arch)
 
   def Run(self, runner):
     """Iterates over several runs and handles the output."""
     for stdout in runner():
       self.ConsumeOutput(stdout)
     return self.GetResults()
 
 
+class RunnableGeneric(Runnable):
+  """Represents a runnable benchmark suite definition with generic traces."""
+  def __init__(self, suite, parent, arch):
+    super(RunnableGeneric, self).__init__(suite, parent, arch)
+
+  def Run(self, runner):
+    """Iterates over several runs and handles the output."""
+    traces = {}
+    for stdout in runner():
+      for line in stdout.strip().splitlines():
+        match = GENERIC_RESULTS_RE.match(line)
+        if match:
+          trace = match.group(1)
+          result = match.group(2)
+          stddev = match.group(3)
+          trace_result = traces.setdefault(trace, Results([{
+            "graphs": self.graphs + [trace],
+            "units": self.units,
+            "results": [],
+            "stddev": "",
+          }], []))
+          trace_result.traces[0]["results"].append(result)
+          trace_result.traces[0]["stddev"] = stddev

[Michael Achenbach, 2014/08/22 14:41:26]

Note: Having multiple reruns by the driver and a stddev from the executable
makes no sense. But this error case is not tested (instead, the provided stddev
would be overwritten in the end).

+
+    return reduce(lambda r, t: r + t, traces.itervalues(), Results())
+
+
 def MakeGraph(suite, arch, parent):
   """Factory method for making graph objects."""
   if isinstance(parent, Runnable):
     # Below a runnable can only be traces.
     return Trace(suite, parent, arch)
   elif suite.get("main"):
     # A main file makes this graph runnable.
     if suite.get("benchmarks"):
       # This graph has subbenchmarks (traces).
       return Runnable(suite, parent, arch)
     else:
       # This graph has no subbenchmarks, it's a leaf.
       return RunnableTrace(suite, parent, arch)
+  elif suite.get("generic"):
+    # This is a generic suite definition. It is either a runnable executable
+    # or has a main js file.
+    return RunnableGeneric(suite, parent, arch)
   elif suite.get("benchmarks"):
     # This is neither a leaf nor a runnable.
     return Graph(suite, parent, arch)
   else:  # pragma: no cover
     raise Exception("Invalid benchmark suite configuration.")
 
 
 def BuildGraphs(suite, arch, parent=None):
   """Builds a tree structure of graph objects that corresponds to the suite
   configuration.
@@ skipping 97 matching lines @@
 
   if options.json_test_results:
     results.WriteToFile(options.json_test_results)
   else:  # pragma: no cover
     print results
 
   return min(1, len(results.errors))
 
 if __name__ == "__main__":  # pragma: no cover
   sys.exit(Main(sys.argv[1:]))