Allow benchmarks to provide the standard deviation.

tools/run_benchmarks.py

 #!/usr/bin/env python
 # Copyright 2014 the V8 project authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """
 Performance runner for d8.
 
 Call e.g. with tools/run-benchmarks.py --arch ia32 some_suite.json
 
@@ skipping 138 matching lines @@
   """Fake parent node with all default values."""
   def __init__(self):
     super(DefaultSentinel, self).__init__()
     self.binary = "d8"
     self.run_count = 10
     self.path = []
     self.graphs = []
     self.flags = []
     self.resources = []
     self.results_regexp = None
+    self.stddev_regexp = None
     self.units = "score"
 
 
 class Graph(Node):
   """Represents a benchmark suite definition.
 
   Can either be a leaf or an inner node that provides default values.
   """
   def __init__(self, suite, parent, arch):
     super(Graph, self).__init__()
@@ skipping 20 matching lines @@
     # regexp and the current suite has none, a string place holder for the
     # suite name is expected.
     # TODO(machenbach): Currently that makes only sense for the leaf level.
     # Multiple place holders for multiple levels are not supported.
     if parent.results_regexp:
       regexp_default = parent.results_regexp % suite["name"]
     else:
       regexp_default = None
     self.results_regexp = suite.get("results_regexp", regexp_default)
 
+    # A similar regular expression for the standard deviation (optional).
+    if parent.stddev_regexp:
+      stddev_default = parent.stddev_regexp % suite["name"]
+    else:
+      stddev_default = None
+    self.stddev_regexp = suite.get("stddev_regexp", stddev_default)
+
 
 class Trace(Graph):
   """Represents a leaf in the benchmark suite tree structure.
 
   Handles collection of measurements.
   """
   def __init__(self, suite, parent, arch):
     super(Trace, self).__init__(suite, parent, arch)
     assert self.results_regexp
     self.results = []
     self.errors = []
+    self.stddev = ""
 
   def ConsumeOutput(self, stdout):
     try:
       self.results.append(
           re.search(self.results_regexp, stdout, re.M).group(1))
     except:
       self.errors.append("Regexp \"%s\" didn't match for benchmark %s."
                          % (self.results_regexp, self.graphs[-1]))
 
+    try:
+      if self.stddev_regexp and self.stddev:
+        self.errors.append("Benchmark %s should only run once since a stddev "
+                           "is provided by the benchmark." % self.graphs[-1])
+      if self.stddev_regexp:
+        self.stddev = re.search(self.stddev_regexp, stdout, re.M).group(1)
+    except:
+      self.errors.append("Regexp \"%s\" didn't match for benchmark %s."
+                         % (self.stddev_regexp, self.graphs[-1]))
+
   def GetResults(self):
     return Results([{
       "graphs": self.graphs,
       "units": self.units,
       "results": self.results,
+      "stddev": self.stddev,
     }], self.errors)
 
 
 class Runnable(Graph):
   """Represents a runnable benchmark suite definition (i.e. has a main file).
   """
   @property
   def main(self):
     return self._suite["main"]
 
@@ skipping 158 matching lines @@
 
   if options.json_test_results:
     results.WriteToFile(options.json_test_results)
   else:  # pragma: no cover
     print results
 
   return min(1, len(results.errors))
 
 if __name__ == "__main__":  # pragma: no cover
   sys.exit(Main(sys.argv[1:]))