Add test flags feature to perf runner.

tools/run_perf.py

 #!/usr/bin/env python
 # Copyright 2014 the V8 project authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """
 Performance runner for d8.
 
 Call e.g. with tools/run-perf.py --arch ia32 some_suite.json
 
 The suite json format is expected to be:
 {
   "path": <relative path chunks to perf resources and main file>,
   "name": <optional suite name, file name is default>,
   "archs": [<architecture name for which this suite is run>, ...],
   "binary": <name of binary to run, default "d8">,
   "flags": [<flag to d8>, ...],
+  "test_flags": [<flag to the test file>, ...],
   "run_count": <how often will this suite run (optional)>,
   "run_count_XXX": <how often will this suite run for arch XXX (optional)>,
   "resources": [<js file to be loaded before main>, ...]
   "main": <main js perf runner file>,
   "results_regexp": <optional regexp>,
   "results_processor": <optional python results processor script>,
   "units": <the unit specification for the performance dashboard>,
   "tests": [
     {
       "name": <name of the trace>,
@@ skipping 19 matching lines @@
 
 The <output> is a temporary file containing d8 output. The results_regexp will
 be applied to the output of this script.
 
 A suite without "tests" is considered a performance test itself.
 
 Full example (suite with one runner):
 {
   "path": ["."],
   "flags": ["--expose-gc"],
+  "test_flags": ["5"],
   "archs": ["ia32", "x64"],
   "run_count": 5,
   "run_count_ia32": 3,
   "main": "run.js",
   "results_regexp": "^%s: (.+)$",
   "units": "score",
   "tests": [
     {"name": "Richards"},
     {"name": "DeltaBlue"},
     {"name": "NavierStokes",
@@ skipping 15 matching lines @@
      "run_count": 3,
      "results_regexp": "^Richards: (.+)$"},
     {"name": "NavierStokes",
      "path": ["navier_stokes"],
      "main": "run.js",
      "results_regexp": "^NavierStokes: (.+)$"}
   ]
 }
 
 Path pieces are concatenated. D8 is always run with the suite's path as cwd.
+
+The test flags are passed to the js test file after '--'.
 """
 
 from collections import OrderedDict
 import json
 import math
 import optparse
 import os
 import re
 import sys
 
@@ skipping 62 matching lines @@
 class DefaultSentinel(Node):
   """Fake parent node with all default values."""
   def __init__(self):
     super(DefaultSentinel, self).__init__()
     self.binary = "d8"
     self.run_count = 10
     self.timeout = 60
     self.path = []
     self.graphs = []
     self.flags = []
+    self.test_flags = []
     self.resources = []
     self.results_regexp = None
     self.stddev_regexp = None
     self.units = "score"
     self.total = False
 
 
 class Graph(Node):
   """Represents a suite definition.
 
   Can either be a leaf or an inner node that provides default values.
   """
   def __init__(self, suite, parent, arch):
     super(Graph, self).__init__()
     self._suite = suite
 
     assert isinstance(suite.get("path", []), list)
     assert isinstance(suite["name"], basestring)
     assert isinstance(suite.get("flags", []), list)
+    assert isinstance(suite.get("test_flags", []), list)
     assert isinstance(suite.get("resources", []), list)
 
     # Accumulated values.
     self.path = parent.path[:] + suite.get("path", [])
     self.graphs = parent.graphs[:] + [suite["name"]]
     self.flags = parent.flags[:] + suite.get("flags", [])
+    self.test_flags = parent.test_flags[:] + suite.get("test_flags", [])
     self.resources = parent.resources[:] + suite.get("resources", [])
 
     # Descrete values (with parent defaults).
     self.binary = suite.get("binary", parent.binary)
     self.run_count = suite.get("run_count", parent.run_count)
     self.run_count = suite.get("run_count_%s" % arch, self.run_count)
     self.timeout = suite.get("timeout", parent.timeout)
     self.units = suite.get("units", parent.units)
     self.total = suite.get("total", parent.total)
 
@@ skipping 66 matching lines @@
     """Changes the cwd to to path defined in the current graph.
 
     The tests are supposed to be relative to the suite configuration.
     """
     suite_dir = os.path.abspath(os.path.dirname(suite_path))
     bench_dir = os.path.normpath(os.path.join(*self.path))
     os.chdir(os.path.join(suite_dir, bench_dir))
 
   def GetCommand(self, shell_dir):
     # TODO(machenbach): This requires +.exe if run on windows.
+    suffix = ["--"] + self.test_flags if self.test_flags else []
     return (
       [os.path.join(shell_dir, self.binary)] +
       self.flags +
       self.resources +
-      [self.main]
+      [self.main] +
+      suffix
     )
 
   def Run(self, runner):
     """Iterates over several runs and handles the output for all traces."""
     for stdout in runner():
       for trace in self._children:
         trace.ConsumeOutput(stdout)
     res = reduce(lambda r, t: r + t.GetResults(), self._children, Results())
 
     if not res.traces or not self.total:
@@ skipping 199 matching lines @@
 
   if options.json_test_results:
     results.WriteToFile(options.json_test_results)
   else:  # pragma: no cover
     print results
 
   return min(1, len(results.errors))
 
 if __name__ == "__main__":  # pragma: no cover
   sys.exit(Main(sys.argv[1:]))