Add test driver with the notion of perf tests.

tools/unittests/run_perf_test.py

 #!/usr/bin/env python
 # Copyright 2014 the V8 project authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from collections import namedtuple
 import coverage
 import json
 from mock import DEFAULT
 from mock import MagicMock
 import os
 from os import path, sys
 import shutil
 import tempfile
 import unittest
 
 # Requires python-coverage and python-mock. Native python coverage
 # version >= 3.7.1 should be installed to get the best speed.
 
-TEST_WORKSPACE = path.join(tempfile.gettempdir(), "test-v8-run-benchmarks")
+TEST_WORKSPACE = path.join(tempfile.gettempdir(), "test-v8-run-perf")
 
 V8_JSON = {
   "path": ["."],
   "binary": "d7",
   "flags": ["--flag"],
   "main": "run.js",
   "run_count": 1,
   "results_regexp": "^%s: (.+)$",
-  "benchmarks": [
+  "tests": [
     {"name": "Richards"},
     {"name": "DeltaBlue"},
   ]
 }
 
 V8_NESTED_SUITES_JSON = {
   "path": ["."],
   "flags": ["--flag"],
   "run_count": 1,
   "units": "score",
-  "benchmarks": [
+  "tests": [
     {"name": "Richards",
      "path": ["richards"],
      "binary": "d7",
      "main": "run.js",
      "resources": ["file1.js", "file2.js"],
      "run_count": 2,
      "results_regexp": "^Richards: (.+)$"},
     {"name": "Sub",
      "path": ["sub"],
-     "benchmarks": [
+     "tests": [
        {"name": "Leaf",
         "path": ["leaf"],
         "run_count_x64": 3,
         "units": "ms",
         "main": "run.js",
         "results_regexp": "^Simple: (.+) ms.$"},
      ]
     },
     {"name": "DeltaBlue",
      "path": ["delta_blue"],
@@ skipping 11 matching lines @@
   "path": ["."],
   "binary": "cc",
   "flags": ["--flag"],
   "generic": True,
   "run_count": 1,
   "units": "ms",
 }
 
 Output = namedtuple("Output", "stdout, stderr")
 
-class BenchmarksTest(unittest.TestCase):
+class PerfTest(unittest.TestCase):
   @classmethod
   def setUpClass(cls):
     cls.base = path.dirname(path.dirname(path.abspath(__file__)))
     sys.path.append(cls.base)
     cls._cov = coverage.coverage(
-        include=([os.path.join(cls.base, "run_benchmarks.py")]))
+        include=([os.path.join(cls.base, "run_perf.py")]))
     cls._cov.start()
-    import run_benchmarks
+    import run_perf
     from testrunner.local import commands
     global commands
-    global run_benchmarks
+    global run_perf
 
   @classmethod
   def tearDownClass(cls):
     cls._cov.stop()
     print ""
     print cls._cov.report()
 
   def setUp(self):
     self.maxDiff = None
     if path.exists(TEST_WORKSPACE):
       shutil.rmtree(TEST_WORKSPACE)
     os.makedirs(TEST_WORKSPACE)
 
   def tearDown(self):
     if path.exists(TEST_WORKSPACE):
       shutil.rmtree(TEST_WORKSPACE)
 
   def _WriteTestInput(self, json_content):
     self._test_input = path.join(TEST_WORKSPACE, "test.json")
     with open(self._test_input, "w") as f:
       f.write(json.dumps(json_content))
 
   def _MockCommand(self, *args):
-    # Fake output for each benchmark run.
-    benchmark_outputs = [Output(stdout=arg, stderr=None) for arg in args[1]]
+    # Fake output for each test run.
+    test_outputs = [Output(stdout=arg, stderr=None) for arg in args[1]]
     def execute(*args, **kwargs):
-      return benchmark_outputs.pop()
+      return test_outputs.pop()
     commands.Execute = MagicMock(side_effect=execute)
 
-    # Check that d8 is called from the correct cwd for each benchmark run.
+    # Check that d8 is called from the correct cwd for each test run.
     dirs = [path.join(TEST_WORKSPACE, arg) for arg in args[0]]
     def chdir(*args, **kwargs):
       self.assertEquals(dirs.pop(), args[0])
     os.chdir = MagicMock(side_effect=chdir)
 
   def _CallMain(self, *args):
     self._test_output = path.join(TEST_WORKSPACE, "results.json")
     all_args=[
       "--json-test-results",
       self._test_output,
       self._test_input,
     ]
     all_args += args
-    return run_benchmarks.Main(all_args)
+    return run_perf.Main(all_args)
 
   def _LoadResults(self):
     with open(self._test_output) as f:
       return json.load(f)
 
   def _VerifyResults(self, suite, units, traces):
     self.assertEquals([
       {"units": units,
        "graphs": [suite, trace["name"]],
        "results": trace["results"],
@@ skipping 41 matching lines @@
       {"name": "Richards", "results": ["50", "100"], "stddev": ""},
       {"name": "DeltaBlue", "results": ["300", "200"], "stddev": ""},
     ])
     self._VerifyErrors([])
     self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
 
   def testTwoRuns_SubRegexp(self):
     test_input = dict(V8_JSON)
     test_input["run_count"] = 2
     del test_input["results_regexp"]
-    test_input["benchmarks"][0]["results_regexp"] = "^Richards: (.+)$"
-    test_input["benchmarks"][1]["results_regexp"] = "^DeltaBlue: (.+)$"
+    test_input["tests"][0]["results_regexp"] = "^Richards: (.+)$"
+    test_input["tests"][1]["results_regexp"] = "^DeltaBlue: (.+)$"
     self._WriteTestInput(test_input)
     self._MockCommand([".", "."],
                       ["Richards: 100\nDeltaBlue: 200\n",
                        "Richards: 50\nDeltaBlue: 300\n"])
     self.assertEquals(0, self._CallMain())
     self._VerifyResults("test", "score", [
       {"name": "Richards", "results": ["50", "100"], "stddev": ""},
       {"name": "DeltaBlue", "results": ["300", "200"], "stddev": ""},
     ])
     self._VerifyErrors([])
@@ skipping 56 matching lines @@
     self._MockCommand(["."], ["Richards: 3\nRichards-stddev: 0.7\n"
                               "DeltaBlue: 6\nDeltaBlue-boom: 0.9\n",
                               "Richards: 2\nRichards-stddev: 0.5\n"
                               "DeltaBlue: 5\nDeltaBlue-stddev: 0.8\n"])
     self.assertEquals(1, self._CallMain())
     self._VerifyResults("test", "score", [
       {"name": "Richards", "results": ["2", "3"], "stddev": "0.7"},
       {"name": "DeltaBlue", "results": ["5", "6"], "stddev": "0.8"},
     ])
     self._VerifyErrors(
-        ["Benchmark Richards should only run once since a stddev is provided "
-         "by the benchmark.",
-         "Benchmark DeltaBlue should only run once since a stddev is provided "
-         "by the benchmark.",
-         "Regexp \"^DeltaBlue\-stddev: (.+)$\" didn't match for benchmark "
+        ["Test Richards should only run once since a stddev is provided "
+         "by the test.",
+         "Test DeltaBlue should only run once since a stddev is provided "
+         "by the test.",
+         "Regexp \"^DeltaBlue\-stddev: (.+)$\" didn't match for test "
          "DeltaBlue."])
     self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
 
   def testBuildbot(self):
     self._WriteTestInput(V8_JSON)
     self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567\n"])
     self.assertEquals(0, self._CallMain("--buildbot"))
     self._VerifyResults("test", "score", [
       {"name": "Richards", "results": ["1.234"], "stddev": ""},
       {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
@@ skipping 19 matching lines @@
     test_input = dict(V8_JSON)
     test_input["total"] = True
     self._WriteTestInput(test_input)
     self._MockCommand(["."], ["x\nRichaards: 1.234\nDeltaBlue: 10657567\ny\n"])
     self.assertEquals(1, self._CallMain("--buildbot"))
     self._VerifyResults("test", "score", [
       {"name": "Richards", "results": [], "stddev": ""},
       {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
     ])
     self._VerifyErrors(
-        ["Regexp \"^Richards: (.+)$\" didn't match for benchmark Richards.",
+        ["Regexp \"^Richards: (.+)$\" didn't match for test Richards.",
          "Not all traces have the same number of results."])
     self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js")
 
   def testRegexpNoMatch(self):
     self._WriteTestInput(V8_JSON)
     self._MockCommand(["."], ["x\nRichaards: 1.234\nDeltaBlue: 10657567\ny\n"])
     self.assertEquals(1, self._CallMain())
     self._VerifyResults("test", "score", [
       {"name": "Richards", "results": [], "stddev": ""},
       {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
     ])
     self._VerifyErrors(
-        ["Regexp \"^Richards: (.+)$\" didn't match for benchmark Richards."])
+        ["Regexp \"^Richards: (.+)$\" didn't match for test Richards."])
     self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
 
   def testOneRunGeneric(self):
     test_input = dict(V8_GENERIC_JSON)
     self._WriteTestInput(test_input)
     self._MockCommand(["."], [
       "Trace(Test1), Result(1.234), StdDev(0.23)\n"
       "Trace(Test2), Result(10657567), StdDev(106)\n"])
     self.assertEquals(0, self._CallMain())
     self._VerifyResults("test", "ms", [
       {"name": "Test1", "results": ["1.234"], "stddev": "0.23"},
       {"name": "Test2", "results": ["10657567"], "stddev": "106"},
     ])
     self._VerifyErrors([])
     self._VerifyMock(path.join("out", "x64.release", "cc"), "--flag", "")