Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(240)

Issues Search
    My Issues | Starred     Open | Closed | All

Unified Diff: tools/unittests/run_benchmarks_test.py

Issue 564373003: Remove benchmark runner. (Closed) Base URL: https://v8.googlecode.com/svn/branches/bleeding_edge
Patch Set: Created 6 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « tools/run_benchmarks.py ('k') | no next file » | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: tools/unittests/run_benchmarks_test.py
diff --git a/tools/unittests/run_benchmarks_test.py b/tools/unittests/run_benchmarks_test.py
deleted file mode 100644
index d17025232c98356bbc1a69449f914ffb16c705c1..0000000000000000000000000000000000000000
--- a/tools/unittests/run_benchmarks_test.py
+++ /dev/null
@@ -1,349 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2014 the V8 project authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-from collections import namedtuple
-import coverage
-import json
-from mock import DEFAULT
-from mock import MagicMock
-import os
-from os import path, sys
-import shutil
-import tempfile
-import unittest
-
-# Requires python-coverage and python-mock. Native python coverage
-# version >= 3.7.1 should be installed to get the best speed.
-
-TEST_WORKSPACE = path.join(tempfile.gettempdir(), "test-v8-run-benchmarks")
-
-V8_JSON = {
- "path": ["."],
- "binary": "d7",
- "flags": ["--flag"],
- "main": "run.js",
- "run_count": 1,
- "results_regexp": "^%s: (.+)$",
- "benchmarks": [
- {"name": "Richards"},
- {"name": "DeltaBlue"},
- ]
-}
-
-V8_NESTED_SUITES_JSON = {
- "path": ["."],
- "flags": ["--flag"],
- "run_count": 1,
- "units": "score",
- "benchmarks": [
- {"name": "Richards",
- "path": ["richards"],
- "binary": "d7",
- "main": "run.js",
- "resources": ["file1.js", "file2.js"],
- "run_count": 2,
- "results_regexp": "^Richards: (.+)$"},
- {"name": "Sub",
- "path": ["sub"],
- "benchmarks": [
- {"name": "Leaf",
- "path": ["leaf"],
- "run_count_x64": 3,
- "units": "ms",
- "main": "run.js",
- "results_regexp": "^Simple: (.+) ms.$"},
- ]
- },
- {"name": "DeltaBlue",
- "path": ["delta_blue"],
- "main": "run.js",
- "flags": ["--flag2"],
- "results_regexp": "^DeltaBlue: (.+)$"},
- {"name": "ShouldntRun",
- "path": ["."],
- "archs": ["arm"],
- "main": "run.js"},
- ]
-}
-
-V8_GENERIC_JSON = {
- "path": ["."],
- "binary": "cc",
- "flags": ["--flag"],
- "generic": True,
- "run_count": 1,
- "units": "ms",
-}
-
-Output = namedtuple("Output", "stdout, stderr")
-
-class BenchmarksTest(unittest.TestCase):
- @classmethod
- def setUpClass(cls):
- cls.base = path.dirname(path.dirname(path.abspath(__file__)))
- sys.path.append(cls.base)
- cls._cov = coverage.coverage(
- include=([os.path.join(cls.base, "run_benchmarks.py")]))
- cls._cov.start()
- import run_benchmarks
- from testrunner.local import commands
- global commands
- global run_benchmarks
-
- @classmethod
- def tearDownClass(cls):
- cls._cov.stop()
- print ""
- print cls._cov.report()
-
- def setUp(self):
- self.maxDiff = None
- if path.exists(TEST_WORKSPACE):
- shutil.rmtree(TEST_WORKSPACE)
- os.makedirs(TEST_WORKSPACE)
-
- def tearDown(self):
- if path.exists(TEST_WORKSPACE):
- shutil.rmtree(TEST_WORKSPACE)
-
- def _WriteTestInput(self, json_content):
- self._test_input = path.join(TEST_WORKSPACE, "test.json")
- with open(self._test_input, "w") as f:
- f.write(json.dumps(json_content))
-
- def _MockCommand(self, *args):
- # Fake output for each benchmark run.
- benchmark_outputs = [Output(stdout=arg, stderr=None) for arg in args[1]]
- def execute(*args, **kwargs):
- return benchmark_outputs.pop()
- commands.Execute = MagicMock(side_effect=execute)
-
- # Check that d8 is called from the correct cwd for each benchmark run.
- dirs = [path.join(TEST_WORKSPACE, arg) for arg in args[0]]
- def chdir(*args, **kwargs):
- self.assertEquals(dirs.pop(), args[0])
- os.chdir = MagicMock(side_effect=chdir)
-
- def _CallMain(self, *args):
- self._test_output = path.join(TEST_WORKSPACE, "results.json")
- all_args=[
- "--json-test-results",
- self._test_output,
- self._test_input,
- ]
- all_args += args
- return run_benchmarks.Main(all_args)
-
- def _LoadResults(self):
- with open(self._test_output) as f:
- return json.load(f)
-
- def _VerifyResults(self, suite, units, traces):
- self.assertEquals([
- {"units": units,
- "graphs": [suite, trace["name"]],
- "results": trace["results"],
- "stddev": trace["stddev"]} for trace in traces],
- self._LoadResults()["traces"])
-
- def _VerifyErrors(self, errors):
- self.assertEquals(errors, self._LoadResults()["errors"])
-
- def _VerifyMock(self, binary, *args):
- arg = [path.join(path.dirname(self.base), binary)]
- arg += args
- commands.Execute.assert_called_with(arg, timeout=60)
-
- def _VerifyMockMultiple(self, *args):
- expected = []
- for arg in args:
- a = [path.join(path.dirname(self.base), arg[0])]
- a += arg[1:]
- expected.append(((a,), {"timeout": 60}))
- self.assertEquals(expected, commands.Execute.call_args_list)
-
- def testOneRun(self):
- self._WriteTestInput(V8_JSON)
- self._MockCommand(["."], ["x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n"])
- self.assertEquals(0, self._CallMain())
- self._VerifyResults("test", "score", [
- {"name": "Richards", "results": ["1.234"], "stddev": ""},
- {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
- ])
- self._VerifyErrors([])
- self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
-
- def testTwoRuns_Units_SuiteName(self):
- test_input = dict(V8_JSON)
- test_input["run_count"] = 2
- test_input["name"] = "v8"
- test_input["units"] = "ms"
- self._WriteTestInput(test_input)
- self._MockCommand([".", "."],
- ["Richards: 100\nDeltaBlue: 200\n",
- "Richards: 50\nDeltaBlue: 300\n"])
- self.assertEquals(0, self._CallMain())
- self._VerifyResults("v8", "ms", [
- {"name": "Richards", "results": ["50", "100"], "stddev": ""},
- {"name": "DeltaBlue", "results": ["300", "200"], "stddev": ""},
- ])
- self._VerifyErrors([])
- self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
-
- def testTwoRuns_SubRegexp(self):
- test_input = dict(V8_JSON)
- test_input["run_count"] = 2
- del test_input["results_regexp"]
- test_input["benchmarks"][0]["results_regexp"] = "^Richards: (.+)$"
- test_input["benchmarks"][1]["results_regexp"] = "^DeltaBlue: (.+)$"
- self._WriteTestInput(test_input)
- self._MockCommand([".", "."],
- ["Richards: 100\nDeltaBlue: 200\n",
- "Richards: 50\nDeltaBlue: 300\n"])
- self.assertEquals(0, self._CallMain())
- self._VerifyResults("test", "score", [
- {"name": "Richards", "results": ["50", "100"], "stddev": ""},
- {"name": "DeltaBlue", "results": ["300", "200"], "stddev": ""},
- ])
- self._VerifyErrors([])
- self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
-
- def testNestedSuite(self):
- self._WriteTestInput(V8_NESTED_SUITES_JSON)
- self._MockCommand(["delta_blue", "sub/leaf", "richards"],
- ["DeltaBlue: 200\n",
- "Simple: 1 ms.\n",
- "Simple: 2 ms.\n",
- "Simple: 3 ms.\n",
- "Richards: 100\n",
- "Richards: 50\n"])
- self.assertEquals(0, self._CallMain())
- self.assertEquals([
- {"units": "score",
- "graphs": ["test", "Richards"],
- "results": ["50", "100"],
- "stddev": ""},
- {"units": "ms",
- "graphs": ["test", "Sub", "Leaf"],
- "results": ["3", "2", "1"],
- "stddev": ""},
- {"units": "score",
- "graphs": ["test", "DeltaBlue"],
- "results": ["200"],
- "stddev": ""},
- ], self._LoadResults()["traces"])
- self._VerifyErrors([])
- self._VerifyMockMultiple(
- (path.join("out", "x64.release", "d7"), "--flag", "file1.js",
- "file2.js", "run.js"),
- (path.join("out", "x64.release", "d7"), "--flag", "file1.js",
- "file2.js", "run.js"),
- (path.join("out", "x64.release", "d8"), "--flag", "run.js"),
- (path.join("out", "x64.release", "d8"), "--flag", "run.js"),
- (path.join("out", "x64.release", "d8"), "--flag", "run.js"),
- (path.join("out", "x64.release", "d8"), "--flag", "--flag2", "run.js"))
-
- def testOneRunStdDevRegExp(self):
- test_input = dict(V8_JSON)
- test_input["stddev_regexp"] = "^%s\-stddev: (.+)$"
- self._WriteTestInput(test_input)
- self._MockCommand(["."], ["Richards: 1.234\nRichards-stddev: 0.23\n"
- "DeltaBlue: 10657567\nDeltaBlue-stddev: 106\n"])
- self.assertEquals(0, self._CallMain())
- self._VerifyResults("test", "score", [
- {"name": "Richards", "results": ["1.234"], "stddev": "0.23"},
- {"name": "DeltaBlue", "results": ["10657567"], "stddev": "106"},
- ])
- self._VerifyErrors([])
- self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
-
- def testTwoRunsStdDevRegExp(self):
- test_input = dict(V8_JSON)
- test_input["stddev_regexp"] = "^%s\-stddev: (.+)$"
- test_input["run_count"] = 2
- self._WriteTestInput(test_input)
- self._MockCommand(["."], ["Richards: 3\nRichards-stddev: 0.7\n"
- "DeltaBlue: 6\nDeltaBlue-boom: 0.9\n",
- "Richards: 2\nRichards-stddev: 0.5\n"
- "DeltaBlue: 5\nDeltaBlue-stddev: 0.8\n"])
- self.assertEquals(1, self._CallMain())
- self._VerifyResults("test", "score", [
- {"name": "Richards", "results": ["2", "3"], "stddev": "0.7"},
- {"name": "DeltaBlue", "results": ["5", "6"], "stddev": "0.8"},
- ])
- self._VerifyErrors(
- ["Benchmark Richards should only run once since a stddev is provided "
- "by the benchmark.",
- "Benchmark DeltaBlue should only run once since a stddev is provided "
- "by the benchmark.",
- "Regexp \"^DeltaBlue\-stddev: (.+)$\" didn't match for benchmark "
- "DeltaBlue."])
- self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
-
- def testBuildbot(self):
- self._WriteTestInput(V8_JSON)
- self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567\n"])
- self.assertEquals(0, self._CallMain("--buildbot"))
- self._VerifyResults("test", "score", [
- {"name": "Richards", "results": ["1.234"], "stddev": ""},
- {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
- ])
- self._VerifyErrors([])
- self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js")
-
- def testBuildbotWithTotal(self):
- test_input = dict(V8_JSON)
- test_input["total"] = True
- self._WriteTestInput(test_input)
- self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567\n"])
- self.assertEquals(0, self._CallMain("--buildbot"))
- self._VerifyResults("test", "score", [
- {"name": "Richards", "results": ["1.234"], "stddev": ""},
- {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
- {"name": "Total", "results": ["3626.49109719"], "stddev": ""},
- ])
- self._VerifyErrors([])
- self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js")
-
- def testBuildbotWithTotalAndErrors(self):
- test_input = dict(V8_JSON)
- test_input["total"] = True
- self._WriteTestInput(test_input)
- self._MockCommand(["."], ["x\nRichaards: 1.234\nDeltaBlue: 10657567\ny\n"])
- self.assertEquals(1, self._CallMain("--buildbot"))
- self._VerifyResults("test", "score", [
- {"name": "Richards", "results": [], "stddev": ""},
- {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
- ])
- self._VerifyErrors(
- ["Regexp \"^Richards: (.+)$\" didn't match for benchmark Richards.",
- "Not all traces have the same number of results."])
- self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js")
-
- def testRegexpNoMatch(self):
- self._WriteTestInput(V8_JSON)
- self._MockCommand(["."], ["x\nRichaards: 1.234\nDeltaBlue: 10657567\ny\n"])
- self.assertEquals(1, self._CallMain())
- self._VerifyResults("test", "score", [
- {"name": "Richards", "results": [], "stddev": ""},
- {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
- ])
- self._VerifyErrors(
- ["Regexp \"^Richards: (.+)$\" didn't match for benchmark Richards."])
- self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
-
- def testOneRunGeneric(self):
- test_input = dict(V8_GENERIC_JSON)
- self._WriteTestInput(test_input)
- self._MockCommand(["."], [
- "Trace(Test1), Result(1.234), StdDev(0.23)\n"
- "Trace(Test2), Result(10657567), StdDev(106)\n"])
- self.assertEquals(0, self._CallMain())
- self._VerifyResults("test", "ms", [
- {"name": "Test1", "results": ["1.234"], "stddev": "0.23"},
- {"name": "Test2", "results": ["10657567"], "stddev": "106"},
- ])
- self._VerifyErrors([])
- self._VerifyMock(path.join("out", "x64.release", "cc"), "--flag", "")
« no previous file with comments | « tools/run_benchmarks.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698