OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2014 the V8 project authors. All rights reserved. | 2 # Copyright 2014 the V8 project authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 from collections import namedtuple | 6 from collections import namedtuple |
7 import coverage | 7 import coverage |
8 import json | 8 import json |
9 from mock import DEFAULT | 9 from mock import DEFAULT |
10 from mock import MagicMock | 10 from mock import MagicMock |
(...skipping 107 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
118 os.chdir = MagicMock(side_effect=chdir) | 118 os.chdir = MagicMock(side_effect=chdir) |
119 | 119 |
120 def _CallMain(self, *args): | 120 def _CallMain(self, *args): |
121 self._test_output = path.join(TEST_WORKSPACE, "results.json") | 121 self._test_output = path.join(TEST_WORKSPACE, "results.json") |
122 all_args=[ | 122 all_args=[ |
123 "--json-test-results", | 123 "--json-test-results", |
124 self._test_output, | 124 self._test_output, |
125 self._test_input, | 125 self._test_input, |
126 ] | 126 ] |
127 all_args += args | 127 all_args += args |
128 run_benchmarks.Main(all_args) | 128 return run_benchmarks.Main(all_args) |
129 | 129 |
130 def _LoadResults(self): | 130 def _LoadResults(self): |
131 with open(self._test_output) as f: | 131 with open(self._test_output) as f: |
132 return json.load(f) | 132 return json.load(f) |
133 | 133 |
134 def _VerifyResults(self, suite, units, traces): | 134 def _VerifyResults(self, suite, units, traces): |
135 self.assertEquals([ | 135 self.assertEquals([ |
136 {"units": units, | 136 {"units": units, |
137 "graphs": [suite, trace["name"]], | 137 "graphs": [suite, trace["name"]], |
138 "results": trace["results"]} for trace in traces], | 138 "results": trace["results"]} for trace in traces], |
(...skipping 11 matching lines...) Expand all Loading... |
150 expected = [] | 150 expected = [] |
151 for arg in args: | 151 for arg in args: |
152 a = [path.join(path.dirname(self.base), arg[0])] | 152 a = [path.join(path.dirname(self.base), arg[0])] |
153 a += arg[1:] | 153 a += arg[1:] |
154 expected.append(((a,), {"timeout": 60})) | 154 expected.append(((a,), {"timeout": 60})) |
155 self.assertEquals(expected, commands.Execute.call_args_list) | 155 self.assertEquals(expected, commands.Execute.call_args_list) |
156 | 156 |
157 def testOneRun(self): | 157 def testOneRun(self): |
158 self._WriteTestInput(V8_JSON) | 158 self._WriteTestInput(V8_JSON) |
159 self._MockCommand(["."], ["x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n"]) | 159 self._MockCommand(["."], ["x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n"]) |
160 self._CallMain() | 160 self.assertEquals(0, self._CallMain()) |
161 self._VerifyResults("test", "score", [ | 161 self._VerifyResults("test", "score", [ |
162 {"name": "Richards", "results": ["1.234"]}, | 162 {"name": "Richards", "results": ["1.234"]}, |
163 {"name": "DeltaBlue", "results": ["10657567"]}, | 163 {"name": "DeltaBlue", "results": ["10657567"]}, |
164 ]) | 164 ]) |
165 self._VerifyErrors([]) | 165 self._VerifyErrors([]) |
166 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") | 166 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") |
167 | 167 |
168 def testTwoRuns_Units_SuiteName(self): | 168 def testTwoRuns_Units_SuiteName(self): |
169 test_input = dict(V8_JSON) | 169 test_input = dict(V8_JSON) |
170 test_input["run_count"] = 2 | 170 test_input["run_count"] = 2 |
171 test_input["name"] = "v8" | 171 test_input["name"] = "v8" |
172 test_input["units"] = "ms" | 172 test_input["units"] = "ms" |
173 self._WriteTestInput(test_input) | 173 self._WriteTestInput(test_input) |
174 self._MockCommand([".", "."], | 174 self._MockCommand([".", "."], |
175 ["Richards: 100\nDeltaBlue: 200\n", | 175 ["Richards: 100\nDeltaBlue: 200\n", |
176 "Richards: 50\nDeltaBlue: 300\n"]) | 176 "Richards: 50\nDeltaBlue: 300\n"]) |
177 self._CallMain() | 177 self.assertEquals(0, self._CallMain()) |
178 self._VerifyResults("v8", "ms", [ | 178 self._VerifyResults("v8", "ms", [ |
179 {"name": "Richards", "results": ["50", "100"]}, | 179 {"name": "Richards", "results": ["50", "100"]}, |
180 {"name": "DeltaBlue", "results": ["300", "200"]}, | 180 {"name": "DeltaBlue", "results": ["300", "200"]}, |
181 ]) | 181 ]) |
182 self._VerifyErrors([]) | 182 self._VerifyErrors([]) |
183 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") | 183 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") |
184 | 184 |
185 def testTwoRuns_SubRegexp(self): | 185 def testTwoRuns_SubRegexp(self): |
186 test_input = dict(V8_JSON) | 186 test_input = dict(V8_JSON) |
187 test_input["run_count"] = 2 | 187 test_input["run_count"] = 2 |
188 del test_input["results_regexp"] | 188 del test_input["results_regexp"] |
189 test_input["benchmarks"][0]["results_regexp"] = "^Richards: (.+)$" | 189 test_input["benchmarks"][0]["results_regexp"] = "^Richards: (.+)$" |
190 test_input["benchmarks"][1]["results_regexp"] = "^DeltaBlue: (.+)$" | 190 test_input["benchmarks"][1]["results_regexp"] = "^DeltaBlue: (.+)$" |
191 self._WriteTestInput(test_input) | 191 self._WriteTestInput(test_input) |
192 self._MockCommand([".", "."], | 192 self._MockCommand([".", "."], |
193 ["Richards: 100\nDeltaBlue: 200\n", | 193 ["Richards: 100\nDeltaBlue: 200\n", |
194 "Richards: 50\nDeltaBlue: 300\n"]) | 194 "Richards: 50\nDeltaBlue: 300\n"]) |
195 self._CallMain() | 195 self.assertEquals(0, self._CallMain()) |
196 self._VerifyResults("test", "score", [ | 196 self._VerifyResults("test", "score", [ |
197 {"name": "Richards", "results": ["50", "100"]}, | 197 {"name": "Richards", "results": ["50", "100"]}, |
198 {"name": "DeltaBlue", "results": ["300", "200"]}, | 198 {"name": "DeltaBlue", "results": ["300", "200"]}, |
199 ]) | 199 ]) |
200 self._VerifyErrors([]) | 200 self._VerifyErrors([]) |
201 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") | 201 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") |
202 | 202 |
203 def testNestedSuite(self): | 203 def testNestedSuite(self): |
204 self._WriteTestInput(V8_NESTED_SUITES_JSON) | 204 self._WriteTestInput(V8_NESTED_SUITES_JSON) |
205 self._MockCommand(["delta_blue", "sub/leaf", "richards"], | 205 self._MockCommand(["delta_blue", "sub/leaf", "richards"], |
206 ["DeltaBlue: 200\n", | 206 ["DeltaBlue: 200\n", |
207 "Simple: 1 ms.\n", | 207 "Simple: 1 ms.\n", |
208 "Simple: 2 ms.\n", | 208 "Simple: 2 ms.\n", |
209 "Simple: 3 ms.\n", | 209 "Simple: 3 ms.\n", |
210 "Richards: 100\n", | 210 "Richards: 100\n", |
211 "Richards: 50\n"]) | 211 "Richards: 50\n"]) |
212 self._CallMain() | 212 self.assertEquals(0, self._CallMain()) |
213 self.assertEquals([ | 213 self.assertEquals([ |
214 {"units": "score", | 214 {"units": "score", |
215 "graphs": ["test", "Richards"], | 215 "graphs": ["test", "Richards"], |
216 "results": ["50", "100"]}, | 216 "results": ["50", "100"]}, |
217 {"units": "ms", | 217 {"units": "ms", |
218 "graphs": ["test", "Sub", "Leaf"], | 218 "graphs": ["test", "Sub", "Leaf"], |
219 "results": ["3", "2", "1"]}, | 219 "results": ["3", "2", "1"]}, |
220 {"units": "score", | 220 {"units": "score", |
221 "graphs": ["test", "DeltaBlue"], | 221 "graphs": ["test", "DeltaBlue"], |
222 "results": ["200"]}, | 222 "results": ["200"]}, |
223 ], self._LoadResults()["traces"]) | 223 ], self._LoadResults()["traces"]) |
224 self._VerifyErrors([]) | 224 self._VerifyErrors([]) |
225 self._VerifyMockMultiple( | 225 self._VerifyMockMultiple( |
226 (path.join("out", "x64.release", "d7"), "--flag", "file1.js", | 226 (path.join("out", "x64.release", "d7"), "--flag", "file1.js", |
227 "file2.js", "run.js"), | 227 "file2.js", "run.js"), |
228 (path.join("out", "x64.release", "d7"), "--flag", "file1.js", | 228 (path.join("out", "x64.release", "d7"), "--flag", "file1.js", |
229 "file2.js", "run.js"), | 229 "file2.js", "run.js"), |
230 (path.join("out", "x64.release", "d8"), "--flag", "run.js"), | 230 (path.join("out", "x64.release", "d8"), "--flag", "run.js"), |
231 (path.join("out", "x64.release", "d8"), "--flag", "run.js"), | 231 (path.join("out", "x64.release", "d8"), "--flag", "run.js"), |
232 (path.join("out", "x64.release", "d8"), "--flag", "run.js"), | 232 (path.join("out", "x64.release", "d8"), "--flag", "run.js"), |
233 (path.join("out", "x64.release", "d8"), "--flag", "--flag2", "run.js")) | 233 (path.join("out", "x64.release", "d8"), "--flag", "--flag2", "run.js")) |
234 | 234 |
235 def testBuildbot(self): | 235 def testBuildbot(self): |
236 self._WriteTestInput(V8_JSON) | 236 self._WriteTestInput(V8_JSON) |
237 self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567\n"]) | 237 self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567\n"]) |
238 self._CallMain("--buildbot") | 238 self.assertEquals(0, self._CallMain("--buildbot")) |
239 self._VerifyResults("test", "score", [ | 239 self._VerifyResults("test", "score", [ |
240 {"name": "Richards", "results": ["1.234"]}, | 240 {"name": "Richards", "results": ["1.234"]}, |
241 {"name": "DeltaBlue", "results": ["10657567"]}, | 241 {"name": "DeltaBlue", "results": ["10657567"]}, |
242 ]) | 242 ]) |
243 self._VerifyErrors([]) | 243 self._VerifyErrors([]) |
244 self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js") | 244 self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js") |
245 | 245 |
246 def testRegexpNoMatch(self): | 246 def testRegexpNoMatch(self): |
247 self._WriteTestInput(V8_JSON) | 247 self._WriteTestInput(V8_JSON) |
248 self._MockCommand(["."], ["x\nRichaards: 1.234\nDeltaBlue: 10657567\ny\n"]) | 248 self._MockCommand(["."], ["x\nRichaards: 1.234\nDeltaBlue: 10657567\ny\n"]) |
249 self._CallMain() | 249 self.assertEquals(1, self._CallMain()) |
250 self._VerifyResults("test", "score", [ | 250 self._VerifyResults("test", "score", [ |
251 {"name": "Richards", "results": []}, | 251 {"name": "Richards", "results": []}, |
252 {"name": "DeltaBlue", "results": ["10657567"]}, | 252 {"name": "DeltaBlue", "results": ["10657567"]}, |
253 ]) | 253 ]) |
254 self._VerifyErrors( | 254 self._VerifyErrors( |
255 ["Regexp \"^Richards: (.+)$\" didn't match for benchmark Richards."]) | 255 ["Regexp \"^Richards: (.+)$\" didn't match for benchmark Richards."]) |
256 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") | 256 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") |
OLD | NEW |