OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2014 the V8 project authors. All rights reserved. | 2 # Copyright 2014 the V8 project authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 from collections import namedtuple | 6 from collections import namedtuple |
7 import coverage | 7 import coverage |
8 import json | 8 import json |
9 from mock import DEFAULT | 9 from mock import DEFAULT |
10 from mock import MagicMock | 10 from mock import MagicMock |
(...skipping 117 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
128 return run_benchmarks.Main(all_args) | 128 return run_benchmarks.Main(all_args) |
129 | 129 |
130 def _LoadResults(self): | 130 def _LoadResults(self): |
131 with open(self._test_output) as f: | 131 with open(self._test_output) as f: |
132 return json.load(f) | 132 return json.load(f) |
133 | 133 |
134 def _VerifyResults(self, suite, units, traces): | 134 def _VerifyResults(self, suite, units, traces): |
135 self.assertEquals([ | 135 self.assertEquals([ |
136 {"units": units, | 136 {"units": units, |
137 "graphs": [suite, trace["name"]], | 137 "graphs": [suite, trace["name"]], |
138 "results": trace["results"]} for trace in traces], | 138 "results": trace["results"], |
139 self._LoadResults()["traces"]) | 139 "stddev": trace["stddev"]} for trace in traces], |
| 140 self._LoadResults()["traces"]) |
140 | 141 |
141 def _VerifyErrors(self, errors): | 142 def _VerifyErrors(self, errors): |
142 self.assertEquals(errors, self._LoadResults()["errors"]) | 143 self.assertEquals(errors, self._LoadResults()["errors"]) |
143 | 144 |
144 def _VerifyMock(self, binary, *args): | 145 def _VerifyMock(self, binary, *args): |
145 arg = [path.join(path.dirname(self.base), binary)] | 146 arg = [path.join(path.dirname(self.base), binary)] |
146 arg += args | 147 arg += args |
147 commands.Execute.assert_called_with(arg, timeout=60) | 148 commands.Execute.assert_called_with(arg, timeout=60) |
148 | 149 |
149 def _VerifyMockMultiple(self, *args): | 150 def _VerifyMockMultiple(self, *args): |
150 expected = [] | 151 expected = [] |
151 for arg in args: | 152 for arg in args: |
152 a = [path.join(path.dirname(self.base), arg[0])] | 153 a = [path.join(path.dirname(self.base), arg[0])] |
153 a += arg[1:] | 154 a += arg[1:] |
154 expected.append(((a,), {"timeout": 60})) | 155 expected.append(((a,), {"timeout": 60})) |
155 self.assertEquals(expected, commands.Execute.call_args_list) | 156 self.assertEquals(expected, commands.Execute.call_args_list) |
156 | 157 |
157 def testOneRun(self): | 158 def testOneRun(self): |
158 self._WriteTestInput(V8_JSON) | 159 self._WriteTestInput(V8_JSON) |
159 self._MockCommand(["."], ["x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n"]) | 160 self._MockCommand(["."], ["x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n"]) |
160 self.assertEquals(0, self._CallMain()) | 161 self.assertEquals(0, self._CallMain()) |
161 self._VerifyResults("test", "score", [ | 162 self._VerifyResults("test", "score", [ |
162 {"name": "Richards", "results": ["1.234"]}, | 163 {"name": "Richards", "results": ["1.234"], "stddev": ""}, |
163 {"name": "DeltaBlue", "results": ["10657567"]}, | 164 {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""}, |
164 ]) | 165 ]) |
165 self._VerifyErrors([]) | 166 self._VerifyErrors([]) |
166 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") | 167 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") |
167 | 168 |
168 def testTwoRuns_Units_SuiteName(self): | 169 def testTwoRuns_Units_SuiteName(self): |
169 test_input = dict(V8_JSON) | 170 test_input = dict(V8_JSON) |
170 test_input["run_count"] = 2 | 171 test_input["run_count"] = 2 |
171 test_input["name"] = "v8" | 172 test_input["name"] = "v8" |
172 test_input["units"] = "ms" | 173 test_input["units"] = "ms" |
173 self._WriteTestInput(test_input) | 174 self._WriteTestInput(test_input) |
174 self._MockCommand([".", "."], | 175 self._MockCommand([".", "."], |
175 ["Richards: 100\nDeltaBlue: 200\n", | 176 ["Richards: 100\nDeltaBlue: 200\n", |
176 "Richards: 50\nDeltaBlue: 300\n"]) | 177 "Richards: 50\nDeltaBlue: 300\n"]) |
177 self.assertEquals(0, self._CallMain()) | 178 self.assertEquals(0, self._CallMain()) |
178 self._VerifyResults("v8", "ms", [ | 179 self._VerifyResults("v8", "ms", [ |
179 {"name": "Richards", "results": ["50", "100"]}, | 180 {"name": "Richards", "results": ["50", "100"], "stddev": ""}, |
180 {"name": "DeltaBlue", "results": ["300", "200"]}, | 181 {"name": "DeltaBlue", "results": ["300", "200"], "stddev": ""}, |
181 ]) | 182 ]) |
182 self._VerifyErrors([]) | 183 self._VerifyErrors([]) |
183 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") | 184 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") |
184 | 185 |
185 def testTwoRuns_SubRegexp(self): | 186 def testTwoRuns_SubRegexp(self): |
186 test_input = dict(V8_JSON) | 187 test_input = dict(V8_JSON) |
187 test_input["run_count"] = 2 | 188 test_input["run_count"] = 2 |
188 del test_input["results_regexp"] | 189 del test_input["results_regexp"] |
189 test_input["benchmarks"][0]["results_regexp"] = "^Richards: (.+)$" | 190 test_input["benchmarks"][0]["results_regexp"] = "^Richards: (.+)$" |
190 test_input["benchmarks"][1]["results_regexp"] = "^DeltaBlue: (.+)$" | 191 test_input["benchmarks"][1]["results_regexp"] = "^DeltaBlue: (.+)$" |
191 self._WriteTestInput(test_input) | 192 self._WriteTestInput(test_input) |
192 self._MockCommand([".", "."], | 193 self._MockCommand([".", "."], |
193 ["Richards: 100\nDeltaBlue: 200\n", | 194 ["Richards: 100\nDeltaBlue: 200\n", |
194 "Richards: 50\nDeltaBlue: 300\n"]) | 195 "Richards: 50\nDeltaBlue: 300\n"]) |
195 self.assertEquals(0, self._CallMain()) | 196 self.assertEquals(0, self._CallMain()) |
196 self._VerifyResults("test", "score", [ | 197 self._VerifyResults("test", "score", [ |
197 {"name": "Richards", "results": ["50", "100"]}, | 198 {"name": "Richards", "results": ["50", "100"], "stddev": ""}, |
198 {"name": "DeltaBlue", "results": ["300", "200"]}, | 199 {"name": "DeltaBlue", "results": ["300", "200"], "stddev": ""}, |
199 ]) | 200 ]) |
200 self._VerifyErrors([]) | 201 self._VerifyErrors([]) |
201 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") | 202 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") |
202 | 203 |
203 def testNestedSuite(self): | 204 def testNestedSuite(self): |
204 self._WriteTestInput(V8_NESTED_SUITES_JSON) | 205 self._WriteTestInput(V8_NESTED_SUITES_JSON) |
205 self._MockCommand(["delta_blue", "sub/leaf", "richards"], | 206 self._MockCommand(["delta_blue", "sub/leaf", "richards"], |
206 ["DeltaBlue: 200\n", | 207 ["DeltaBlue: 200\n", |
207 "Simple: 1 ms.\n", | 208 "Simple: 1 ms.\n", |
208 "Simple: 2 ms.\n", | 209 "Simple: 2 ms.\n", |
209 "Simple: 3 ms.\n", | 210 "Simple: 3 ms.\n", |
210 "Richards: 100\n", | 211 "Richards: 100\n", |
211 "Richards: 50\n"]) | 212 "Richards: 50\n"]) |
212 self.assertEquals(0, self._CallMain()) | 213 self.assertEquals(0, self._CallMain()) |
213 self.assertEquals([ | 214 self.assertEquals([ |
214 {"units": "score", | 215 {"units": "score", |
215 "graphs": ["test", "Richards"], | 216 "graphs": ["test", "Richards"], |
216 "results": ["50", "100"]}, | 217 "results": ["50", "100"], |
| 218 "stddev": ""}, |
217 {"units": "ms", | 219 {"units": "ms", |
218 "graphs": ["test", "Sub", "Leaf"], | 220 "graphs": ["test", "Sub", "Leaf"], |
219 "results": ["3", "2", "1"]}, | 221 "results": ["3", "2", "1"], |
| 222 "stddev": ""}, |
220 {"units": "score", | 223 {"units": "score", |
221 "graphs": ["test", "DeltaBlue"], | 224 "graphs": ["test", "DeltaBlue"], |
222 "results": ["200"]}, | 225 "results": ["200"], |
| 226 "stddev": ""}, |
223 ], self._LoadResults()["traces"]) | 227 ], self._LoadResults()["traces"]) |
224 self._VerifyErrors([]) | 228 self._VerifyErrors([]) |
225 self._VerifyMockMultiple( | 229 self._VerifyMockMultiple( |
226 (path.join("out", "x64.release", "d7"), "--flag", "file1.js", | 230 (path.join("out", "x64.release", "d7"), "--flag", "file1.js", |
227 "file2.js", "run.js"), | 231 "file2.js", "run.js"), |
228 (path.join("out", "x64.release", "d7"), "--flag", "file1.js", | 232 (path.join("out", "x64.release", "d7"), "--flag", "file1.js", |
229 "file2.js", "run.js"), | 233 "file2.js", "run.js"), |
230 (path.join("out", "x64.release", "d8"), "--flag", "run.js"), | 234 (path.join("out", "x64.release", "d8"), "--flag", "run.js"), |
231 (path.join("out", "x64.release", "d8"), "--flag", "run.js"), | 235 (path.join("out", "x64.release", "d8"), "--flag", "run.js"), |
232 (path.join("out", "x64.release", "d8"), "--flag", "run.js"), | 236 (path.join("out", "x64.release", "d8"), "--flag", "run.js"), |
233 (path.join("out", "x64.release", "d8"), "--flag", "--flag2", "run.js")) | 237 (path.join("out", "x64.release", "d8"), "--flag", "--flag2", "run.js")) |
234 | 238 |
| 239 def testOneRunStdDevRegExp(self): |
| 240 test_input = dict(V8_JSON) |
| 241 test_input["stddev_regexp"] = "^%s\-stddev: (.+)$" |
| 242 self._WriteTestInput(test_input) |
| 243 self._MockCommand(["."], ["Richards: 1.234\nRichards-stddev: 0.23\n" |
| 244 "DeltaBlue: 10657567\nDeltaBlue-stddev: 106\n"]) |
| 245 self.assertEquals(0, self._CallMain()) |
| 246 self._VerifyResults("test", "score", [ |
| 247 {"name": "Richards", "results": ["1.234"], "stddev": "0.23"}, |
| 248 {"name": "DeltaBlue", "results": ["10657567"], "stddev": "106"}, |
| 249 ]) |
| 250 self._VerifyErrors([]) |
| 251 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") |
| 252 |
| 253 def testTwoRunsStdDevRegExp(self): |
| 254 test_input = dict(V8_JSON) |
| 255 test_input["stddev_regexp"] = "^%s\-stddev: (.+)$" |
| 256 test_input["run_count"] = 2 |
| 257 self._WriteTestInput(test_input) |
| 258 self._MockCommand(["."], ["Richards: 3\nRichards-stddev: 0.7\n" |
| 259 "DeltaBlue: 6\nDeltaBlue-boom: 0.9\n", |
| 260 "Richards: 2\nRichards-stddev: 0.5\n" |
| 261 "DeltaBlue: 5\nDeltaBlue-stddev: 0.8\n"]) |
| 262 self.assertEquals(1, self._CallMain()) |
| 263 self._VerifyResults("test", "score", [ |
| 264 {"name": "Richards", "results": ["2", "3"], "stddev": "0.7"}, |
| 265 {"name": "DeltaBlue", "results": ["5", "6"], "stddev": "0.8"}, |
| 266 ]) |
| 267 self._VerifyErrors( |
| 268 ["Benchmark Richards should only run once since a stddev is provided " |
| 269 "by the benchmark.", |
| 270 "Benchmark DeltaBlue should only run once since a stddev is provided " |
| 271 "by the benchmark.", |
| 272 "Regexp \"^DeltaBlue\-stddev: (.+)$\" didn't match for benchmark " |
| 273 "DeltaBlue."]) |
| 274 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") |
| 275 |
235 def testBuildbot(self): | 276 def testBuildbot(self): |
236 self._WriteTestInput(V8_JSON) | 277 self._WriteTestInput(V8_JSON) |
237 self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567\n"]) | 278 self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567\n"]) |
238 self.assertEquals(0, self._CallMain("--buildbot")) | 279 self.assertEquals(0, self._CallMain("--buildbot")) |
239 self._VerifyResults("test", "score", [ | 280 self._VerifyResults("test", "score", [ |
240 {"name": "Richards", "results": ["1.234"]}, | 281 {"name": "Richards", "results": ["1.234"], "stddev": ""}, |
241 {"name": "DeltaBlue", "results": ["10657567"]}, | 282 {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""}, |
242 ]) | 283 ]) |
243 self._VerifyErrors([]) | 284 self._VerifyErrors([]) |
244 self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js") | 285 self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js") |
245 | 286 |
246 def testRegexpNoMatch(self): | 287 def testRegexpNoMatch(self): |
247 self._WriteTestInput(V8_JSON) | 288 self._WriteTestInput(V8_JSON) |
248 self._MockCommand(["."], ["x\nRichaards: 1.234\nDeltaBlue: 10657567\ny\n"]) | 289 self._MockCommand(["."], ["x\nRichaards: 1.234\nDeltaBlue: 10657567\ny\n"]) |
249 self.assertEquals(1, self._CallMain()) | 290 self.assertEquals(1, self._CallMain()) |
250 self._VerifyResults("test", "score", [ | 291 self._VerifyResults("test", "score", [ |
251 {"name": "Richards", "results": []}, | 292 {"name": "Richards", "results": [], "stddev": ""}, |
252 {"name": "DeltaBlue", "results": ["10657567"]}, | 293 {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""}, |
253 ]) | 294 ]) |
254 self._VerifyErrors( | 295 self._VerifyErrors( |
255 ["Regexp \"^Richards: (.+)$\" didn't match for benchmark Richards."]) | 296 ["Regexp \"^Richards: (.+)$\" didn't match for benchmark Richards."]) |
256 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") | 297 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") |
OLD | NEW |