OLD | NEW |
| (Empty) |
1 #!/usr/bin/env python | |
2 # Copyright 2014 the V8 project authors. All rights reserved. | |
3 # Use of this source code is governed by a BSD-style license that can be | |
4 # found in the LICENSE file. | |
5 | |
6 """ | |
7 Performance runner for d8. | |
8 | |
9 Call e.g. with tools/run-benchmarks.py --arch ia32 some_suite.json | |
10 | |
11 The suite json format is expected to be: | |
12 { | |
13 "path": <relative path chunks to benchmark resources and main file>, | |
14 "name": <optional suite name, file name is default>, | |
15 "archs": [<architecture name for which this suite is run>, ...], | |
16 "binary": <name of binary to run, default "d8">, | |
17 "flags": [<flag to d8>, ...], | |
18 "run_count": <how often will this suite run (optional)>, | |
19 "run_count_XXX": <how often will this suite run for arch XXX (optional)>, | |
20 "resources": [<js file to be loaded before main>, ...] | |
21 "main": <main js benchmark runner file>, | |
22 "results_regexp": <optional regexp>, | |
23 "results_processor": <optional python results processor script>, | |
24 "units": <the unit specification for the performance dashboard>, | |
25 "benchmarks": [ | |
26 { | |
27 "name": <name of the benchmark>, | |
28 "results_regexp": <optional more specific regexp>, | |
29 "results_processor": <optional python results processor script>, | |
30 "units": <the unit specification for the performance dashboard>, | |
31 }, ... | |
32 ] | |
33 } | |
34 | |
35 The benchmarks field can also nest other suites in arbitrary depth. A suite | |
36 with a "main" file is a leaf suite that can contain one more level of | |
37 benchmarks. | |
38 | |
39 A suite's results_regexp is expected to have one string place holder | |
40 "%s" for the benchmark name. A benchmark's results_regexp overwrites suite | |
41 defaults. | |
42 | |
43 A suite's results_processor may point to an optional python script. If | |
44 specified, it is called after running the benchmarks like this (with a path | |
45 relatve to the suite level's path): | |
46 <results_processor file> <same flags as for d8> <suite level name> <output> | |
47 | |
48 The <output> is a temporary file containing d8 output. The results_regexp will | |
49 be applied to the output of this script. | |
50 | |
51 A suite without "benchmarks" is considered a benchmark itself. | |
52 | |
53 Full example (suite with one runner): | |
54 { | |
55 "path": ["."], | |
56 "flags": ["--expose-gc"], | |
57 "archs": ["ia32", "x64"], | |
58 "run_count": 5, | |
59 "run_count_ia32": 3, | |
60 "main": "run.js", | |
61 "results_regexp": "^%s: (.+)$", | |
62 "units": "score", | |
63 "benchmarks": [ | |
64 {"name": "Richards"}, | |
65 {"name": "DeltaBlue"}, | |
66 {"name": "NavierStokes", | |
67 "results_regexp": "^NavierStokes: (.+)$"} | |
68 ] | |
69 } | |
70 | |
71 Full example (suite with several runners): | |
72 { | |
73 "path": ["."], | |
74 "flags": ["--expose-gc"], | |
75 "archs": ["ia32", "x64"], | |
76 "run_count": 5, | |
77 "units": "score", | |
78 "benchmarks": [ | |
79 {"name": "Richards", | |
80 "path": ["richards"], | |
81 "main": "run.js", | |
82 "run_count": 3, | |
83 "results_regexp": "^Richards: (.+)$"}, | |
84 {"name": "NavierStokes", | |
85 "path": ["navier_stokes"], | |
86 "main": "run.js", | |
87 "results_regexp": "^NavierStokes: (.+)$"} | |
88 ] | |
89 } | |
90 | |
91 Path pieces are concatenated. D8 is always run with the suite's path as cwd. | |
92 """ | |
93 | |
94 import json | |
95 import math | |
96 import optparse | |
97 import os | |
98 import re | |
99 import sys | |
100 | |
101 from testrunner.local import commands | |
102 from testrunner.local import utils | |
103 | |
104 ARCH_GUESS = utils.DefaultArch() | |
105 SUPPORTED_ARCHS = ["android_arm", | |
106 "android_arm64", | |
107 "android_ia32", | |
108 "arm", | |
109 "ia32", | |
110 "mips", | |
111 "mipsel", | |
112 "nacl_ia32", | |
113 "nacl_x64", | |
114 "x64", | |
115 "arm64"] | |
116 | |
117 GENERIC_RESULTS_RE = re.compile( | |
118 r"^Trace\(([^\)]+)\), Result\(([^\)]+)\), StdDev\(([^\)]+)\)$") | |
119 | |
120 | |
121 def GeometricMean(values): | |
122 """Returns the geometric mean of a list of values. | |
123 | |
124 The mean is calculated using log to avoid overflow. | |
125 """ | |
126 values = map(float, values) | |
127 return str(math.exp(sum(map(math.log, values)) / len(values))) | |
128 | |
129 | |
130 class Results(object): | |
131 """Place holder for result traces.""" | |
132 def __init__(self, traces=None, errors=None): | |
133 self.traces = traces or [] | |
134 self.errors = errors or [] | |
135 | |
136 def ToDict(self): | |
137 return {"traces": self.traces, "errors": self.errors} | |
138 | |
139 def WriteToFile(self, file_name): | |
140 with open(file_name, "w") as f: | |
141 f.write(json.dumps(self.ToDict())) | |
142 | |
143 def __add__(self, other): | |
144 self.traces += other.traces | |
145 self.errors += other.errors | |
146 return self | |
147 | |
148 def __str__(self): # pragma: no cover | |
149 return str(self.ToDict()) | |
150 | |
151 | |
152 class Node(object): | |
153 """Represents a node in the benchmark suite tree structure.""" | |
154 def __init__(self, *args): | |
155 self._children = [] | |
156 | |
157 def AppendChild(self, child): | |
158 self._children.append(child) | |
159 | |
160 | |
161 class DefaultSentinel(Node): | |
162 """Fake parent node with all default values.""" | |
163 def __init__(self): | |
164 super(DefaultSentinel, self).__init__() | |
165 self.binary = "d8" | |
166 self.run_count = 10 | |
167 self.path = [] | |
168 self.graphs = [] | |
169 self.flags = [] | |
170 self.resources = [] | |
171 self.results_regexp = None | |
172 self.stddev_regexp = None | |
173 self.units = "score" | |
174 self.total = False | |
175 | |
176 | |
177 class Graph(Node): | |
178 """Represents a benchmark suite definition. | |
179 | |
180 Can either be a leaf or an inner node that provides default values. | |
181 """ | |
182 def __init__(self, suite, parent, arch): | |
183 super(Graph, self).__init__() | |
184 self._suite = suite | |
185 | |
186 assert isinstance(suite.get("path", []), list) | |
187 assert isinstance(suite["name"], basestring) | |
188 assert isinstance(suite.get("flags", []), list) | |
189 assert isinstance(suite.get("resources", []), list) | |
190 | |
191 # Accumulated values. | |
192 self.path = parent.path[:] + suite.get("path", []) | |
193 self.graphs = parent.graphs[:] + [suite["name"]] | |
194 self.flags = parent.flags[:] + suite.get("flags", []) | |
195 self.resources = parent.resources[:] + suite.get("resources", []) | |
196 | |
197 # Descrete values (with parent defaults). | |
198 self.binary = suite.get("binary", parent.binary) | |
199 self.run_count = suite.get("run_count", parent.run_count) | |
200 self.run_count = suite.get("run_count_%s" % arch, self.run_count) | |
201 self.units = suite.get("units", parent.units) | |
202 self.total = suite.get("total", parent.total) | |
203 | |
204 # A regular expression for results. If the parent graph provides a | |
205 # regexp and the current suite has none, a string place holder for the | |
206 # suite name is expected. | |
207 # TODO(machenbach): Currently that makes only sense for the leaf level. | |
208 # Multiple place holders for multiple levels are not supported. | |
209 if parent.results_regexp: | |
210 regexp_default = parent.results_regexp % re.escape(suite["name"]) | |
211 else: | |
212 regexp_default = None | |
213 self.results_regexp = suite.get("results_regexp", regexp_default) | |
214 | |
215 # A similar regular expression for the standard deviation (optional). | |
216 if parent.stddev_regexp: | |
217 stddev_default = parent.stddev_regexp % re.escape(suite["name"]) | |
218 else: | |
219 stddev_default = None | |
220 self.stddev_regexp = suite.get("stddev_regexp", stddev_default) | |
221 | |
222 | |
223 class Trace(Graph): | |
224 """Represents a leaf in the benchmark suite tree structure. | |
225 | |
226 Handles collection of measurements. | |
227 """ | |
228 def __init__(self, suite, parent, arch): | |
229 super(Trace, self).__init__(suite, parent, arch) | |
230 assert self.results_regexp | |
231 self.results = [] | |
232 self.errors = [] | |
233 self.stddev = "" | |
234 | |
235 def ConsumeOutput(self, stdout): | |
236 try: | |
237 self.results.append( | |
238 re.search(self.results_regexp, stdout, re.M).group(1)) | |
239 except: | |
240 self.errors.append("Regexp \"%s\" didn't match for benchmark %s." | |
241 % (self.results_regexp, self.graphs[-1])) | |
242 | |
243 try: | |
244 if self.stddev_regexp and self.stddev: | |
245 self.errors.append("Benchmark %s should only run once since a stddev " | |
246 "is provided by the benchmark." % self.graphs[-1]) | |
247 if self.stddev_regexp: | |
248 self.stddev = re.search(self.stddev_regexp, stdout, re.M).group(1) | |
249 except: | |
250 self.errors.append("Regexp \"%s\" didn't match for benchmark %s." | |
251 % (self.stddev_regexp, self.graphs[-1])) | |
252 | |
253 def GetResults(self): | |
254 return Results([{ | |
255 "graphs": self.graphs, | |
256 "units": self.units, | |
257 "results": self.results, | |
258 "stddev": self.stddev, | |
259 }], self.errors) | |
260 | |
261 | |
262 class Runnable(Graph): | |
263 """Represents a runnable benchmark suite definition (i.e. has a main file). | |
264 """ | |
265 @property | |
266 def main(self): | |
267 return self._suite.get("main", "") | |
268 | |
269 def ChangeCWD(self, suite_path): | |
270 """Changes the cwd to to path defined in the current graph. | |
271 | |
272 The benchmarks are supposed to be relative to the suite configuration. | |
273 """ | |
274 suite_dir = os.path.abspath(os.path.dirname(suite_path)) | |
275 bench_dir = os.path.normpath(os.path.join(*self.path)) | |
276 os.chdir(os.path.join(suite_dir, bench_dir)) | |
277 | |
278 def GetCommand(self, shell_dir): | |
279 # TODO(machenbach): This requires +.exe if run on windows. | |
280 return ( | |
281 [os.path.join(shell_dir, self.binary)] + | |
282 self.flags + | |
283 self.resources + | |
284 [self.main] | |
285 ) | |
286 | |
287 def Run(self, runner): | |
288 """Iterates over several runs and handles the output for all traces.""" | |
289 for stdout in runner(): | |
290 for trace in self._children: | |
291 trace.ConsumeOutput(stdout) | |
292 res = reduce(lambda r, t: r + t.GetResults(), self._children, Results()) | |
293 | |
294 if not res.traces or not self.total: | |
295 return res | |
296 | |
297 # Assume all traces have the same structure. | |
298 if len(set(map(lambda t: len(t["results"]), res.traces))) != 1: | |
299 res.errors.append("Not all traces have the same number of results.") | |
300 return res | |
301 | |
302 # Calculate the geometric means for all traces. Above we made sure that | |
303 # there is at least one trace and that the number of results is the same | |
304 # for each trace. | |
305 n_results = len(res.traces[0]["results"]) | |
306 total_results = [GeometricMean(t["results"][i] for t in res.traces) | |
307 for i in range(0, n_results)] | |
308 res.traces.append({ | |
309 "graphs": self.graphs + ["Total"], | |
310 "units": res.traces[0]["units"], | |
311 "results": total_results, | |
312 "stddev": "", | |
313 }) | |
314 return res | |
315 | |
316 class RunnableTrace(Trace, Runnable): | |
317 """Represents a runnable benchmark suite definition that is a leaf.""" | |
318 def __init__(self, suite, parent, arch): | |
319 super(RunnableTrace, self).__init__(suite, parent, arch) | |
320 | |
321 def Run(self, runner): | |
322 """Iterates over several runs and handles the output.""" | |
323 for stdout in runner(): | |
324 self.ConsumeOutput(stdout) | |
325 return self.GetResults() | |
326 | |
327 | |
328 class RunnableGeneric(Runnable): | |
329 """Represents a runnable benchmark suite definition with generic traces.""" | |
330 def __init__(self, suite, parent, arch): | |
331 super(RunnableGeneric, self).__init__(suite, parent, arch) | |
332 | |
333 def Run(self, runner): | |
334 """Iterates over several runs and handles the output.""" | |
335 traces = {} | |
336 for stdout in runner(): | |
337 for line in stdout.strip().splitlines(): | |
338 match = GENERIC_RESULTS_RE.match(line) | |
339 if match: | |
340 trace = match.group(1) | |
341 result = match.group(2) | |
342 stddev = match.group(3) | |
343 trace_result = traces.setdefault(trace, Results([{ | |
344 "graphs": self.graphs + [trace], | |
345 "units": self.units, | |
346 "results": [], | |
347 "stddev": "", | |
348 }], [])) | |
349 trace_result.traces[0]["results"].append(result) | |
350 trace_result.traces[0]["stddev"] = stddev | |
351 | |
352 return reduce(lambda r, t: r + t, traces.itervalues(), Results()) | |
353 | |
354 | |
355 def MakeGraph(suite, arch, parent): | |
356 """Factory method for making graph objects.""" | |
357 if isinstance(parent, Runnable): | |
358 # Below a runnable can only be traces. | |
359 return Trace(suite, parent, arch) | |
360 elif suite.get("main"): | |
361 # A main file makes this graph runnable. | |
362 if suite.get("benchmarks"): | |
363 # This graph has subbenchmarks (traces). | |
364 return Runnable(suite, parent, arch) | |
365 else: | |
366 # This graph has no subbenchmarks, it's a leaf. | |
367 return RunnableTrace(suite, parent, arch) | |
368 elif suite.get("generic"): | |
369 # This is a generic suite definition. It is either a runnable executable | |
370 # or has a main js file. | |
371 return RunnableGeneric(suite, parent, arch) | |
372 elif suite.get("benchmarks"): | |
373 # This is neither a leaf nor a runnable. | |
374 return Graph(suite, parent, arch) | |
375 else: # pragma: no cover | |
376 raise Exception("Invalid benchmark suite configuration.") | |
377 | |
378 | |
379 def BuildGraphs(suite, arch, parent=None): | |
380 """Builds a tree structure of graph objects that corresponds to the suite | |
381 configuration. | |
382 """ | |
383 parent = parent or DefaultSentinel() | |
384 | |
385 # TODO(machenbach): Implement notion of cpu type? | |
386 if arch not in suite.get("archs", ["ia32", "x64"]): | |
387 return None | |
388 | |
389 graph = MakeGraph(suite, arch, parent) | |
390 for subsuite in suite.get("benchmarks", []): | |
391 BuildGraphs(subsuite, arch, graph) | |
392 parent.AppendChild(graph) | |
393 return graph | |
394 | |
395 | |
396 def FlattenRunnables(node): | |
397 """Generator that traverses the tree structure and iterates over all | |
398 runnables. | |
399 """ | |
400 if isinstance(node, Runnable): | |
401 yield node | |
402 elif isinstance(node, Node): | |
403 for child in node._children: | |
404 for result in FlattenRunnables(child): | |
405 yield result | |
406 else: # pragma: no cover | |
407 raise Exception("Invalid benchmark suite configuration.") | |
408 | |
409 | |
410 # TODO: Implement results_processor. | |
411 def Main(args): | |
412 parser = optparse.OptionParser() | |
413 parser.add_option("--arch", | |
414 help=("The architecture to run tests for, " | |
415 "'auto' or 'native' for auto-detect"), | |
416 default="x64") | |
417 parser.add_option("--buildbot", | |
418 help="Adapt to path structure used on buildbots", | |
419 default=False, action="store_true") | |
420 parser.add_option("--json-test-results", | |
421 help="Path to a file for storing json results.") | |
422 parser.add_option("--outdir", help="Base directory with compile output", | |
423 default="out") | |
424 (options, args) = parser.parse_args(args) | |
425 | |
426 if len(args) == 0: # pragma: no cover | |
427 parser.print_help() | |
428 return 1 | |
429 | |
430 if options.arch in ["auto", "native"]: # pragma: no cover | |
431 options.arch = ARCH_GUESS | |
432 | |
433 if not options.arch in SUPPORTED_ARCHS: # pragma: no cover | |
434 print "Unknown architecture %s" % options.arch | |
435 return 1 | |
436 | |
437 workspace = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) | |
438 | |
439 if options.buildbot: | |
440 shell_dir = os.path.join(workspace, options.outdir, "Release") | |
441 else: | |
442 shell_dir = os.path.join(workspace, options.outdir, | |
443 "%s.release" % options.arch) | |
444 | |
445 results = Results() | |
446 for path in args: | |
447 path = os.path.abspath(path) | |
448 | |
449 if not os.path.exists(path): # pragma: no cover | |
450 results.errors.append("Benchmark file %s does not exist." % path) | |
451 continue | |
452 | |
453 with open(path) as f: | |
454 suite = json.loads(f.read()) | |
455 | |
456 # If no name is given, default to the file name without .json. | |
457 suite.setdefault("name", os.path.splitext(os.path.basename(path))[0]) | |
458 | |
459 for runnable in FlattenRunnables(BuildGraphs(suite, options.arch)): | |
460 print ">>> Running suite: %s" % "/".join(runnable.graphs) | |
461 runnable.ChangeCWD(path) | |
462 | |
463 def Runner(): | |
464 """Output generator that reruns several times.""" | |
465 for i in xrange(0, max(1, runnable.run_count)): | |
466 # TODO(machenbach): Make timeout configurable in the suite definition. | |
467 # Allow timeout per arch like with run_count per arch. | |
468 output = commands.Execute(runnable.GetCommand(shell_dir), timeout=60) | |
469 print ">>> Stdout (#%d):" % (i + 1) | |
470 print output.stdout | |
471 if output.stderr: # pragma: no cover | |
472 # Print stderr for debugging. | |
473 print ">>> Stderr (#%d):" % (i + 1) | |
474 print output.stderr | |
475 yield output.stdout | |
476 | |
477 # Let runnable iterate over all runs and handle output. | |
478 results += runnable.Run(Runner) | |
479 | |
480 if options.json_test_results: | |
481 results.WriteToFile(options.json_test_results) | |
482 else: # pragma: no cover | |
483 print results | |
484 | |
485 return min(1, len(results.errors)) | |
486 | |
487 if __name__ == "__main__": # pragma: no cover | |
488 sys.exit(Main(sys.argv[1:])) | |
OLD | NEW |