OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2014 the V8 project authors. All rights reserved. | 2 # Copyright 2014 the V8 project authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """ | 6 """ |
7 Performance runner for d8. | 7 Performance runner for d8. |
8 | 8 |
9 Call e.g. with tools/run-benchmarks.py --arch ia32 some_suite.json | 9 Call e.g. with tools/run-benchmarks.py --arch ia32 some_suite.json |
10 | 10 |
(...skipping 74 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
85 "path": ["navier_stokes"], | 85 "path": ["navier_stokes"], |
86 "main": "run.js", | 86 "main": "run.js", |
87 "results_regexp": "^NavierStokes: (.+)$"} | 87 "results_regexp": "^NavierStokes: (.+)$"} |
88 ] | 88 ] |
89 } | 89 } |
90 | 90 |
91 Path pieces are concatenated. D8 is always run with the suite's path as cwd. | 91 Path pieces are concatenated. D8 is always run with the suite's path as cwd. |
92 """ | 92 """ |
93 | 93 |
94 import json | 94 import json |
| 95 import math |
95 import optparse | 96 import optparse |
96 import os | 97 import os |
97 import re | 98 import re |
98 import sys | 99 import sys |
99 | 100 |
100 from testrunner.local import commands | 101 from testrunner.local import commands |
101 from testrunner.local import utils | 102 from testrunner.local import utils |
102 | 103 |
103 ARCH_GUESS = utils.DefaultArch() | 104 ARCH_GUESS = utils.DefaultArch() |
104 SUPPORTED_ARCHS = ["android_arm", | 105 SUPPORTED_ARCHS = ["android_arm", |
105 "android_arm64", | 106 "android_arm64", |
106 "android_ia32", | 107 "android_ia32", |
107 "arm", | 108 "arm", |
108 "ia32", | 109 "ia32", |
109 "mips", | 110 "mips", |
110 "mipsel", | 111 "mipsel", |
111 "nacl_ia32", | 112 "nacl_ia32", |
112 "nacl_x64", | 113 "nacl_x64", |
113 "x64", | 114 "x64", |
114 "arm64"] | 115 "arm64"] |
115 | 116 |
116 GENERIC_RESULTS_RE = re.compile( | 117 GENERIC_RESULTS_RE = re.compile( |
117 r"^Trace\(([^\)]+)\), Result\(([^\)]+)\), StdDev\(([^\)]+)\)$") | 118 r"^Trace\(([^\)]+)\), Result\(([^\)]+)\), StdDev\(([^\)]+)\)$") |
118 | 119 |
| 120 |
| 121 def GeometricMean(values): |
| 122 """Returns the geometric mean of a list of values. |
| 123 |
| 124 The mean is calculated using log to avoid overflow. |
| 125 """ |
| 126 values = map(float, values) |
| 127 return str(math.exp(sum(map(math.log, values)) / len(values))) |
| 128 |
| 129 |
119 class Results(object): | 130 class Results(object): |
120 """Place holder for result traces.""" | 131 """Place holder for result traces.""" |
121 def __init__(self, traces=None, errors=None): | 132 def __init__(self, traces=None, errors=None): |
122 self.traces = traces or [] | 133 self.traces = traces or [] |
123 self.errors = errors or [] | 134 self.errors = errors or [] |
124 | 135 |
125 def ToDict(self): | 136 def ToDict(self): |
126 return {"traces": self.traces, "errors": self.errors} | 137 return {"traces": self.traces, "errors": self.errors} |
127 | 138 |
128 def WriteToFile(self, file_name): | 139 def WriteToFile(self, file_name): |
(...skipping 24 matching lines...) Expand all Loading... |
153 super(DefaultSentinel, self).__init__() | 164 super(DefaultSentinel, self).__init__() |
154 self.binary = "d8" | 165 self.binary = "d8" |
155 self.run_count = 10 | 166 self.run_count = 10 |
156 self.path = [] | 167 self.path = [] |
157 self.graphs = [] | 168 self.graphs = [] |
158 self.flags = [] | 169 self.flags = [] |
159 self.resources = [] | 170 self.resources = [] |
160 self.results_regexp = None | 171 self.results_regexp = None |
161 self.stddev_regexp = None | 172 self.stddev_regexp = None |
162 self.units = "score" | 173 self.units = "score" |
| 174 self.total = False |
163 | 175 |
164 | 176 |
165 class Graph(Node): | 177 class Graph(Node): |
166 """Represents a benchmark suite definition. | 178 """Represents a benchmark suite definition. |
167 | 179 |
168 Can either be a leaf or an inner node that provides default values. | 180 Can either be a leaf or an inner node that provides default values. |
169 """ | 181 """ |
170 def __init__(self, suite, parent, arch): | 182 def __init__(self, suite, parent, arch): |
171 super(Graph, self).__init__() | 183 super(Graph, self).__init__() |
172 self._suite = suite | 184 self._suite = suite |
173 | 185 |
174 assert isinstance(suite.get("path", []), list) | 186 assert isinstance(suite.get("path", []), list) |
175 assert isinstance(suite["name"], basestring) | 187 assert isinstance(suite["name"], basestring) |
176 assert isinstance(suite.get("flags", []), list) | 188 assert isinstance(suite.get("flags", []), list) |
177 assert isinstance(suite.get("resources", []), list) | 189 assert isinstance(suite.get("resources", []), list) |
178 | 190 |
179 # Accumulated values. | 191 # Accumulated values. |
180 self.path = parent.path[:] + suite.get("path", []) | 192 self.path = parent.path[:] + suite.get("path", []) |
181 self.graphs = parent.graphs[:] + [suite["name"]] | 193 self.graphs = parent.graphs[:] + [suite["name"]] |
182 self.flags = parent.flags[:] + suite.get("flags", []) | 194 self.flags = parent.flags[:] + suite.get("flags", []) |
183 self.resources = parent.resources[:] + suite.get("resources", []) | 195 self.resources = parent.resources[:] + suite.get("resources", []) |
184 | 196 |
185 # Descrete values (with parent defaults). | 197 # Descrete values (with parent defaults). |
186 self.binary = suite.get("binary", parent.binary) | 198 self.binary = suite.get("binary", parent.binary) |
187 self.run_count = suite.get("run_count", parent.run_count) | 199 self.run_count = suite.get("run_count", parent.run_count) |
188 self.run_count = suite.get("run_count_%s" % arch, self.run_count) | 200 self.run_count = suite.get("run_count_%s" % arch, self.run_count) |
189 self.units = suite.get("units", parent.units) | 201 self.units = suite.get("units", parent.units) |
| 202 self.total = suite.get("total", parent.total) |
190 | 203 |
191 # A regular expression for results. If the parent graph provides a | 204 # A regular expression for results. If the parent graph provides a |
192 # regexp and the current suite has none, a string place holder for the | 205 # regexp and the current suite has none, a string place holder for the |
193 # suite name is expected. | 206 # suite name is expected. |
194 # TODO(machenbach): Currently that makes only sense for the leaf level. | 207 # TODO(machenbach): Currently that makes only sense for the leaf level. |
195 # Multiple place holders for multiple levels are not supported. | 208 # Multiple place holders for multiple levels are not supported. |
196 if parent.results_regexp: | 209 if parent.results_regexp: |
197 regexp_default = parent.results_regexp % re.escape(suite["name"]) | 210 regexp_default = parent.results_regexp % re.escape(suite["name"]) |
198 else: | 211 else: |
199 regexp_default = None | 212 regexp_default = None |
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
269 self.flags + | 282 self.flags + |
270 self.resources + | 283 self.resources + |
271 [self.main] | 284 [self.main] |
272 ) | 285 ) |
273 | 286 |
274 def Run(self, runner): | 287 def Run(self, runner): |
275 """Iterates over several runs and handles the output for all traces.""" | 288 """Iterates over several runs and handles the output for all traces.""" |
276 for stdout in runner(): | 289 for stdout in runner(): |
277 for trace in self._children: | 290 for trace in self._children: |
278 trace.ConsumeOutput(stdout) | 291 trace.ConsumeOutput(stdout) |
279 return reduce(lambda r, t: r + t.GetResults(), self._children, Results()) | 292 res = reduce(lambda r, t: r + t.GetResults(), self._children, Results()) |
280 | 293 |
| 294 if not res.traces or not self.total: |
| 295 return res |
| 296 |
| 297 # Assume all traces have the same structure. |
| 298 if len(set(map(lambda t: len(t["results"]), res.traces))) != 1: |
| 299 res.errors.append("Not all traces have the same number of results.") |
| 300 return res |
| 301 |
| 302 # Calculate the geometric means for all traces. Above we made sure that |
| 303 # there is at least one trace and that the number of results is the same |
| 304 # for each trace. |
| 305 n_results = len(res.traces[0]["results"]) |
| 306 total_results = [GeometricMean(t["results"][i] for t in res.traces) |
| 307 for i in range(0, n_results)] |
| 308 res.traces.append({ |
| 309 "graphs": self.graphs + ["Total"], |
| 310 "units": res.traces[0]["units"], |
| 311 "results": total_results, |
| 312 "stddev": "", |
| 313 }) |
| 314 return res |
281 | 315 |
282 class RunnableTrace(Trace, Runnable): | 316 class RunnableTrace(Trace, Runnable): |
283 """Represents a runnable benchmark suite definition that is a leaf.""" | 317 """Represents a runnable benchmark suite definition that is a leaf.""" |
284 def __init__(self, suite, parent, arch): | 318 def __init__(self, suite, parent, arch): |
285 super(RunnableTrace, self).__init__(suite, parent, arch) | 319 super(RunnableTrace, self).__init__(suite, parent, arch) |
286 | 320 |
287 def Run(self, runner): | 321 def Run(self, runner): |
288 """Iterates over several runs and handles the output.""" | 322 """Iterates over several runs and handles the output.""" |
289 for stdout in runner(): | 323 for stdout in runner(): |
290 self.ConsumeOutput(stdout) | 324 self.ConsumeOutput(stdout) |
(...skipping 154 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
445 | 479 |
446 if options.json_test_results: | 480 if options.json_test_results: |
447 results.WriteToFile(options.json_test_results) | 481 results.WriteToFile(options.json_test_results) |
448 else: # pragma: no cover | 482 else: # pragma: no cover |
449 print results | 483 print results |
450 | 484 |
451 return min(1, len(results.errors)) | 485 return min(1, len(results.errors)) |
452 | 486 |
453 if __name__ == "__main__": # pragma: no cover | 487 if __name__ == "__main__": # pragma: no cover |
454 sys.exit(Main(sys.argv[1:])) | 488 sys.exit(Main(sys.argv[1:])) |
OLD | NEW |