OLD | NEW |
---|---|
(Empty) | |
1 #!/usr/bin/env python | |
2 # | |
3 # Copyright 2012 the V8 project authors. All rights reserved. | |
4 # Redistribution and use in source and binary forms, with or without | |
5 # modification, are permitted provided that the following conditions are | |
6 # met: | |
7 # | |
8 # * Redistributions of source code must retain the above copyright | |
9 # notice, this list of conditions and the following disclaimer. | |
10 # * Redistributions in binary form must reproduce the above | |
11 # copyright notice, this list of conditions and the following | |
12 # disclaimer in the documentation and/or other materials provided | |
13 # with the distribution. | |
14 # * Neither the name of Google Inc. nor the names of its | |
15 # contributors may be used to endorse or promote products derived | |
16 # from this software without specific prior written permission. | |
17 # | |
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
29 | |
30 | |
31 import multiprocessing | |
32 import optparse | |
33 import os | |
34 from os.path import join | |
35 import subprocess | |
36 import sys | |
37 import time | |
38 | |
39 from testrunner.local import execution | |
40 from testrunner.local import progress | |
41 from testrunner.local import testsuite | |
42 from testrunner.local import utils | |
43 from testrunner.local import verbose | |
44 from testrunner.network import network_execution | |
45 from testrunner.objects import context | |
46 | |
47 | |
48 ARCH_GUESS = utils.DefaultArch() | |
49 DEFAULT_TESTS = ["mjsunit", "cctest", "message", "preparser"] | |
50 TIMEOUT_DEFAULT = 60 | |
51 TIMEOUT_SCALEFACTOR = {"debug" : 4, | |
52 "release" : 1 } | |
53 | |
54 # Use this to run several variants of the tests. | |
55 VARIANT_FLAGS = [[], | |
56 ["--stress-opt", "--always-opt"], | |
57 ["--nocrankshaft"]] | |
58 MODE_FLAGS = { | |
59 "debug" : ["--nobreak-on-abort", "--enable-slow-asserts", | |
60 "--debug-code", "--verify-heap"], | |
61 "release" : ["--nobreak-on-abort"]} | |
62 | |
63 | |
64 def BuildOptions(): | |
65 result = optparse.OptionParser() | |
66 result.add_option("--arch", | |
67 help=("The architecture to run tests for, " | |
68 "'auto' or 'native' for auto-detect"), | |
69 default="ia32,x64,arm") | |
70 result.add_option("--arch-and-mode", | |
71 help="Architecture and mode in the format 'arch.mode'", | |
72 default=None) | |
73 result.add_option("--buildbot", | |
74 help="Adapt to path structure used on buildbots", | |
75 default=False, action="store_true") | |
76 result.add_option("--cat", help="Print the source of the tests", | |
77 default=False, action="store_true") | |
78 result.add_option("--command-prefix", | |
79 help="Prepended to each shell command used to run a test", | |
80 default="") | |
81 result.add_option("--download-data", help="Download missing test suite data", | |
82 default=False, action="store_true") | |
83 result.add_option("--extra-flags", | |
84 help="Additional flags to pass to each test command", | |
85 default="") | |
86 result.add_option("--isolates", help="Whether to test isolates", | |
87 default=False, action="store_true") | |
88 result.add_option("-j", help="The number of parallel tasks to run", | |
89 default=0, type="int") | |
90 result.add_option("-m", "--mode", | |
91 help="The test modes in which to run (comma-separated)", | |
92 default="release,debug") | |
93 result.add_option("--no-network", "--nonetwork", | |
94 help="Don't distribute tests on the network", | |
95 default=(utils.GuessOS() != "linux"), | |
96 dest="no_network", action="store_true") | |
97 result.add_option("--no-presubmit", "--nopresubmit", | |
98 help='Skip presubmit checks', | |
99 default=False, dest="no_presubmit", action="store_true") | |
100 result.add_option("--no-stress", "--nostress", | |
101 help="Don't run crankshaft --always-opt --stress-op test", | |
102 default=False, dest="no_stress", action="store_true") | |
103 result.add_option("--outdir", help="Base directory with compile output", | |
104 default="out") | |
105 result.add_option("-p", "--progress", | |
106 help=("The style of progress indicator" | |
107 " (verbose, dots, color, mono)"), | |
108 choices=progress.PROGRESS_INDICATORS.keys(), default="mono") | |
109 result.add_option("--report", help="Print a summary of the tests to be run", | |
110 default=False, action="store_true") | |
111 result.add_option("--shard-count", | |
112 help="Split testsuites into this number of shards", | |
113 default=1, type="int") | |
114 result.add_option("--shard-run", | |
115 help="Run this shard from the split up tests.", | |
116 default=1, type="int") | |
117 result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="") | |
118 result.add_option("--shell-dir", help="Directory containing executables", | |
119 default="") | |
120 result.add_option("--stress-only", | |
121 help="Only run tests with --always-opt --stress-opt", | |
122 default=False, action="store_true") | |
123 result.add_option("--time", help="Print timing information after running", | |
124 default=False, action="store_true") | |
125 result.add_option("-t", "--timeout", help="Timeout in seconds", | |
126 default= -1, type="int") | |
127 result.add_option("-v", "--verbose", help="Verbose output", | |
128 default=False, action="store_true") | |
129 result.add_option("--valgrind", help="Run tests through valgrind", | |
130 default=False, action="store_true") | |
131 result.add_option("--warn-unused", help="Report unused rules", | |
132 default=False, action="store_true") | |
133 return result | |
134 | |
135 | |
136 def ProcessOptions(options): | |
137 global VARIANT_FLAGS | |
138 | |
139 # Architecture and mode related stuff. | |
140 if options.arch_and_mode: | |
141 tokens = options.arch_and_mode.split(".") | |
142 options.arch = tokens[0] | |
143 options.mode = tokens[1] | |
144 options.mode = options.mode.split(",") | |
145 for mode in options.mode: | |
146 if not mode in ["debug", "release"]: | |
147 print "Unknown mode %s" % mode | |
148 return False | |
149 if options.arch in ["auto", "native"]: | |
150 options.arch = ARCH_GUESS | |
151 options.arch = options.arch.split(",") | |
152 for arch in options.arch: | |
153 if not arch in ['ia32', 'x64', 'arm', 'mips']: | |
Michael Starzinger
2012/09/21 13:20:40
Shouldn't that be "mipsel"?
Jakob Kummerow
2012/09/21 17:02:24
Done.
| |
154 print "Unknown architecture %s" % arch | |
155 return False | |
156 | |
157 # Special processing of other options, sorted alphabetically. | |
158 | |
159 if options.buildbot: | |
160 # Buildbots run presubmit tests as a separate step. | |
161 options.no_presubmit = True | |
162 options.no_network = True | |
163 if options.command_prefix: | |
164 print("Specifying --command-prefix disables network distribution, " | |
165 "running tests locally.") | |
166 options.no_network = True | |
167 if options.j == 0: | |
168 options.j = multiprocessing.cpu_count() | |
169 if options.no_stress: | |
170 VARIANT_FLAGS = [[], ["--nocrankshaft"]] | |
171 if not options.shell_dir: | |
172 if options.shell: | |
173 print "Warning: --shell is deprecated, use --shell-dir instead." | |
174 options.shell_dir = os.path.dirname(options.shell) | |
175 if options.stress_only: | |
176 VARIANT_FLAGS = [["--stress-opt", "--always-opt"]] | |
177 # Simulators are slow, therefore allow a longer default timeout. | |
178 if options.timeout == -1: | |
179 if options.arch == "arm" or options.arch == "mips": | |
Michael Starzinger
2012/09/21 13:20:40
Likewise.
Jakob Kummerow
2012/09/21 17:02:24
Done.
| |
180 options.timeout = 2 * TIMEOUT_DEFAULT; | |
181 else: | |
182 options.timeout = TIMEOUT_DEFAULT; | |
183 if options.valgrind: | |
184 run_valgrind = os.path.join("tools", "run-valgrind.py") | |
185 # This is OK for distributed running, so we don't need to set no_network. | |
186 options.command_prefix = ("python -u " + run_valgrind + | |
187 options.command_prefix) | |
188 return True | |
189 | |
190 | |
191 def ShardTests(tests, shard_count, shard_run): | |
192 if shard_count < 2: | |
193 return tests | |
194 if shard_run < 1 or shard_run > shard_count: | |
195 print "shard-run not a valid number, should be in [1:shard-count]" | |
196 print "defaulting back to running all tests" | |
197 return tests | |
198 count = 0 | |
199 shard = [] | |
200 for test in tests: | |
201 if count % shard_count == shard_run - 1: | |
202 shard.append(test) | |
203 count += 1 | |
204 return shard | |
205 | |
206 | |
207 def Main(): | |
208 parser = BuildOptions() | |
209 (options, args) = parser.parse_args() | |
210 if not ProcessOptions(options): | |
211 parser.print_help() | |
212 return 1 | |
213 | |
214 exit_code = 0 | |
215 workspace = os.path.abspath(join(os.path.dirname(sys.argv[0]), "..")) | |
216 if not options.no_presubmit: | |
217 print ">>> running presubmit tests" | |
218 code = subprocess.call( | |
219 [sys.executable, join(workspace, "tools", "presubmit.py")]) | |
220 exit_code = code | |
221 | |
222 suite_paths = utils.GetSuitePaths(join(workspace, "test")) | |
223 print "all suite_paths:", suite_paths | |
224 | |
225 if len(args) == 0: | |
226 suite_paths = [ s for s in suite_paths if s in DEFAULT_TESTS ] | |
227 else: | |
228 args_suites = set() | |
229 for arg in args: | |
230 suite = arg.split(os.path.sep)[0] | |
231 if not suite in args_suites: | |
232 args_suites.add(suite) | |
233 suite_paths = [ s for s in suite_paths if s in args_suites ] | |
234 | |
235 suites = [] | |
236 for root in suite_paths: | |
237 suite = testsuite.TestSuite.LoadTestSuite( | |
238 os.path.join(workspace, "test", root)) | |
239 if suite: | |
240 suites.append(suite) | |
241 | |
242 if options.download_data: | |
243 for s in suites: | |
244 s.DownloadData() | |
245 | |
246 for mode in options.mode: | |
247 for arch in options.arch: | |
248 code = Execute(arch, mode, args, options, suites, workspace) | |
249 exit_code = exit_code or code | |
250 return exit_code | |
251 | |
252 | |
253 def Execute(arch, mode, args, options, suites, workspace): | |
254 print(">>> Running tests for %s.%s" % (arch, mode)) | |
255 | |
256 if not options.shell_dir: | |
257 if options.buildbot: | |
258 default_dir = os.path.join(workspace, options.outdir, mode) | |
259 mode = mode.lower() | |
260 else: | |
261 default_dir = os.path.join(workspace, | |
262 "%s" % options.outdir, | |
263 "%s.%s" % (arch, mode)) | |
264 options.shell_dir = os.path.relpath(default_dir) | |
265 | |
266 # Populate context object. | |
267 mode_flags = MODE_FLAGS[mode] | |
268 options.timeout *= TIMEOUT_SCALEFACTOR[mode] | |
269 ctx = context.Context(arch, mode, | |
270 os.path.relpath(options.shell_dir), | |
271 mode_flags, options.verbose, | |
272 options.timeout, options.isolates, | |
273 options.command_prefix, | |
274 options.extra_flags) | |
275 | |
276 # Find available test suites and read test cases from them. | |
277 variables = { | |
278 "mode": mode, | |
279 "arch": arch, | |
280 "system": utils.GuessOS(), | |
281 "isolates": options.isolates | |
282 } | |
283 all_tests = [] | |
284 num_tests = 0 | |
285 test_id = 0 | |
286 for s in suites: | |
287 s.ReadStatusFile(variables) | |
288 s.ReadTestCases(ctx) | |
289 all_tests += s.tests | |
290 if len(args) > 0: | |
291 s.FilterTestCasesByArgs(args) | |
292 s.FilterTestCasesByStatus(options.warn_unused) | |
293 if options.cat: | |
294 verbose.PrintTestSource(s.tests) | |
295 continue | |
296 variant_flags = s.VariantFlags() or VARIANT_FLAGS | |
297 s.tests = [ t.CopyAddingFlags(v) for t in s.tests for v in variant_flags ] | |
298 s.tests = ShardTests(s.tests, options.shard_count, options.shard_run) | |
299 num_tests += len(s.tests) | |
300 for t in s.tests: | |
301 t.id = test_id | |
302 test_id += 1 | |
303 | |
304 if options.cat: | |
305 return 0 # We're done here. | |
306 | |
307 if options.report: | |
308 verbose.PrintReport(all_tests) | |
309 | |
310 if num_tests == 0: | |
311 print "No tests to run." | |
312 return 0 | |
313 | |
314 # Run the tests, either locally or distributed on the network. | |
315 try: | |
316 start_time = time.time() | |
317 progress_indicator = progress.PROGRESS_INDICATORS[options.progress]() | |
318 | |
319 run_networked = not options.no_network | |
320 if not run_networked: | |
321 print("Network distribution disabled, running tests locally.") | |
322 elif utils.GuessOS() != "linux": | |
323 print("Network distribution is only supported on Linux, sorry!") | |
324 run_networked = False | |
325 peers = [] | |
326 if run_networked: | |
327 peers = network_execution.GetPeers() | |
328 if not peers: | |
329 print("No connection to distribution server; running tests locally.") | |
330 run_networked = False | |
331 elif len(peers) == 1: | |
332 print("No other peers on the network; running tests locally.") | |
333 run_networked = False | |
334 elif num_tests <= 100: | |
335 print("Less than 100 tests, running them locally.") | |
336 run_networked = False | |
337 | |
338 if run_networked: | |
339 runner = network_execution.NetworkedRunner(suites, progress_indicator, | |
340 ctx, peers, workspace) | |
341 else: | |
342 runner = execution.Runner(suites, progress_indicator, ctx) | |
343 | |
344 exit_code = runner.Run(options.j) | |
345 if runner.terminate: | |
346 return exit_code | |
347 overall_duration = time.time() - start_time | |
348 except KeyboardInterrupt: | |
349 return 1 | |
350 | |
351 if options.time: | |
352 verbose.PrintTestDurations(suites, overall_duration) | |
353 return exit_code | |
354 | |
355 | |
356 if __name__ == "__main__": | |
357 sys.exit(Main()) | |
OLD | NEW |