OLD | NEW |
| (Empty) |
1 #!/usr/bin/env python | |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
3 # Use of this source code is governed by a BSD-style license that can be | |
4 # found in the LICENSE file. | |
5 | |
6 ''' Runs various chrome tests through heapcheck_test.py. | |
7 | |
8 Most of this code is copied from ../valgrind/chrome_tests.py. | |
9 TODO(glider): put common functions to a standalone module. | |
10 ''' | |
11 | |
12 import glob | |
13 import logging | |
14 import multiprocessing | |
15 import optparse | |
16 import os | |
17 import stat | |
18 import sys | |
19 | |
20 import logging_utils | |
21 import path_utils | |
22 | |
23 import common | |
24 import heapcheck_test | |
25 | |
26 class TestNotFound(Exception): pass | |
27 | |
28 def Dir2IsNewer(dir1, dir2): | |
29 if dir2 is None or not os.path.isdir(dir2): | |
30 return False | |
31 if dir1 is None or not os.path.isdir(dir1): | |
32 return True | |
33 return os.stat(dir2)[stat.ST_MTIME] > os.stat(dir1)[stat.ST_MTIME] | |
34 | |
35 def FindNewestDir(dirs): | |
36 newest_dir = None | |
37 for dir in dirs: | |
38 if Dir2IsNewer(newest_dir, dir): | |
39 newest_dir = dir | |
40 return newest_dir | |
41 | |
42 def File2IsNewer(file1, file2): | |
43 if file2 is None or not os.path.isfile(file2): | |
44 return False | |
45 if file1 is None or not os.path.isfile(file1): | |
46 return True | |
47 return os.stat(file2)[stat.ST_MTIME] > os.stat(file1)[stat.ST_MTIME] | |
48 | |
49 def FindDirContainingNewestFile(dirs, file): | |
50 """Searches for the directory containing the newest copy of |file|. | |
51 | |
52 Args: | |
53 dirs: A list of paths to the directories to search among. | |
54 file: A string containing the file name to search. | |
55 | |
56 Returns: | |
57 The string representing the the directory containing the newest copy of | |
58 |file|. | |
59 | |
60 Raises: | |
61 IOError: |file| was not found. | |
62 """ | |
63 newest_dir = None | |
64 newest_file = None | |
65 for dir in dirs: | |
66 the_file = os.path.join(dir, file) | |
67 if File2IsNewer(newest_file, the_file): | |
68 newest_dir = dir | |
69 newest_file = the_file | |
70 if newest_dir is None: | |
71 raise IOError("cannot find file %s anywhere, have you built it?" % file) | |
72 return newest_dir | |
73 | |
74 class ChromeTests(object): | |
75 '''This class is derived from the chrome_tests.py file in ../purify/. | |
76 ''' | |
77 | |
78 def __init__(self, options, args, test): | |
79 # The known list of tests. | |
80 # Recognise the original abbreviations as well as full executable names. | |
81 self._test_list = { | |
82 "app_list": self.TestAppList, "app_list_unittests": self.TestAppList, | |
83 "ash": self.TestAsh, "ash_unittests": self.TestAsh, | |
84 "aura": self.TestAura, "aura_unittests": self.TestAura, | |
85 "base": self.TestBase, "base_unittests": self.TestBase, | |
86 "browser": self.TestBrowser, "browser_tests": self.TestBrowser, | |
87 "chromeos": self.TestChromeOS, "chromeos_unittests": self.TestChromeOS, | |
88 "components": self.TestComponents, | |
89 "components_unittests": self.TestComponents, | |
90 "compositor": self.TestCompositor, | |
91 "compositor_unittests": self.TestCompositor, | |
92 "content": self.TestContent, "content_unittests": self.TestContent, | |
93 "content_browsertests": self.TestContentBrowser, | |
94 "courgette": self.TestCourgette, | |
95 "courgette_unittests": self.TestCourgette, | |
96 "crypto": self.TestCrypto, "crypto_unittests": self.TestCrypto, | |
97 "device": self.TestDevice, "device_unittests": self.TestDevice, | |
98 "events": self.TestEvents, "events_unittests": self.TestEvents, | |
99 "gpu": self.TestGPU, "gpu_unittests": self.TestGPU, | |
100 "ipc": self.TestIpc, "ipc_tests": self.TestIpc, | |
101 "jingle": self.TestJingle, "jingle_unittests": self.TestJingle, | |
102 "layout": self.TestLayout, "layout_tests": self.TestLayout, | |
103 "media": self.TestMedia, "media_unittests": self.TestMedia, | |
104 "message_center": self.TestMessageCenter, | |
105 "message_center_unittests" : self.TestMessageCenter, | |
106 "net": self.TestNet, "net_unittests": self.TestNet, | |
107 "ppapi": self.TestPPAPI, "ppapi_unittests": self.TestPPAPI, | |
108 "printing": self.TestPrinting, "printing_unittests": self.TestPrinting, | |
109 "remoting": self.TestRemoting, "remoting_unittests": self.TestRemoting, | |
110 "sql": self.TestSql, "sql_unittests": self.TestSql, | |
111 "startup": self.TestStartup, "startup_tests": self.TestStartup, | |
112 "sync": self.TestSync, "sync_unit_tests": self.TestSync, | |
113 "ui_unit": self.TestUIUnit, "ui_unittests": self.TestUIUnit, | |
114 "unit": self.TestUnit, "unit_tests": self.TestUnit, | |
115 "url": self.TestURL, "url_unittests": self.TestURL, | |
116 "views": self.TestViews, "views_unittests": self.TestViews, | |
117 } | |
118 | |
119 if test not in self._test_list: | |
120 raise TestNotFound("Unknown test: %s" % test) | |
121 | |
122 self._options = options | |
123 self._args = args | |
124 self._test = test | |
125 | |
126 script_dir = path_utils.ScriptDir() | |
127 | |
128 # Compute the top of the tree (the "source dir") from the script dir (where | |
129 # this script lives). We assume that the script dir is in tools/heapcheck/ | |
130 # relative to the top of the tree. | |
131 self._source_dir = os.path.dirname(os.path.dirname(script_dir)) | |
132 | |
133 # Since this path is used for string matching, make sure it's always | |
134 # an absolute Unix-style path. | |
135 self._source_dir = os.path.abspath(self._source_dir).replace('\\', '/') | |
136 | |
137 heapcheck_test_script = os.path.join(script_dir, "heapcheck_test.py") | |
138 self._command_preamble = [heapcheck_test_script] | |
139 | |
140 def _DefaultCommand(self, module, exe=None, heapcheck_test_args=None): | |
141 '''Generates the default command array that most tests will use. | |
142 | |
143 Args: | |
144 module: The module name (corresponds to the dir in src/ where the test | |
145 data resides). | |
146 exe: The executable name. | |
147 heapcheck_test_args: additional arguments to append to the command line. | |
148 Returns: | |
149 A string with the command to run the test. | |
150 ''' | |
151 if not self._options.build_dir: | |
152 dirs = [ | |
153 os.path.join(self._source_dir, "xcodebuild", "Debug"), | |
154 os.path.join(self._source_dir, "out", "Debug"), | |
155 ] | |
156 if exe: | |
157 self._options.build_dir = FindDirContainingNewestFile(dirs, exe) | |
158 else: | |
159 self._options.build_dir = FindNewestDir(dirs) | |
160 | |
161 cmd = list(self._command_preamble) | |
162 | |
163 if heapcheck_test_args != None: | |
164 for arg in heapcheck_test_args: | |
165 cmd.append(arg) | |
166 if exe: | |
167 cmd.append(os.path.join(self._options.build_dir, exe)) | |
168 # Heapcheck runs tests slowly, so slow tests hurt more; show elapased time | |
169 # so we can find the slowpokes. | |
170 cmd.append("--gtest_print_time") | |
171 if self._options.gtest_repeat: | |
172 cmd.append("--gtest_repeat=%s" % self._options.gtest_repeat) | |
173 return cmd | |
174 | |
175 def Suppressions(self): | |
176 '''Builds the list of available suppressions files.''' | |
177 ret = [] | |
178 directory = path_utils.ScriptDir() | |
179 suppression_file = os.path.join(directory, "suppressions.txt") | |
180 if os.path.exists(suppression_file): | |
181 ret.append(suppression_file) | |
182 suppression_file = os.path.join(directory, "suppressions_linux.txt") | |
183 if os.path.exists(suppression_file): | |
184 ret.append(suppression_file) | |
185 return ret | |
186 | |
187 def Run(self): | |
188 '''Runs the test specified by command-line argument --test.''' | |
189 logging.info("running test %s" % (self._test)) | |
190 return self._test_list[self._test]() | |
191 | |
192 def _ReadGtestFilterFile(self, name, cmd): | |
193 '''Reads files which contain lists of tests to filter out with | |
194 --gtest_filter and appends the command-line option to |cmd|. | |
195 | |
196 Args: | |
197 name: the test executable name. | |
198 cmd: the test running command line to be modified. | |
199 ''' | |
200 filters = [] | |
201 directory = path_utils.ScriptDir() | |
202 gtest_filter_files = [ | |
203 os.path.join(directory, name + ".gtest-heapcheck.txt"), | |
204 # TODO(glider): Linux vs. CrOS? | |
205 ] | |
206 logging.info("Reading gtest exclude filter files:") | |
207 for filename in gtest_filter_files: | |
208 # strip the leading absolute path (may be very long on the bot) | |
209 # and the following / or \. | |
210 readable_filename = filename.replace(self._source_dir, "")[1:] | |
211 if not os.path.exists(filename): | |
212 logging.info(" \"%s\" - not found" % readable_filename) | |
213 continue | |
214 logging.info(" \"%s\" - OK" % readable_filename) | |
215 f = open(filename, 'r') | |
216 for line in f.readlines(): | |
217 if line.startswith("#") or line.startswith("//") or line.isspace(): | |
218 continue | |
219 line = line.rstrip() | |
220 filters.append(line) | |
221 gtest_filter = self._options.gtest_filter | |
222 if len(filters): | |
223 if gtest_filter: | |
224 gtest_filter += ":" | |
225 if gtest_filter.find("-") < 0: | |
226 gtest_filter += "-" | |
227 else: | |
228 gtest_filter = "-" | |
229 gtest_filter += ":".join(filters) | |
230 if gtest_filter: | |
231 cmd.append("--gtest_filter=%s" % gtest_filter) | |
232 | |
233 def SimpleTest(self, module, name, heapcheck_test_args=None, cmd_args=None): | |
234 '''Builds the command line and runs the specified test. | |
235 | |
236 Args: | |
237 module: The module name (corresponds to the dir in src/ where the test | |
238 data resides). | |
239 name: The executable name. | |
240 heapcheck_test_args: Additional command line args for heap checker. | |
241 cmd_args: Additional command line args for the test. | |
242 ''' | |
243 cmd = self._DefaultCommand(module, name, heapcheck_test_args) | |
244 supp = self.Suppressions() | |
245 self._ReadGtestFilterFile(name, cmd) | |
246 if cmd_args: | |
247 cmd.extend(["--"]) | |
248 cmd.extend(cmd_args) | |
249 | |
250 # Sets LD_LIBRARY_PATH to the build folder so external libraries can be | |
251 # loaded. | |
252 if (os.getenv("LD_LIBRARY_PATH")): | |
253 os.putenv("LD_LIBRARY_PATH", "%s:%s" % (os.getenv("LD_LIBRARY_PATH"), | |
254 self._options.build_dir)) | |
255 else: | |
256 os.putenv("LD_LIBRARY_PATH", self._options.build_dir) | |
257 return heapcheck_test.RunTool(cmd, supp, module) | |
258 | |
259 # TODO(glider): it's an overkill to define a method for each simple test. | |
260 def TestAppList(self): | |
261 return self.SimpleTest("app_list", "app_list_unittests") | |
262 | |
263 def TestAsh(self): | |
264 return self.SimpleTest("ash", "ash_unittests") | |
265 | |
266 def TestAura(self): | |
267 return self.SimpleTest("aura", "aura_unittests") | |
268 | |
269 def TestBase(self): | |
270 return self.SimpleTest("base", "base_unittests") | |
271 | |
272 def TestBrowser(self): | |
273 return self.SimpleTest("chrome", "browser_tests") | |
274 | |
275 def TestChromeOS(self): | |
276 return self.SimpleTest("chromeos", "chromeos_unittests") | |
277 | |
278 def TestComponents(self): | |
279 return self.SimpleTest("components", "components_unittests") | |
280 | |
281 def TestCompositor(self): | |
282 return self.SimpleTest("compositor", "compositor_unittests") | |
283 | |
284 def TestContent(self): | |
285 return self.SimpleTest("content", "content_unittests") | |
286 | |
287 def TestContentBrowser(self): | |
288 return self.SimpleTest("content", "content_browsertests") | |
289 | |
290 def TestCourgette(self): | |
291 return self.SimpleTest("courgette", "courgette_unittests") | |
292 | |
293 def TestCrypto(self): | |
294 return self.SimpleTest("crypto", "crypto_unittests") | |
295 | |
296 def TestDevice(self): | |
297 return self.SimpleTest("device", "device_unittests") | |
298 | |
299 def TestEvents(self): | |
300 return self.SimpleTest("events", "events_unittests") | |
301 | |
302 def TestGPU(self): | |
303 return self.SimpleTest("gpu", "gpu_unittests") | |
304 | |
305 def TestIpc(self): | |
306 return self.SimpleTest("ipc", "ipc_tests") | |
307 | |
308 def TestJingle(self): | |
309 return self.SimpleTest("chrome", "jingle_unittests") | |
310 | |
311 def TestMedia(self): | |
312 return self.SimpleTest("chrome", "media_unittests") | |
313 | |
314 def TestMessageCenter(self): | |
315 return self.SimpleTest("message_center", "message_center_unittests") | |
316 | |
317 def TestNet(self): | |
318 return self.SimpleTest("net", "net_unittests") | |
319 | |
320 def TestPPAPI(self): | |
321 return self.SimpleTest("chrome", "ppapi_unittests") | |
322 | |
323 def TestPrinting(self): | |
324 return self.SimpleTest("chrome", "printing_unittests") | |
325 | |
326 def TestRemoting(self): | |
327 return self.SimpleTest("chrome", "remoting_unittests") | |
328 | |
329 def TestSync(self): | |
330 return self.SimpleTest("chrome", "sync_unit_tests") | |
331 | |
332 def TestStartup(self): | |
333 # We don't need the performance results, we're just looking for pointer | |
334 # errors, so set number of iterations down to the minimum. | |
335 os.putenv("STARTUP_TESTS_NUMCYCLES", "1") | |
336 logging.info("export STARTUP_TESTS_NUMCYCLES=1"); | |
337 return self.SimpleTest("chrome", "startup_tests") | |
338 | |
339 def TestUIUnit(self): | |
340 return self.SimpleTest("chrome", "ui_unittests") | |
341 | |
342 def TestUnit(self): | |
343 return self.SimpleTest("chrome", "unit_tests") | |
344 | |
345 def TestURL(self): | |
346 return self.SimpleTest("chrome", "url_unittests") | |
347 | |
348 def TestSql(self): | |
349 return self.SimpleTest("chrome", "sql_unittests") | |
350 | |
351 def TestViews(self): | |
352 return self.SimpleTest("views", "views_unittests") | |
353 | |
354 def TestLayoutChunk(self, chunk_num, chunk_size): | |
355 '''Runs tests [chunk_num*chunk_size .. (chunk_num+1)*chunk_size). | |
356 | |
357 Wrap around to beginning of list at end. If chunk_size is zero, run all | |
358 tests in the list once. If a text file is given as argument, it is used as | |
359 the list of tests. | |
360 ''' | |
361 # Build the ginormous commandline in 'cmd'. | |
362 # It's going to be roughly | |
363 # python heapcheck_test.py ... python run_webkit_tests.py ... | |
364 # but we'll use the --indirect flag to heapcheck_test.py | |
365 # to avoid heapchecking python. | |
366 # Start by building the heapcheck_test.py commandline. | |
367 cmd = self._DefaultCommand("webkit") | |
368 | |
369 # Now build script_cmd, the run_webkits_tests.py commandline | |
370 # Store each chunk in its own directory so that we can find the data later | |
371 chunk_dir = os.path.join("layout", "chunk_%05d" % chunk_num) | |
372 out_dir = os.path.join(path_utils.ScriptDir(), "latest") | |
373 out_dir = os.path.join(out_dir, chunk_dir) | |
374 if os.path.exists(out_dir): | |
375 old_files = glob.glob(os.path.join(out_dir, "*.txt")) | |
376 for f in old_files: | |
377 os.remove(f) | |
378 else: | |
379 os.makedirs(out_dir) | |
380 | |
381 script = os.path.join(self._source_dir, "webkit", "tools", "layout_tests", | |
382 "run_webkit_tests.py") | |
383 # While Heapcheck is not memory bound like Valgrind for running layout tests | |
384 # in parallel, it is still CPU bound. Many machines have hyper-threading | |
385 # turned on, so the real number of cores is actually half. | |
386 jobs = max(1, int(multiprocessing.cpu_count() * 0.5)) | |
387 script_cmd = ["python", script, "-v", | |
388 "--run-singly", # run a separate DumpRenderTree for each test | |
389 "--fully-parallel", | |
390 "--child-processes=%d" % jobs, | |
391 "--time-out-ms=200000", | |
392 "--no-retry-failures", # retrying takes too much time | |
393 # http://crbug.com/176908: Don't launch a browser when done. | |
394 "--no-show-results", | |
395 "--nocheck-sys-deps"] | |
396 | |
397 # Pass build mode to run_webkit_tests.py. We aren't passed it directly, | |
398 # so parse it out of build_dir. run_webkit_tests.py can only handle | |
399 # the two values "Release" and "Debug". | |
400 # TODO(Hercules): unify how all our scripts pass around build mode | |
401 # (--mode / --target / --build-dir / --debug) | |
402 if self._options.build_dir.endswith("Debug"): | |
403 script_cmd.append("--debug"); | |
404 if (chunk_size > 0): | |
405 script_cmd.append("--run-chunk=%d:%d" % (chunk_num, chunk_size)) | |
406 if len(self._args): | |
407 # if the arg is a txt file, then treat it as a list of tests | |
408 if os.path.isfile(self._args[0]) and self._args[0][-4:] == ".txt": | |
409 script_cmd.append("--test-list=%s" % self._args[0]) | |
410 else: | |
411 script_cmd.extend(self._args) | |
412 self._ReadGtestFilterFile("layout", script_cmd) | |
413 | |
414 # Now run script_cmd with the wrapper in cmd | |
415 cmd.extend(["--"]) | |
416 cmd.extend(script_cmd) | |
417 supp = self.Suppressions() | |
418 return heapcheck_test.RunTool(cmd, supp, "layout") | |
419 | |
420 def TestLayout(self): | |
421 '''Runs the layout tests.''' | |
422 # A "chunk file" is maintained in the local directory so that each test | |
423 # runs a slice of the layout tests of size chunk_size that increments with | |
424 # each run. Since tests can be added and removed from the layout tests at | |
425 # any time, this is not going to give exact coverage, but it will allow us | |
426 # to continuously run small slices of the layout tests under purify rather | |
427 # than having to run all of them in one shot. | |
428 chunk_size = self._options.num_tests | |
429 if (chunk_size == 0): | |
430 return self.TestLayoutChunk(0, 0) | |
431 chunk_num = 0 | |
432 chunk_file = os.path.join("heapcheck_layout_chunk.txt") | |
433 | |
434 logging.info("Reading state from " + chunk_file) | |
435 try: | |
436 f = open(chunk_file) | |
437 if f: | |
438 str = f.read() | |
439 if len(str): | |
440 chunk_num = int(str) | |
441 # This should be enough so that we have a couple of complete runs | |
442 # of test data stored in the archive (although note that when we loop | |
443 # that we almost guaranteed won't be at the end of the test list) | |
444 if chunk_num > 10000: | |
445 chunk_num = 0 | |
446 f.close() | |
447 except IOError, (errno, strerror): | |
448 logging.error("error reading from file %s (%d, %s)" % (chunk_file, | |
449 errno, strerror)) | |
450 ret = self.TestLayoutChunk(chunk_num, chunk_size) | |
451 | |
452 # Wait until after the test runs to completion to write out the new chunk | |
453 # number. This way, if the bot is killed, we'll start running again from | |
454 # the current chunk rather than skipping it. | |
455 logging.info("Saving state to " + chunk_file) | |
456 try: | |
457 f = open(chunk_file, "w") | |
458 chunk_num += 1 | |
459 f.write("%d" % chunk_num) | |
460 f.close() | |
461 except IOError, (errno, strerror): | |
462 logging.error("error writing to file %s (%d, %s)" % (chunk_file, errno, | |
463 strerror)) | |
464 | |
465 # Since we're running small chunks of the layout tests, it's important to | |
466 # mark the ones that have errors in them. These won't be visible in the | |
467 # summary list for long, but will be useful for someone reviewing this bot. | |
468 return ret | |
469 | |
470 | |
471 def main(): | |
472 if not sys.platform.startswith('linux'): | |
473 logging.error("Heap checking works only on Linux at the moment.") | |
474 return 1 | |
475 parser = optparse.OptionParser("usage: %prog -b <dir> -t <test> " | |
476 "[-t <test> ...]") | |
477 parser.add_option("-b", "--build-dir", | |
478 help="the location of the output of the compiler output") | |
479 parser.add_option("--target", help="Debug or Release") | |
480 parser.add_option("-t", "--test", action="append", help="which test to run") | |
481 parser.add_option("--gtest_filter", | |
482 help="additional arguments to --gtest_filter") | |
483 parser.add_option("--gtest_repeat", help="argument for --gtest_repeat") | |
484 parser.add_option("-v", "--verbose", action="store_true", default=False, | |
485 help="verbose output - enable debug log messages") | |
486 # My machine can do about 120 layout tests/hour in release mode. | |
487 # Let's do 30 minutes worth per run. | |
488 # The CPU is mostly idle, so perhaps we can raise this when | |
489 # we figure out how to run them more efficiently. | |
490 parser.add_option("-n", "--num_tests", default=60, type="int", | |
491 help="for layout tests: # of subtests per run. 0 for all.") | |
492 | |
493 options, args = parser.parse_args() | |
494 | |
495 # Bake target into build_dir. | |
496 if options.target and options.build_dir: | |
497 assert (options.target != | |
498 os.path.basename(os.path.dirname(options.build_dir))) | |
499 options.build_dir = os.path.join(os.path.abspath(options.build_dir), | |
500 options.target) | |
501 | |
502 if options.verbose: | |
503 logging_utils.config_root(logging.DEBUG) | |
504 else: | |
505 logging_utils.config_root() | |
506 | |
507 if not options.test or not len(options.test): | |
508 parser.error("--test not specified") | |
509 | |
510 for t in options.test: | |
511 tests = ChromeTests(options, args, t) | |
512 ret = tests.Run() | |
513 if ret: | |
514 return ret | |
515 return 0 | |
516 | |
517 | |
518 if __name__ == "__main__": | |
519 sys.exit(main()) | |
OLD | NEW |