OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # | 2 # |
3 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 3 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
4 # Use of this source code is governed by a BSD-style license that can be | 4 # Use of this source code is governed by a BSD-style license that can be |
5 # found in the LICENSE file. | 5 # found in the LICENSE file. |
6 | 6 |
7 """Runs all the native unit tests. | 7 """Runs all the native unit tests. |
8 | 8 |
9 1. Copy over test binary to /data/local on device. | 9 1. Copy over test binary to /data/local on device. |
10 2. Resources: chrome/unit_tests requires resources (chrome.pak and en-US.pak) | 10 2. Resources: chrome/unit_tests requires resources (chrome.pak and en-US.pak) |
(...skipping 64 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
75 'ipc_tests', | 75 'ipc_tests', |
76 'media_unittests', | 76 'media_unittests', |
77 'net_unittests', | 77 'net_unittests', |
78 'sql_unittests', | 78 'sql_unittests', |
79 'sync_unit_tests', | 79 'sync_unit_tests', |
80 'ui_unittests', | 80 'ui_unittests', |
81 'unit_tests', | 81 'unit_tests', |
82 ] | 82 ] |
83 | 83 |
84 | 84 |
85 def FullyQualifiedTestSuites(exe, option_test_suite): | 85 def TestSuiteDir(build_type): |
| 86 """Return the base directory of test suites.""" |
| 87 return os.path.abspath(os.path.join(constants.CHROME_DIR, 'out', build_type)) |
| 88 |
| 89 def FullyQualifiedTestSuites(exe, option_test_suite, build_type): |
86 """Return a fully qualified list | 90 """Return a fully qualified list |
87 | 91 |
88 Args: | 92 Args: |
89 exe: if True, use the executable-based test runner. | 93 exe: if True, use the executable-based test runner. |
90 option_test_suite: the test_suite specified as an option. | 94 option_test_suite: the test_suite specified as an option. |
| 95 build_type: 'Release' or 'Debug'. |
91 """ | 96 """ |
92 # Assume the test suites are in out/Release. | 97 test_suite_dir = TestSuiteDir(build_type) |
93 test_suite_dir = os.path.abspath(os.path.join(constants.CHROME_DIR, | |
94 'out', 'Release')) | |
95 if option_test_suite: | 98 if option_test_suite: |
96 all_test_suites = [option_test_suite] | 99 all_test_suites = [option_test_suite] |
97 else: | 100 else: |
98 all_test_suites = _TEST_SUITES | 101 all_test_suites = _TEST_SUITES |
99 | 102 |
100 if exe: | 103 if exe: |
101 qualified_test_suites = [os.path.join(test_suite_dir, t) | 104 qualified_test_suites = [os.path.join(test_suite_dir, t) |
102 for t in all_test_suites] | 105 for t in all_test_suites] |
103 else: | 106 else: |
104 # out/Release/$SUITE_apk/$SUITE-debug.apk | 107 # out/(Debug|Release)/$SUITE_apk/$SUITE-debug.apk |
105 qualified_test_suites = [os.path.join(test_suite_dir, | 108 qualified_test_suites = [os.path.join(test_suite_dir, |
106 t + '_apk', | 109 t + '_apk', |
107 t + '-debug.apk') | 110 t + '-debug.apk') |
108 for t in all_test_suites] | 111 for t in all_test_suites] |
109 for t, q in zip(all_test_suites, qualified_test_suites): | 112 for t, q in zip(all_test_suites, qualified_test_suites): |
110 if not os.path.exists(q): | 113 if not os.path.exists(q): |
111 logging.critical('Test suite %s not found in %s.\n' | 114 logging.critical('Test suite %s not found in %s.\n' |
112 'Supported test suites:\n %s\n' | 115 'Supported test suites:\n %s\n' |
113 'Ensure it has been built.\n', | 116 'Ensure it has been built.\n', |
114 t, q, _TEST_SUITES) | 117 t, q, _TEST_SUITES) |
(...skipping 73 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
188 buildbot_report.PrintError() | 191 buildbot_report.PrintError() |
189 else: | 192 else: |
190 print 'Step success!' # No annotation needed | 193 print 'Step success!' # No annotation needed |
191 | 194 |
192 | 195 |
193 class TestSharder(BaseTestSharder): | 196 class TestSharder(BaseTestSharder): |
194 """Responsible for sharding the tests on the connected devices.""" | 197 """Responsible for sharding the tests on the connected devices.""" |
195 | 198 |
196 def __init__(self, attached_devices, test_suite, gtest_filter, | 199 def __init__(self, attached_devices, test_suite, gtest_filter, |
197 test_arguments, timeout, rebaseline, performance_test, | 200 test_arguments, timeout, rebaseline, performance_test, |
198 cleanup_test_files, tool, log_dump_name, fast_and_loose): | 201 cleanup_test_files, tool, log_dump_name, fast_and_loose, |
| 202 build_type): |
199 BaseTestSharder.__init__(self, attached_devices) | 203 BaseTestSharder.__init__(self, attached_devices) |
200 self.test_suite = test_suite | 204 self.test_suite = test_suite |
201 self.test_suite_basename = os.path.basename(test_suite) | 205 self.test_suite_basename = os.path.basename(test_suite) |
202 self.gtest_filter = gtest_filter or '' | 206 self.gtest_filter = gtest_filter or '' |
203 self.test_arguments = test_arguments | 207 self.test_arguments = test_arguments |
204 self.timeout = timeout | 208 self.timeout = timeout |
205 self.rebaseline = rebaseline | 209 self.rebaseline = rebaseline |
206 self.performance_test = performance_test | 210 self.performance_test = performance_test |
207 self.cleanup_test_files = cleanup_test_files | 211 self.cleanup_test_files = cleanup_test_files |
208 self.tool = tool | 212 self.tool = tool |
209 self.log_dump_name = log_dump_name | 213 self.log_dump_name = log_dump_name |
210 self.fast_and_loose = fast_and_loose | 214 self.fast_and_loose = fast_and_loose |
| 215 self.build_type = build_type |
211 test = SingleTestRunner(self.attached_devices[0], test_suite, gtest_filter, | 216 test = SingleTestRunner(self.attached_devices[0], test_suite, gtest_filter, |
212 test_arguments, timeout, rebaseline, | 217 test_arguments, timeout, rebaseline, |
213 performance_test, cleanup_test_files, tool, 0, | 218 performance_test, cleanup_test_files, tool, 0, |
214 not not self.log_dump_name, fast_and_loose) | 219 not not self.log_dump_name, fast_and_loose) |
215 self.tests = [] | 220 self.tests = [] |
216 if not self.gtest_filter: | 221 if not self.gtest_filter: |
217 # No filter has been specified, let's add all tests then. | 222 # No filter has been specified, let's add all tests then. |
218 # The executable/apk needs to be copied before we can call GetAllTests. | 223 # The executable/apk needs to be copied before we can call GetAllTests. |
219 test.test_package.StripAndCopyExecutable() | 224 test.test_package.StripAndCopyExecutable() |
220 all_tests = test.test_package.GetAllTests() | 225 all_tests = test.test_package.GetAllTests() |
(...skipping 28 matching lines...) Expand all Loading... |
249 | 254 |
250 def OnTestsCompleted(self, test_runners, test_results): | 255 def OnTestsCompleted(self, test_runners, test_results): |
251 """Notifies that we completed the tests.""" | 256 """Notifies that we completed the tests.""" |
252 test_results.LogFull('Unit test', os.path.basename(self.test_suite)) | 257 test_results.LogFull('Unit test', os.path.basename(self.test_suite)) |
253 PrintAnnotationForTestResults(test_results) | 258 PrintAnnotationForTestResults(test_results) |
254 if test_results.failed and self.rebaseline: | 259 if test_results.failed and self.rebaseline: |
255 test_runners[0].UpdateFilter(test_results.failed) | 260 test_runners[0].UpdateFilter(test_results.failed) |
256 if self.log_dump_name: | 261 if self.log_dump_name: |
257 # Zip all debug info outputs into a file named by log_dump_name. | 262 # Zip all debug info outputs into a file named by log_dump_name. |
258 debug_info.GTestDebugInfo.ZipAndCleanResults( | 263 debug_info.GTestDebugInfo.ZipAndCleanResults( |
259 os.path.join(constants.CHROME_DIR, 'out', 'Release', | 264 os.path.join(TestSuiteDir(self.build_type), 'debug_info_dumps'), |
260 'debug_info_dumps'), | |
261 self.log_dump_name) | 265 self.log_dump_name) |
262 | 266 |
263 | 267 |
264 def _RunATestSuite(options): | 268 def _RunATestSuite(options): |
265 """Run a single test suite. | 269 """Run a single test suite. |
266 | 270 |
267 Helper for Dispatch() to allow stop/restart of the emulator across | 271 Helper for Dispatch() to allow stop/restart of the emulator across |
268 test bundles. If using the emulator, we start it on entry and stop | 272 test bundles. If using the emulator, we start it on entry and stop |
269 it on exit. | 273 it on exit. |
270 | 274 |
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
310 raise Exception('Failed to reset test server port.') | 314 raise Exception('Failed to reset test server port.') |
311 | 315 |
312 if options.performance_test or options.gtest_filter: | 316 if options.performance_test or options.gtest_filter: |
313 # These configuration can't be split in multiple devices. | 317 # These configuration can't be split in multiple devices. |
314 attached_devices = [attached_devices[0]] | 318 attached_devices = [attached_devices[0]] |
315 sharder = TestSharder(attached_devices, options.test_suite, | 319 sharder = TestSharder(attached_devices, options.test_suite, |
316 options.gtest_filter, options.test_arguments, | 320 options.gtest_filter, options.test_arguments, |
317 options.timeout, options.rebaseline, | 321 options.timeout, options.rebaseline, |
318 options.performance_test, | 322 options.performance_test, |
319 options.cleanup_test_files, options.tool, | 323 options.cleanup_test_files, options.tool, |
320 options.log_dump, options.fast_and_loose) | 324 options.log_dump, options.fast_and_loose, |
| 325 options.build_type) |
321 test_results = sharder.RunShardedTests() | 326 test_results = sharder.RunShardedTests() |
322 | 327 |
323 for buildbot_emulator in buildbot_emulators: | 328 for buildbot_emulator in buildbot_emulators: |
324 buildbot_emulator.Shutdown() | 329 buildbot_emulator.Shutdown() |
325 | 330 |
326 # Another chance if we timed out? At this point It is safe(r) to | 331 # Another chance if we timed out? At this point It is safe(r) to |
327 # run fast and loose since we just uploaded all the test data and | 332 # run fast and loose since we just uploaded all the test data and |
328 # binary. | 333 # binary. |
329 if test_results.timed_out and options.repeat: | 334 if test_results.timed_out and options.repeat: |
330 logging.critical('Timed out; repeating in fast_and_loose mode.') | 335 logging.critical('Timed out; repeating in fast_and_loose mode.') |
(...skipping 17 matching lines...) Expand all Loading... |
348 0 if successful, number of failing tests otherwise. | 353 0 if successful, number of failing tests otherwise. |
349 """ | 354 """ |
350 if options.test_suite == 'help': | 355 if options.test_suite == 'help': |
351 ListTestSuites() | 356 ListTestSuites() |
352 return 0 | 357 return 0 |
353 | 358 |
354 if options.use_xvfb: | 359 if options.use_xvfb: |
355 xvfb = Xvfb() | 360 xvfb = Xvfb() |
356 xvfb.Start() | 361 xvfb.Start() |
357 | 362 |
358 all_test_suites = FullyQualifiedTestSuites(options.exe, options.test_suite) | 363 all_test_suites = FullyQualifiedTestSuites(options.exe, options.test_suite, |
| 364 options.build_type) |
359 failures = 0 | 365 failures = 0 |
360 for suite in all_test_suites: | 366 for suite in all_test_suites: |
361 options.test_suite = suite | 367 options.test_suite = suite |
362 failures += _RunATestSuite(options) | 368 failures += _RunATestSuite(options) |
363 | 369 |
364 if options.use_xvfb: | 370 if options.use_xvfb: |
365 xvfb.Stop() | 371 xvfb.Stop() |
366 return failures | 372 return failures |
367 | 373 |
368 | 374 |
(...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
415 'Don\'t use on bots by default!') | 421 'Don\'t use on bots by default!') |
416 option_parser.add_option('--repeat', dest='repeat', type='int', | 422 option_parser.add_option('--repeat', dest='repeat', type='int', |
417 default=2, | 423 default=2, |
418 help='Repeat count on test timeout') | 424 help='Repeat count on test timeout') |
419 option_parser.add_option('--exit_code', action='store_true', | 425 option_parser.add_option('--exit_code', action='store_true', |
420 help='If set, the exit code will be total number ' | 426 help='If set, the exit code will be total number ' |
421 'of failures.') | 427 'of failures.') |
422 option_parser.add_option('--exe', action='store_true', | 428 option_parser.add_option('--exe', action='store_true', |
423 help='If set, use the exe test runner instead of ' | 429 help='If set, use the exe test runner instead of ' |
424 'the APK.') | 430 'the APK.') |
| 431 |
| 432 # TODO(wangxianzhu): Change to Debug when we build Debug by default. |
| 433 default_build_type = 'Release' |
| 434 if 'BUILDTYPE' in os.environ: |
| 435 default_build_type = os.environ['BUILDTYPE'] |
| 436 option_parser.add_option('--debug', action='store_const', const='Debug', |
| 437 dest='build_type', default=default_build_type, |
| 438 help='If set, run test suite under out/Debug.') |
| 439 option_parser.add_option('--release', action='store_const', const='Release', |
| 440 dest='build_type', |
| 441 help='If set, run test suite under out/Release.') |
| 442 |
425 options, args = option_parser.parse_args(argv) | 443 options, args = option_parser.parse_args(argv) |
426 if len(args) > 1: | 444 if len(args) > 1: |
427 print 'Unknown argument:', args[1:] | 445 print 'Unknown argument:', args[1:] |
428 option_parser.print_usage() | 446 option_parser.print_usage() |
429 sys.exit(1) | 447 sys.exit(1) |
430 run_tests_helper.SetLogLevel(options.verbose_count) | 448 run_tests_helper.SetLogLevel(options.verbose_count) |
431 emulator.DeleteAllTempAVDs() | 449 emulator.DeleteAllTempAVDs() |
432 failed_tests_count = Dispatch(options) | 450 failed_tests_count = Dispatch(options) |
433 | 451 |
434 # Failures of individual test suites are communicated by printing a | 452 # Failures of individual test suites are communicated by printing a |
435 # STEP_FAILURE message. | 453 # STEP_FAILURE message. |
436 # Returning a success exit status also prevents the buildbot from incorrectly | 454 # Returning a success exit status also prevents the buildbot from incorrectly |
437 # marking the last suite as failed if there were failures in other suites in | 455 # marking the last suite as failed if there were failures in other suites in |
438 # the batch (this happens because the exit status is a sum of all failures | 456 # the batch (this happens because the exit status is a sum of all failures |
439 # from all suites, but the buildbot associates the exit status only with the | 457 # from all suites, but the buildbot associates the exit status only with the |
440 # most recent step). | 458 # most recent step). |
441 if options.exit_code: | 459 if options.exit_code: |
442 return failed_tests_count | 460 return failed_tests_count |
443 return 0 | 461 return 0 |
444 | 462 |
445 | 463 |
446 if __name__ == '__main__': | 464 if __name__ == '__main__': |
447 sys.exit(main(sys.argv)) | 465 sys.exit(main(sys.argv)) |
OLD | NEW |