OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # | 2 # |
3 # Copyright 2013 The Chromium Authors. All rights reserved. | 3 # Copyright 2013 The Chromium Authors. All rights reserved. |
4 # Use of this source code is governed by a BSD-style license that can be | 4 # Use of this source code is governed by a BSD-style license that can be |
5 # found in the LICENSE file. | 5 # found in the LICENSE file. |
6 | 6 |
7 """Runs all types of tests from one unified interface.""" | 7 """Runs all types of tests from one unified interface.""" |
8 | 8 |
9 import argparse | 9 import argparse |
10 import collections | 10 import collections |
(...skipping 13 matching lines...) Expand all Loading... |
24 from devil.android import device_errors | 24 from devil.android import device_errors |
25 from devil.android import device_utils | 25 from devil.android import device_utils |
26 from devil.android import forwarder | 26 from devil.android import forwarder |
27 from devil.android import ports | 27 from devil.android import ports |
28 from devil.utils import reraiser_thread | 28 from devil.utils import reraiser_thread |
29 from devil.utils import run_tests_helper | 29 from devil.utils import run_tests_helper |
30 | 30 |
31 from pylib import constants | 31 from pylib import constants |
32 from pylib.base import base_test_result | 32 from pylib.base import base_test_result |
33 from pylib.base import environment_factory | 33 from pylib.base import environment_factory |
34 from pylib.base import test_dispatcher | |
35 from pylib.base import test_instance_factory | 34 from pylib.base import test_instance_factory |
36 from pylib.base import test_run_factory | 35 from pylib.base import test_run_factory |
37 from pylib.constants import host_paths | 36 from pylib.constants import host_paths |
38 from pylib.linker import setup as linker_setup | |
39 from pylib.results import json_results | 37 from pylib.results import json_results |
40 from pylib.results import report_results | 38 from pylib.results import report_results |
41 | 39 |
42 from py_utils import contextlib_ext | 40 from py_utils import contextlib_ext |
43 | 41 |
44 | 42 |
45 _DEVIL_STATIC_CONFIG_FILE = os.path.abspath(os.path.join( | 43 _DEVIL_STATIC_CONFIG_FILE = os.path.abspath(os.path.join( |
46 host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'devil_config.json')) | 44 host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'devil_config.json')) |
47 | 45 |
48 | 46 |
(...skipping 24 matching lines...) Expand all Loading... |
73 ' located (must include build type). This will take' | 71 ' located (must include build type). This will take' |
74 ' precedence over --debug, --release and' | 72 ' precedence over --debug, --release and' |
75 ' --build-directory')) | 73 ' --build-directory')) |
76 group.add_argument('--num_retries', '--num-retries', | 74 group.add_argument('--num_retries', '--num-retries', |
77 '--test_launcher_retry_limit', | 75 '--test_launcher_retry_limit', |
78 '--test-launcher-retry-limit', | 76 '--test-launcher-retry-limit', |
79 dest='num_retries', | 77 dest='num_retries', |
80 type=int, default=2, | 78 type=int, default=2, |
81 help=('Number of retries for a test before ' | 79 help=('Number of retries for a test before ' |
82 'giving up (default: %(default)s).')) | 80 'giving up (default: %(default)s).')) |
| 81 group.add_argument('--repeat', '--gtest_repeat', '--gtest-repeat', |
| 82 dest='repeat', type=int, default=0, |
| 83 help='Number of times to repeat the specified set of ' |
| 84 'tests.') |
| 85 group.add_argument('--break-on-failure', '--break_on_failure', |
| 86 dest='break_on_failure', action='store_true', |
| 87 help='Whether to break on failure.') |
83 group.add_argument('-v', | 88 group.add_argument('-v', |
84 '--verbose', | 89 '--verbose', |
85 dest='verbose_count', | 90 dest='verbose_count', |
86 default=0, | 91 default=0, |
87 action='count', | 92 action='count', |
88 help='Verbose level (multiple times for more)') | 93 help='Verbose level (multiple times for more)') |
89 group.add_argument('--flakiness-dashboard-server', | 94 group.add_argument('--flakiness-dashboard-server', |
90 dest='flakiness_dashboard_server', | 95 dest='flakiness_dashboard_server', |
91 help=('Address of the server that is hosting the ' | 96 help=('Address of the server that is hosting the ' |
92 'Chrome for Android flakiness dashboard.')) | 97 'Chrome for Android flakiness dashboard.')) |
(...skipping 120 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
213 help='Runtime data dependency file from GN.') | 218 help='Runtime data dependency file from GN.') |
214 group.add_argument('--app-data-file', action='append', dest='app_data_files', | 219 group.add_argument('--app-data-file', action='append', dest='app_data_files', |
215 help='A file path relative to the app data directory ' | 220 help='A file path relative to the app data directory ' |
216 'that should be saved to the host.') | 221 'that should be saved to the host.') |
217 group.add_argument('--app-data-file-dir', | 222 group.add_argument('--app-data-file-dir', |
218 help='Host directory to which app data files will be' | 223 help='Host directory to which app data files will be' |
219 ' saved. Used with --app-data-file.') | 224 ' saved. Used with --app-data-file.') |
220 group.add_argument('--delete-stale-data', dest='delete_stale_data', | 225 group.add_argument('--delete-stale-data', dest='delete_stale_data', |
221 action='store_true', | 226 action='store_true', |
222 help='Delete stale test data on the device.') | 227 help='Delete stale test data on the device.') |
223 group.add_argument('--repeat', '--gtest_repeat', '--gtest-repeat', | |
224 dest='repeat', type=int, default=0, | |
225 help='Number of times to repeat the specified set of ' | |
226 'tests.') | |
227 group.add_argument('--break-on-failure', '--break_on_failure', | |
228 dest='break_on_failure', action='store_true', | |
229 help='Whether to break on failure.') | |
230 group.add_argument('--extract-test-list-from-filter', | 228 group.add_argument('--extract-test-list-from-filter', |
231 action='store_true', | 229 action='store_true', |
232 help='When a test filter is specified, and the list of ' | 230 help='When a test filter is specified, and the list of ' |
233 'tests can be determined from it, skip querying the ' | 231 'tests can be determined from it, skip querying the ' |
234 'device for the list of all tests. Speeds up local ' | 232 'device for the list of all tests. Speeds up local ' |
235 'development, but is not safe to use on bots (' | 233 'development, but is not safe to use on bots (' |
236 'http://crbug.com/549214') | 234 'http://crbug.com/549214') |
237 group.add_argument('--enable-xml-result-parsing', | 235 group.add_argument('--enable-xml-result-parsing', |
238 action='store_true', | 236 action='store_true', |
239 help=argparse.SUPPRESS) | 237 help=argparse.SUPPRESS) |
(...skipping 12 matching lines...) Expand all Loading... |
252 '//testing/buildbot/filters/README.md.') | 250 '//testing/buildbot/filters/README.md.') |
253 | 251 |
254 AddDeviceOptions(parser) | 252 AddDeviceOptions(parser) |
255 AddCommonOptions(parser) | 253 AddCommonOptions(parser) |
256 | 254 |
257 | 255 |
258 def AddLinkerTestOptions(parser): | 256 def AddLinkerTestOptions(parser): |
259 group = parser.add_argument_group('Linker Test Options') | 257 group = parser.add_argument_group('Linker Test Options') |
260 group.add_argument('-f', '--gtest-filter', dest='test_filter', | 258 group.add_argument('-f', '--gtest-filter', dest='test_filter', |
261 help='googletest-style filter string.') | 259 help='googletest-style filter string.') |
| 260 group.add_argument('--test-apk', type=os.path.realpath, |
| 261 help='Path to the linker test APK.') |
262 AddCommonOptions(parser) | 262 AddCommonOptions(parser) |
263 AddDeviceOptions(parser) | 263 AddDeviceOptions(parser) |
264 | 264 |
265 | 265 |
266 def AddJavaTestOptions(argument_group): | 266 def AddJavaTestOptions(argument_group): |
267 """Adds the Java test options to |option_parser|.""" | 267 """Adds the Java test options to |option_parser|.""" |
268 | 268 |
269 argument_group.add_argument( | 269 argument_group.add_argument( |
270 '-f', '--test-filter', '--gtest_filter', '--gtest-filter', | 270 '-f', '--test-filter', '--gtest_filter', '--gtest-filter', |
271 dest='test_filter', | 271 dest='test_filter', |
272 help=('Test filter (if not fully qualified, will run all matches).')) | 272 help=('Test filter (if not fully qualified, will run all matches).')) |
273 argument_group.add_argument( | 273 argument_group.add_argument( |
274 '--repeat', '--gtest_repeat', '--gtest-repeat', dest='repeat', | |
275 type=int, default=0, | |
276 help='Number of times to repeat the specified set of tests.') | |
277 argument_group.add_argument( | |
278 '--break-on-failure', '--break_on_failure', | |
279 dest='break_on_failure', action='store_true', | |
280 help='Whether to break on failure.') | |
281 argument_group.add_argument( | |
282 '-A', '--annotation', dest='annotation_str', | 274 '-A', '--annotation', dest='annotation_str', |
283 help=('Comma-separated list of annotations. Run only tests with any of ' | 275 help=('Comma-separated list of annotations. Run only tests with any of ' |
284 'the given annotations. An annotation can be either a key or a ' | 276 'the given annotations. An annotation can be either a key or a ' |
285 'key-values pair. A test that has no annotation is considered ' | 277 'key-values pair. A test that has no annotation is considered ' |
286 '"SmallTest".')) | 278 '"SmallTest".')) |
287 argument_group.add_argument( | 279 argument_group.add_argument( |
288 '-E', '--exclude-annotation', dest='exclude_annotation_str', | 280 '-E', '--exclude-annotation', dest='exclude_annotation_str', |
289 help=('Comma-separated list of annotations. Exclude tests with these ' | 281 help=('Comma-separated list of annotations. Exclude tests with these ' |
290 'annotations.')) | 282 'annotations.')) |
291 argument_group.add_argument( | 283 argument_group.add_argument( |
(...skipping 122 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
414 group = parser.add_argument_group('JUnit Test Options') | 406 group = parser.add_argument_group('JUnit Test Options') |
415 group.add_argument( | 407 group.add_argument( |
416 '-s', '--test-suite', dest='test_suite', required=True, | 408 '-s', '--test-suite', dest='test_suite', required=True, |
417 help=('JUnit test suite to run.')) | 409 help=('JUnit test suite to run.')) |
418 group.add_argument( | 410 group.add_argument( |
419 '-f', '--test-filter', dest='test_filter', | 411 '-f', '--test-filter', dest='test_filter', |
420 help='Filters tests googletest-style.') | 412 help='Filters tests googletest-style.') |
421 group.add_argument( | 413 group.add_argument( |
422 '--package-filter', dest='package_filter', | 414 '--package-filter', dest='package_filter', |
423 help='Filters tests by package.') | 415 help='Filters tests by package.') |
424 # TODO(mikecase): Add --repeat and --break-on-failure to common options. | |
425 # These options are required for platform-mode support. | |
426 group.add_argument( | |
427 '--repeat', dest='repeat', type=int, default=0, | |
428 help='Number of times to repeat the specified set of tests.') | |
429 group.add_argument( | |
430 '--break-on-failure', '--break_on_failure', | |
431 dest='break_on_failure', action='store_true', | |
432 help='Whether to break on failure.') | |
433 group.add_argument( | 416 group.add_argument( |
434 '--runner-filter', dest='runner_filter', | 417 '--runner-filter', dest='runner_filter', |
435 help='Filters tests by runner class. Must be fully qualified.') | 418 help='Filters tests by runner class. Must be fully qualified.') |
436 group.add_argument( | 419 group.add_argument( |
437 '--coverage-dir', dest='coverage_dir', type=os.path.realpath, | 420 '--coverage-dir', dest='coverage_dir', type=os.path.realpath, |
438 help='Directory to store coverage info.') | 421 help='Directory to store coverage info.') |
439 AddCommonOptions(parser) | 422 AddCommonOptions(parser) |
440 | 423 |
441 | 424 |
442 def AddMonkeyTestOptions(parser): | 425 def AddMonkeyTestOptions(parser): |
(...skipping 10 matching lines...) Expand all Loading... |
453 '--category', nargs='*', dest='categories', default=[], | 436 '--category', nargs='*', dest='categories', default=[], |
454 help='A list of allowed categories. Monkey will only visit activities ' | 437 help='A list of allowed categories. Monkey will only visit activities ' |
455 'that are listed with one of the specified categories.') | 438 'that are listed with one of the specified categories.') |
456 group.add_argument( | 439 group.add_argument( |
457 '--throttle', default=100, type=int, | 440 '--throttle', default=100, type=int, |
458 help='Delay between events (ms) (default: %(default)s). ') | 441 help='Delay between events (ms) (default: %(default)s). ') |
459 group.add_argument( | 442 group.add_argument( |
460 '--seed', type=int, | 443 '--seed', type=int, |
461 help='Seed value for pseudo-random generator. Same seed value generates ' | 444 help='Seed value for pseudo-random generator. Same seed value generates ' |
462 'the same sequence of events. Seed is randomized by default.') | 445 'the same sequence of events. Seed is randomized by default.') |
463 group.add_argument( | |
464 '--repeat', dest='repeat', type=int, default=0, | |
465 help='Number of times to repeat the specified set of tests.') | |
466 group.add_argument( | |
467 '--break-on-failure', '--break_on_failure', | |
468 dest='break_on_failure', action='store_true', | |
469 help='Whether to break on failure.') | |
470 AddCommonOptions(parser) | 446 AddCommonOptions(parser) |
471 AddDeviceOptions(parser) | 447 AddDeviceOptions(parser) |
472 | 448 |
473 | 449 |
474 def AddPerfTestOptions(parser): | 450 def AddPerfTestOptions(parser): |
475 """Adds perf test options to |parser|.""" | 451 """Adds perf test options to |parser|.""" |
476 | 452 |
477 group = parser.add_argument_group('Perf Test Options') | 453 group = parser.add_argument_group('Perf Test Options') |
478 | 454 |
479 class SingleStepAction(argparse.Action): | 455 class SingleStepAction(argparse.Action): |
(...skipping 72 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
552 'temperature (0.1 C)') | 528 'temperature (0.1 C)') |
553 group.add_argument( | 529 group.add_argument( |
554 'single_step_command', nargs='*', action=SingleStepAction, | 530 'single_step_command', nargs='*', action=SingleStepAction, |
555 help='If --single-step is specified, the command to run.') | 531 help='If --single-step is specified, the command to run.') |
556 group.add_argument( | 532 group.add_argument( |
557 '--min-battery-level', type=int, | 533 '--min-battery-level', type=int, |
558 help='Only starts tests when the battery is charged above ' | 534 help='Only starts tests when the battery is charged above ' |
559 'given level.') | 535 'given level.') |
560 group.add_argument('--known-devices-file', help='Path to known device list.') | 536 group.add_argument('--known-devices-file', help='Path to known device list.') |
561 group.add_argument( | 537 group.add_argument( |
562 '--repeat', dest='repeat', type=int, default=0, | |
563 help='Number of times to repeat the specified set of tests.') | |
564 group.add_argument( | |
565 '--break-on-failure', '--break_on_failure', dest='break_on_failure', | |
566 action='store_true', help='Whether to break on failure.') | |
567 group.add_argument( | |
568 '--write-buildbot-json', action='store_true', | 538 '--write-buildbot-json', action='store_true', |
569 help='Whether to output buildbot json.') | 539 help='Whether to output buildbot json.') |
570 # TODO(rnephew): Move up to top level options when implemented on all tests. | 540 # TODO(rnephew): Move up to top level options when implemented on all tests. |
571 group.add_argument( | 541 group.add_argument( |
572 '--trace-output', metavar='FILENAME', type=os.path.realpath, | 542 '--trace-output', metavar='FILENAME', type=os.path.realpath, |
573 help='Path to save test_runner trace data to.') | 543 help='Path to save test_runner trace data to.') |
574 AddCommonOptions(parser) | 544 AddCommonOptions(parser) |
575 AddDeviceOptions(parser) | 545 AddDeviceOptions(parser) |
576 | 546 |
577 | 547 |
578 def AddPythonTestOptions(parser): | 548 def AddPythonTestOptions(parser): |
579 group = parser.add_argument_group('Python Test Options') | 549 group = parser.add_argument_group('Python Test Options') |
580 group.add_argument( | 550 group.add_argument( |
581 '-s', '--suite', dest='suite_name', metavar='SUITE_NAME', | 551 '-s', '--suite', dest='suite_name', metavar='SUITE_NAME', |
582 choices=constants.PYTHON_UNIT_TEST_SUITES.keys(), | 552 choices=constants.PYTHON_UNIT_TEST_SUITES.keys(), |
583 help='Name of the test suite to run.') | 553 help='Name of the test suite to run.') |
584 AddCommonOptions(parser) | 554 AddCommonOptions(parser) |
585 | 555 |
586 | 556 |
587 def _RunLinkerTests(args, devices): | |
588 """Subcommand of RunTestsCommands which runs linker tests.""" | |
589 runner_factory, tests = linker_setup.Setup(args, devices) | |
590 | |
591 results, exit_code = test_dispatcher.RunTests( | |
592 tests, runner_factory, devices, shard=True, test_timeout=60, | |
593 num_retries=args.num_retries) | |
594 | |
595 report_results.LogFull( | |
596 results=results, | |
597 test_type='Linker test', | |
598 test_package='ChromiumLinkerTest') | |
599 | |
600 if args.json_results_file: | |
601 json_results.GenerateJsonResultsFile([results], args.json_results_file) | |
602 | |
603 return exit_code | |
604 | |
605 | |
606 def _RunPythonTests(args): | 557 def _RunPythonTests(args): |
607 """Subcommand of RunTestsCommand which runs python unit tests.""" | 558 """Subcommand of RunTestsCommand which runs python unit tests.""" |
608 suite_vars = constants.PYTHON_UNIT_TEST_SUITES[args.suite_name] | 559 suite_vars = constants.PYTHON_UNIT_TEST_SUITES[args.suite_name] |
609 suite_path = suite_vars['path'] | 560 suite_path = suite_vars['path'] |
610 suite_test_modules = suite_vars['test_modules'] | 561 suite_test_modules = suite_vars['test_modules'] |
611 | 562 |
612 sys.path = [suite_path] + sys.path | 563 sys.path = [suite_path] + sys.path |
613 try: | 564 try: |
614 suite = unittest.TestSuite() | 565 suite = unittest.TestSuite() |
615 suite.addTests(unittest.defaultTestLoader.loadTestsFromName(m) | 566 suite.addTests(unittest.defaultTestLoader.loadTestsFromName(m) |
(...skipping 30 matching lines...) Expand all Loading... |
646 % (test_device, ', '.join(attached_devices))) | 597 % (test_device, ', '.join(attached_devices))) |
647 return test_device | 598 return test_device |
648 | 599 |
649 else: | 600 else: |
650 if not attached_devices: | 601 if not attached_devices: |
651 raise device_errors.NoDevicesError() | 602 raise device_errors.NoDevicesError() |
652 return sorted(attached_devices) | 603 return sorted(attached_devices) |
653 | 604 |
654 | 605 |
655 _DEFAULT_PLATFORM_MODE_TESTS = ['gtest', 'instrumentation', 'junit', | 606 _DEFAULT_PLATFORM_MODE_TESTS = ['gtest', 'instrumentation', 'junit', |
656 'monkey', 'perf'] | 607 'linker', 'monkey', 'perf'] |
657 | 608 |
658 | 609 |
659 def RunTestsCommand(args): # pylint: disable=too-many-return-statements | 610 def RunTestsCommand(args): # pylint: disable=too-many-return-statements |
660 """Checks test type and dispatches to the appropriate function. | 611 """Checks test type and dispatches to the appropriate function. |
661 | 612 |
662 Args: | 613 Args: |
663 args: argparse.Namespace object. | 614 args: argparse.Namespace object. |
664 | 615 |
665 Returns: | 616 Returns: |
666 Integer indicated exit code. | 617 Integer indicated exit code. |
(...skipping 11 matching lines...) Expand all Loading... |
678 | 629 |
679 forwarder.Forwarder.RemoveHostLog() | 630 forwarder.Forwarder.RemoveHostLog() |
680 if not ports.ResetTestServerPortAllocation(): | 631 if not ports.ResetTestServerPortAllocation(): |
681 raise Exception('Failed to reset test server port.') | 632 raise Exception('Failed to reset test server port.') |
682 | 633 |
683 # pylint: disable=protected-access | 634 # pylint: disable=protected-access |
684 if os.path.exists(ports._TEST_SERVER_PORT_LOCKFILE): | 635 if os.path.exists(ports._TEST_SERVER_PORT_LOCKFILE): |
685 os.unlink(ports._TEST_SERVER_PORT_LOCKFILE) | 636 os.unlink(ports._TEST_SERVER_PORT_LOCKFILE) |
686 # pylint: enable=protected-access | 637 # pylint: enable=protected-access |
687 | 638 |
688 def get_devices(): | 639 if command == 'python': |
689 return _GetAttachedDevices(args.blacklist_file, args.test_device, | |
690 args.enable_device_cache, args.num_retries) | |
691 | |
692 if command == 'linker': | |
693 return _RunLinkerTests(args, get_devices()) | |
694 elif command == 'python': | |
695 return _RunPythonTests(args) | 640 return _RunPythonTests(args) |
696 else: | 641 else: |
697 raise Exception('Unknown test type.') | 642 raise Exception('Unknown test type.') |
698 | 643 |
699 | 644 |
700 _SUPPORTED_IN_PLATFORM_MODE = [ | 645 _SUPPORTED_IN_PLATFORM_MODE = [ |
701 # TODO(jbudorick): Add support for more test types. | 646 # TODO(jbudorick): Add support for more test types. |
702 'gtest', | 647 'gtest', |
703 'instrumentation', | 648 'instrumentation', |
704 'junit', | 649 'junit', |
| 650 'linker', |
705 'monkey', | 651 'monkey', |
706 'perf', | 652 'perf', |
707 ] | 653 ] |
708 | 654 |
709 | 655 |
710 def RunTestsInPlatformMode(args): | 656 def RunTestsInPlatformMode(args): |
711 | 657 |
712 def infra_error(message): | 658 def infra_error(message): |
713 logging.fatal(message) | 659 logging.fatal(message) |
714 sys.exit(constants.INFRA_EXIT_CODE) | 660 sys.exit(constants.INFRA_EXIT_CODE) |
(...skipping 175 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
890 if e.is_infra_error: | 836 if e.is_infra_error: |
891 return constants.INFRA_EXIT_CODE | 837 return constants.INFRA_EXIT_CODE |
892 return constants.ERROR_EXIT_CODE | 838 return constants.ERROR_EXIT_CODE |
893 except: # pylint: disable=W0702 | 839 except: # pylint: disable=W0702 |
894 logging.exception('Unrecognized error occurred.') | 840 logging.exception('Unrecognized error occurred.') |
895 return constants.ERROR_EXIT_CODE | 841 return constants.ERROR_EXIT_CODE |
896 | 842 |
897 | 843 |
898 if __name__ == '__main__': | 844 if __name__ == '__main__': |
899 sys.exit(main()) | 845 sys.exit(main()) |
OLD | NEW |