OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # | 2 # |
3 # Copyright 2013 The Chromium Authors. All rights reserved. | 3 # Copyright 2013 The Chromium Authors. All rights reserved. |
4 # Use of this source code is governed by a BSD-style license that can be | 4 # Use of this source code is governed by a BSD-style license that can be |
5 # found in the LICENSE file. | 5 # found in the LICENSE file. |
6 | 6 |
7 """Runs all types of tests from one unified interface.""" | 7 """Runs all types of tests from one unified interface.""" |
8 | 8 |
9 import argparse | 9 import argparse |
10 import collections | 10 import collections |
11 import logging | 11 import logging |
12 import os | 12 import os |
13 import shutil | 13 import shutil |
14 import signal | 14 import signal |
15 import sys | 15 import sys |
16 import threading | 16 import threading |
17 import unittest | 17 import unittest |
18 | 18 |
19 from pylib import constants | 19 from pylib import constants |
20 from pylib import forwarder | 20 from pylib import forwarder |
21 from pylib import ports | 21 from pylib import ports |
22 from pylib.base import base_test_result | 22 from pylib.base import base_test_result |
23 from pylib.base import environment_factory | 23 from pylib.base import environment_factory |
24 from pylib.base import test_dispatcher | 24 from pylib.base import test_dispatcher |
25 from pylib.base import test_instance_factory | 25 from pylib.base import test_instance_factory |
26 from pylib.base import test_run_factory | 26 from pylib.base import test_run_factory |
27 from pylib.device import device_blacklist | |
28 from pylib.device import device_errors | 27 from pylib.device import device_errors |
29 from pylib.device import device_utils | 28 from pylib.device import device_utils |
30 from pylib.gtest import gtest_config | 29 from pylib.gtest import gtest_config |
31 # TODO(jbudorick): Remove this once we stop selectively enabling platform mode. | 30 # TODO(jbudorick): Remove this once we stop selectively enabling platform mode. |
32 from pylib.gtest import gtest_test_instance | 31 from pylib.gtest import gtest_test_instance |
33 from pylib.gtest import setup as gtest_setup | 32 from pylib.gtest import setup as gtest_setup |
34 from pylib.gtest import test_options as gtest_test_options | 33 from pylib.gtest import test_options as gtest_test_options |
35 from pylib.linker import setup as linker_setup | 34 from pylib.linker import setup as linker_setup |
36 from pylib.host_driven import setup as host_driven_setup | 35 from pylib.host_driven import setup as host_driven_setup |
37 from pylib.instrumentation import setup as instrumentation_setup | 36 from pylib.instrumentation import setup as instrumentation_setup |
(...skipping 144 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
182 def AddDeviceOptions(parser): | 181 def AddDeviceOptions(parser): |
183 """Adds device options to |parser|.""" | 182 """Adds device options to |parser|.""" |
184 group = parser.add_argument_group(title='Device Options') | 183 group = parser.add_argument_group(title='Device Options') |
185 group.add_argument('--tool', | 184 group.add_argument('--tool', |
186 dest='tool', | 185 dest='tool', |
187 help=('Run the test under a tool ' | 186 help=('Run the test under a tool ' |
188 '(use --tool help to list them)')) | 187 '(use --tool help to list them)')) |
189 group.add_argument('-d', '--device', dest='test_device', | 188 group.add_argument('-d', '--device', dest='test_device', |
190 help=('Target device for the test suite ' | 189 help=('Target device for the test suite ' |
191 'to run on.')) | 190 'to run on.')) |
192 group.add_argument('--blacklist-file', help='Device blacklist file.') | |
193 | 191 |
194 | 192 |
195 def AddGTestOptions(parser): | 193 def AddGTestOptions(parser): |
196 """Adds gtest options to |parser|.""" | 194 """Adds gtest options to |parser|.""" |
197 | 195 |
198 gtest_suites = list(gtest_config.STABLE_TEST_SUITES | 196 gtest_suites = list(gtest_config.STABLE_TEST_SUITES |
199 + gtest_config.EXPERIMENTAL_TEST_SUITES) | 197 + gtest_config.EXPERIMENTAL_TEST_SUITES) |
200 | 198 |
201 group = parser.add_argument_group('GTest Options') | 199 group = parser.add_argument_group('GTest Options') |
202 group.add_argument('-s', '--suite', dest='suite_name', | 200 group.add_argument('-s', '--suite', dest='suite_name', |
(...skipping 559 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
762 if args.json_results_file: | 760 if args.json_results_file: |
763 json_results.GenerateJsonResultsFile(results, args.json_results_file) | 761 json_results.GenerateJsonResultsFile(results, args.json_results_file) |
764 | 762 |
765 return exit_code | 763 return exit_code |
766 | 764 |
767 | 765 |
768 def _RunUIAutomatorTests(args, devices): | 766 def _RunUIAutomatorTests(args, devices): |
769 """Subcommand of RunTestsCommands which runs uiautomator tests.""" | 767 """Subcommand of RunTestsCommands which runs uiautomator tests.""" |
770 uiautomator_options = ProcessUIAutomatorOptions(args) | 768 uiautomator_options = ProcessUIAutomatorOptions(args) |
771 | 769 |
772 runner_factory, tests = uiautomator_setup.Setup(uiautomator_options, devices) | 770 runner_factory, tests = uiautomator_setup.Setup(uiautomator_options) |
773 | 771 |
774 results, exit_code = test_dispatcher.RunTests( | 772 results, exit_code = test_dispatcher.RunTests( |
775 tests, runner_factory, devices, shard=True, test_timeout=None, | 773 tests, runner_factory, devices, shard=True, test_timeout=None, |
776 num_retries=args.num_retries) | 774 num_retries=args.num_retries) |
777 | 775 |
778 report_results.LogFull( | 776 report_results.LogFull( |
779 results=results, | 777 results=results, |
780 test_type='UIAutomator', | 778 test_type='UIAutomator', |
781 test_package=os.path.basename(args.test_jar), | 779 test_package=os.path.basename(args.test_jar), |
782 annotation=args.annotations, | 780 annotation=args.annotations, |
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
818 results=results, | 816 results=results, |
819 test_type='Monkey', | 817 test_type='Monkey', |
820 test_package='Monkey') | 818 test_package='Monkey') |
821 | 819 |
822 if args.json_results_file: | 820 if args.json_results_file: |
823 json_results.GenerateJsonResultsFile(results, args.json_results_file) | 821 json_results.GenerateJsonResultsFile(results, args.json_results_file) |
824 | 822 |
825 return exit_code | 823 return exit_code |
826 | 824 |
827 | 825 |
828 def _RunPerfTests(args, active_devices): | 826 def _RunPerfTests(args): |
829 """Subcommand of RunTestsCommands which runs perf tests.""" | 827 """Subcommand of RunTestsCommands which runs perf tests.""" |
830 perf_options = ProcessPerfTestOptions(args) | 828 perf_options = ProcessPerfTestOptions(args) |
831 | 829 |
832 # Just save a simple json with a list of test names. | 830 # Just save a simple json with a list of test names. |
833 if perf_options.output_json_list: | 831 if perf_options.output_json_list: |
834 return perf_test_runner.OutputJsonList( | 832 return perf_test_runner.OutputJsonList( |
835 perf_options.steps, perf_options.output_json_list) | 833 perf_options.steps, perf_options.output_json_list) |
836 | 834 |
837 # Just print the results from a single previously executed step. | 835 # Just print the results from a single previously executed step. |
838 if perf_options.print_step: | 836 if perf_options.print_step: |
839 return perf_test_runner.PrintTestOutput( | 837 return perf_test_runner.PrintTestOutput( |
840 perf_options.print_step, perf_options.output_chartjson_data) | 838 perf_options.print_step, perf_options.output_chartjson_data) |
841 | 839 |
842 runner_factory, tests, devices = perf_setup.Setup( | 840 runner_factory, tests, devices = perf_setup.Setup(perf_options) |
843 perf_options, active_devices) | |
844 | 841 |
845 # shard=False means that each device will get the full list of tests | 842 # shard=False means that each device will get the full list of tests |
846 # and then each one will decide their own affinity. | 843 # and then each one will decide their own affinity. |
847 # shard=True means each device will pop the next test available from a queue, | 844 # shard=True means each device will pop the next test available from a queue, |
848 # which increases throughput but have no affinity. | 845 # which increases throughput but have no affinity. |
849 results, _ = test_dispatcher.RunTests( | 846 results, _ = test_dispatcher.RunTests( |
850 tests, runner_factory, devices, shard=False, test_timeout=None, | 847 tests, runner_factory, devices, shard=False, test_timeout=None, |
851 num_retries=args.num_retries) | 848 num_retries=args.num_retries) |
852 | 849 |
853 report_results.LogFull( | 850 report_results.LogFull( |
(...skipping 24 matching lines...) Expand all Loading... |
878 try: | 875 try: |
879 suite = unittest.TestSuite() | 876 suite = unittest.TestSuite() |
880 suite.addTests(unittest.defaultTestLoader.loadTestsFromName(m) | 877 suite.addTests(unittest.defaultTestLoader.loadTestsFromName(m) |
881 for m in suite_test_modules) | 878 for m in suite_test_modules) |
882 runner = unittest.TextTestRunner(verbosity=1+args.verbose_count) | 879 runner = unittest.TextTestRunner(verbosity=1+args.verbose_count) |
883 return 0 if runner.run(suite).wasSuccessful() else 1 | 880 return 0 if runner.run(suite).wasSuccessful() else 1 |
884 finally: | 881 finally: |
885 sys.path = sys.path[1:] | 882 sys.path = sys.path[1:] |
886 | 883 |
887 | 884 |
888 def _GetAttachedDevices(blacklist_file, test_device): | 885 def _GetAttachedDevices(test_device=None): |
889 """Get all attached devices. | 886 """Get all attached devices. |
890 | 887 |
891 Args: | 888 Args: |
892 test_device: Name of a specific device to use. | 889 test_device: Name of a specific device to use. |
893 | 890 |
894 Returns: | 891 Returns: |
895 A list of attached devices. | 892 A list of attached devices. |
896 """ | 893 """ |
897 if not blacklist_file: | 894 attached_devices = device_utils.DeviceUtils.HealthyDevices() |
898 # TODO(jbudorick): Remove this once bots pass the blacklist file. | |
899 blacklist_file = device_blacklist.BLACKLIST_JSON | |
900 logging.warning('Using default device blacklist %s', | |
901 device_blacklist.BLACKLIST_JSON) | |
902 | |
903 blacklist = device_blacklist.Blacklist(blacklist_file) | |
904 attached_devices = device_utils.DeviceUtils.HealthyDevices(blacklist) | |
905 if test_device: | 895 if test_device: |
906 test_device = [d for d in attached_devices if d == test_device] | 896 test_device = [d for d in attached_devices if d == test_device] |
907 if not test_device: | 897 if not test_device: |
908 raise device_errors.DeviceUnreachableError( | 898 raise device_errors.DeviceUnreachableError( |
909 'Did not find device %s among attached device. Attached devices: %s' | 899 'Did not find device %s among attached device. Attached devices: %s' |
910 % (test_device, ', '.join(attached_devices))) | 900 % (test_device, ', '.join(attached_devices))) |
911 return test_device | 901 return test_device |
912 | 902 |
913 else: | 903 else: |
914 if not attached_devices: | 904 if not attached_devices: |
(...skipping 18 matching lines...) Expand all Loading... |
933 command = args.command | 923 command = args.command |
934 | 924 |
935 ProcessCommonOptions(args) | 925 ProcessCommonOptions(args) |
936 | 926 |
937 if args.enable_platform_mode: | 927 if args.enable_platform_mode: |
938 return RunTestsInPlatformMode(args, parser) | 928 return RunTestsInPlatformMode(args, parser) |
939 | 929 |
940 if command in constants.LOCAL_MACHINE_TESTS: | 930 if command in constants.LOCAL_MACHINE_TESTS: |
941 devices = [] | 931 devices = [] |
942 else: | 932 else: |
943 devices = _GetAttachedDevices(args.blacklist_file, args.test_device) | 933 devices = _GetAttachedDevices(args.test_device) |
944 | 934 |
945 forwarder.Forwarder.RemoveHostLog() | 935 forwarder.Forwarder.RemoveHostLog() |
946 if not ports.ResetTestServerPortAllocation(): | 936 if not ports.ResetTestServerPortAllocation(): |
947 raise Exception('Failed to reset test server port.') | 937 raise Exception('Failed to reset test server port.') |
948 | 938 |
949 if command == 'gtest': | 939 if command == 'gtest': |
950 if args.suite_name[0] in gtest_test_instance.BROWSER_TEST_SUITES: | 940 if args.suite_name[0] in gtest_test_instance.BROWSER_TEST_SUITES: |
951 return RunTestsInPlatformMode(args, parser) | 941 return RunTestsInPlatformMode(args, parser) |
952 return _RunGTests(args, devices) | 942 return _RunGTests(args, devices) |
953 elif command == 'linker': | 943 elif command == 'linker': |
954 return _RunLinkerTests(args, devices) | 944 return _RunLinkerTests(args, devices) |
955 elif command == 'instrumentation': | 945 elif command == 'instrumentation': |
956 return _RunInstrumentationTests(args, devices) | 946 return _RunInstrumentationTests(args, devices) |
957 elif command == 'uiautomator': | 947 elif command == 'uiautomator': |
958 return _RunUIAutomatorTests(args, devices) | 948 return _RunUIAutomatorTests(args, devices) |
959 elif command == 'junit': | 949 elif command == 'junit': |
960 return _RunJUnitTests(args) | 950 return _RunJUnitTests(args) |
961 elif command == 'monkey': | 951 elif command == 'monkey': |
962 return _RunMonkeyTests(args, devices) | 952 return _RunMonkeyTests(args, devices) |
963 elif command == 'perf': | 953 elif command == 'perf': |
964 return _RunPerfTests(args, devices) | 954 return _RunPerfTests(args) |
965 elif command == 'python': | 955 elif command == 'python': |
966 return _RunPythonTests(args) | 956 return _RunPythonTests(args) |
967 else: | 957 else: |
968 raise Exception('Unknown test type.') | 958 raise Exception('Unknown test type.') |
969 | 959 |
970 | 960 |
971 _SUPPORTED_IN_PLATFORM_MODE = [ | 961 _SUPPORTED_IN_PLATFORM_MODE = [ |
972 # TODO(jbudorick): Add support for more test types. | 962 # TODO(jbudorick): Add support for more test types. |
973 'gtest', | 963 'gtest', |
974 'instrumentation', | 964 'instrumentation', |
(...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1065 if e.is_infra_error: | 1055 if e.is_infra_error: |
1066 return constants.INFRA_EXIT_CODE | 1056 return constants.INFRA_EXIT_CODE |
1067 return constants.ERROR_EXIT_CODE | 1057 return constants.ERROR_EXIT_CODE |
1068 except: # pylint: disable=W0702 | 1058 except: # pylint: disable=W0702 |
1069 logging.exception('Unrecognized error occurred.') | 1059 logging.exception('Unrecognized error occurred.') |
1070 return constants.ERROR_EXIT_CODE | 1060 return constants.ERROR_EXIT_CODE |
1071 | 1061 |
1072 | 1062 |
1073 if __name__ == '__main__': | 1063 if __name__ == '__main__': |
1074 sys.exit(main()) | 1064 sys.exit(main()) |
OLD | NEW |