OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # | 2 # |
3 # Copyright 2013 The Chromium Authors. All rights reserved. | 3 # Copyright 2013 The Chromium Authors. All rights reserved. |
4 # Use of this source code is governed by a BSD-style license that can be | 4 # Use of this source code is governed by a BSD-style license that can be |
5 # found in the LICENSE file. | 5 # found in the LICENSE file. |
6 | 6 |
7 """Runs all types of tests from one unified interface.""" | 7 """Runs all types of tests from one unified interface.""" |
8 | 8 |
9 import argparse | 9 import argparse |
10 import collections | 10 import collections |
11 import logging | 11 import logging |
12 import os | 12 import os |
13 import shutil | 13 import shutil |
14 import signal | 14 import signal |
15 import sys | 15 import sys |
16 import threading | 16 import threading |
17 import unittest | 17 import unittest |
18 | 18 |
19 from pylib import constants | 19 from pylib import constants |
20 from pylib import forwarder | 20 from pylib import forwarder |
21 from pylib import ports | 21 from pylib import ports |
22 from pylib.base import base_test_result | 22 from pylib.base import base_test_result |
23 from pylib.base import environment_factory | 23 from pylib.base import environment_factory |
24 from pylib.base import test_dispatcher | 24 from pylib.base import test_dispatcher |
25 from pylib.base import test_instance_factory | 25 from pylib.base import test_instance_factory |
26 from pylib.base import test_run_factory | 26 from pylib.base import test_run_factory |
| 27 from pylib.device import device_blacklist |
27 from pylib.device import device_errors | 28 from pylib.device import device_errors |
28 from pylib.device import device_utils | 29 from pylib.device import device_utils |
29 from pylib.gtest import gtest_config | 30 from pylib.gtest import gtest_config |
30 # TODO(jbudorick): Remove this once we stop selectively enabling platform mode. | 31 # TODO(jbudorick): Remove this once we stop selectively enabling platform mode. |
31 from pylib.gtest import gtest_test_instance | 32 from pylib.gtest import gtest_test_instance |
32 from pylib.gtest import setup as gtest_setup | 33 from pylib.gtest import setup as gtest_setup |
33 from pylib.gtest import test_options as gtest_test_options | 34 from pylib.gtest import test_options as gtest_test_options |
34 from pylib.linker import setup as linker_setup | 35 from pylib.linker import setup as linker_setup |
35 from pylib.host_driven import setup as host_driven_setup | 36 from pylib.host_driven import setup as host_driven_setup |
36 from pylib.instrumentation import setup as instrumentation_setup | 37 from pylib.instrumentation import setup as instrumentation_setup |
(...skipping 144 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
181 def AddDeviceOptions(parser): | 182 def AddDeviceOptions(parser): |
182 """Adds device options to |parser|.""" | 183 """Adds device options to |parser|.""" |
183 group = parser.add_argument_group(title='Device Options') | 184 group = parser.add_argument_group(title='Device Options') |
184 group.add_argument('--tool', | 185 group.add_argument('--tool', |
185 dest='tool', | 186 dest='tool', |
186 help=('Run the test under a tool ' | 187 help=('Run the test under a tool ' |
187 '(use --tool help to list them)')) | 188 '(use --tool help to list them)')) |
188 group.add_argument('-d', '--device', dest='test_device', | 189 group.add_argument('-d', '--device', dest='test_device', |
189 help=('Target device for the test suite ' | 190 help=('Target device for the test suite ' |
190 'to run on.')) | 191 'to run on.')) |
| 192 group.add_argument('--blacklist-file', help='Device blacklist file.') |
191 | 193 |
192 | 194 |
193 def AddGTestOptions(parser): | 195 def AddGTestOptions(parser): |
194 """Adds gtest options to |parser|.""" | 196 """Adds gtest options to |parser|.""" |
195 | 197 |
196 gtest_suites = list(gtest_config.STABLE_TEST_SUITES | 198 gtest_suites = list(gtest_config.STABLE_TEST_SUITES |
197 + gtest_config.EXPERIMENTAL_TEST_SUITES) | 199 + gtest_config.EXPERIMENTAL_TEST_SUITES) |
198 | 200 |
199 group = parser.add_argument_group('GTest Options') | 201 group = parser.add_argument_group('GTest Options') |
200 group.add_argument('-s', '--suite', dest='suite_name', | 202 group.add_argument('-s', '--suite', dest='suite_name', |
(...skipping 559 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
760 if args.json_results_file: | 762 if args.json_results_file: |
761 json_results.GenerateJsonResultsFile(results, args.json_results_file) | 763 json_results.GenerateJsonResultsFile(results, args.json_results_file) |
762 | 764 |
763 return exit_code | 765 return exit_code |
764 | 766 |
765 | 767 |
766 def _RunUIAutomatorTests(args, devices): | 768 def _RunUIAutomatorTests(args, devices): |
767 """Subcommand of RunTestsCommands which runs uiautomator tests.""" | 769 """Subcommand of RunTestsCommands which runs uiautomator tests.""" |
768 uiautomator_options = ProcessUIAutomatorOptions(args) | 770 uiautomator_options = ProcessUIAutomatorOptions(args) |
769 | 771 |
770 runner_factory, tests = uiautomator_setup.Setup(uiautomator_options) | 772 runner_factory, tests = uiautomator_setup.Setup(uiautomator_options, devices) |
771 | 773 |
772 results, exit_code = test_dispatcher.RunTests( | 774 results, exit_code = test_dispatcher.RunTests( |
773 tests, runner_factory, devices, shard=True, test_timeout=None, | 775 tests, runner_factory, devices, shard=True, test_timeout=None, |
774 num_retries=args.num_retries) | 776 num_retries=args.num_retries) |
775 | 777 |
776 report_results.LogFull( | 778 report_results.LogFull( |
777 results=results, | 779 results=results, |
778 test_type='UIAutomator', | 780 test_type='UIAutomator', |
779 test_package=os.path.basename(args.test_jar), | 781 test_package=os.path.basename(args.test_jar), |
780 annotation=args.annotations, | 782 annotation=args.annotations, |
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
816 results=results, | 818 results=results, |
817 test_type='Monkey', | 819 test_type='Monkey', |
818 test_package='Monkey') | 820 test_package='Monkey') |
819 | 821 |
820 if args.json_results_file: | 822 if args.json_results_file: |
821 json_results.GenerateJsonResultsFile(results, args.json_results_file) | 823 json_results.GenerateJsonResultsFile(results, args.json_results_file) |
822 | 824 |
823 return exit_code | 825 return exit_code |
824 | 826 |
825 | 827 |
826 def _RunPerfTests(args): | 828 def _RunPerfTests(args, active_devices): |
827 """Subcommand of RunTestsCommands which runs perf tests.""" | 829 """Subcommand of RunTestsCommands which runs perf tests.""" |
828 perf_options = ProcessPerfTestOptions(args) | 830 perf_options = ProcessPerfTestOptions(args) |
829 | 831 |
830 # Just save a simple json with a list of test names. | 832 # Just save a simple json with a list of test names. |
831 if perf_options.output_json_list: | 833 if perf_options.output_json_list: |
832 return perf_test_runner.OutputJsonList( | 834 return perf_test_runner.OutputJsonList( |
833 perf_options.steps, perf_options.output_json_list) | 835 perf_options.steps, perf_options.output_json_list) |
834 | 836 |
835 # Just print the results from a single previously executed step. | 837 # Just print the results from a single previously executed step. |
836 if perf_options.print_step: | 838 if perf_options.print_step: |
837 return perf_test_runner.PrintTestOutput( | 839 return perf_test_runner.PrintTestOutput( |
838 perf_options.print_step, perf_options.output_chartjson_data) | 840 perf_options.print_step, perf_options.output_chartjson_data) |
839 | 841 |
840 runner_factory, tests, devices = perf_setup.Setup(perf_options) | 842 runner_factory, tests, devices = perf_setup.Setup( |
| 843 perf_options, active_devices) |
841 | 844 |
842 # shard=False means that each device will get the full list of tests | 845 # shard=False means that each device will get the full list of tests |
843 # and then each one will decide their own affinity. | 846 # and then each one will decide their own affinity. |
844 # shard=True means each device will pop the next test available from a queue, | 847 # shard=True means each device will pop the next test available from a queue, |
845 # which increases throughput but have no affinity. | 848 # which increases throughput but have no affinity. |
846 results, _ = test_dispatcher.RunTests( | 849 results, _ = test_dispatcher.RunTests( |
847 tests, runner_factory, devices, shard=False, test_timeout=None, | 850 tests, runner_factory, devices, shard=False, test_timeout=None, |
848 num_retries=args.num_retries) | 851 num_retries=args.num_retries) |
849 | 852 |
850 report_results.LogFull( | 853 report_results.LogFull( |
(...skipping 24 matching lines...) Expand all Loading... |
875 try: | 878 try: |
876 suite = unittest.TestSuite() | 879 suite = unittest.TestSuite() |
877 suite.addTests(unittest.defaultTestLoader.loadTestsFromName(m) | 880 suite.addTests(unittest.defaultTestLoader.loadTestsFromName(m) |
878 for m in suite_test_modules) | 881 for m in suite_test_modules) |
879 runner = unittest.TextTestRunner(verbosity=1+args.verbose_count) | 882 runner = unittest.TextTestRunner(verbosity=1+args.verbose_count) |
880 return 0 if runner.run(suite).wasSuccessful() else 1 | 883 return 0 if runner.run(suite).wasSuccessful() else 1 |
881 finally: | 884 finally: |
882 sys.path = sys.path[1:] | 885 sys.path = sys.path[1:] |
883 | 886 |
884 | 887 |
885 def _GetAttachedDevices(test_device=None): | 888 def _GetAttachedDevices(blacklist_file, test_device): |
886 """Get all attached devices. | 889 """Get all attached devices. |
887 | 890 |
888 Args: | 891 Args: |
889 test_device: Name of a specific device to use. | 892 test_device: Name of a specific device to use. |
890 | 893 |
891 Returns: | 894 Returns: |
892 A list of attached devices. | 895 A list of attached devices. |
893 """ | 896 """ |
894 attached_devices = device_utils.DeviceUtils.HealthyDevices() | 897 if not blacklist_file: |
| 898 # TODO(jbudorick): Remove this once bots pass the blacklist file. |
| 899 blacklist_file = device_blacklist.BLACKLIST_JSON |
| 900 logging.warning('Using default device blacklist %s', |
| 901 device_blacklist.BLACKLIST_JSON) |
| 902 |
| 903 blacklist = device_blacklist.Blacklist(blacklist_file) |
| 904 attached_devices = device_utils.DeviceUtils.HealthyDevices(blacklist) |
895 if test_device: | 905 if test_device: |
896 test_device = [d for d in attached_devices if d == test_device] | 906 test_device = [d for d in attached_devices if d == test_device] |
897 if not test_device: | 907 if not test_device: |
898 raise device_errors.DeviceUnreachableError( | 908 raise device_errors.DeviceUnreachableError( |
899 'Did not find device %s among attached device. Attached devices: %s' | 909 'Did not find device %s among attached device. Attached devices: %s' |
900 % (test_device, ', '.join(attached_devices))) | 910 % (test_device, ', '.join(attached_devices))) |
901 return test_device | 911 return test_device |
902 | 912 |
903 else: | 913 else: |
904 if not attached_devices: | 914 if not attached_devices: |
(...skipping 18 matching lines...) Expand all Loading... |
923 command = args.command | 933 command = args.command |
924 | 934 |
925 ProcessCommonOptions(args) | 935 ProcessCommonOptions(args) |
926 | 936 |
927 if args.enable_platform_mode: | 937 if args.enable_platform_mode: |
928 return RunTestsInPlatformMode(args, parser) | 938 return RunTestsInPlatformMode(args, parser) |
929 | 939 |
930 if command in constants.LOCAL_MACHINE_TESTS: | 940 if command in constants.LOCAL_MACHINE_TESTS: |
931 devices = [] | 941 devices = [] |
932 else: | 942 else: |
933 devices = _GetAttachedDevices(args.test_device) | 943 devices = _GetAttachedDevices(args.blacklist_file, args.test_device) |
934 | 944 |
935 forwarder.Forwarder.RemoveHostLog() | 945 forwarder.Forwarder.RemoveHostLog() |
936 if not ports.ResetTestServerPortAllocation(): | 946 if not ports.ResetTestServerPortAllocation(): |
937 raise Exception('Failed to reset test server port.') | 947 raise Exception('Failed to reset test server port.') |
938 | 948 |
939 if command == 'gtest': | 949 if command == 'gtest': |
940 if args.suite_name[0] in gtest_test_instance.BROWSER_TEST_SUITES: | 950 if args.suite_name[0] in gtest_test_instance.BROWSER_TEST_SUITES: |
941 return RunTestsInPlatformMode(args, parser) | 951 return RunTestsInPlatformMode(args, parser) |
942 return _RunGTests(args, devices) | 952 return _RunGTests(args, devices) |
943 elif command == 'linker': | 953 elif command == 'linker': |
944 return _RunLinkerTests(args, devices) | 954 return _RunLinkerTests(args, devices) |
945 elif command == 'instrumentation': | 955 elif command == 'instrumentation': |
946 return _RunInstrumentationTests(args, devices) | 956 return _RunInstrumentationTests(args, devices) |
947 elif command == 'uiautomator': | 957 elif command == 'uiautomator': |
948 return _RunUIAutomatorTests(args, devices) | 958 return _RunUIAutomatorTests(args, devices) |
949 elif command == 'junit': | 959 elif command == 'junit': |
950 return _RunJUnitTests(args) | 960 return _RunJUnitTests(args) |
951 elif command == 'monkey': | 961 elif command == 'monkey': |
952 return _RunMonkeyTests(args, devices) | 962 return _RunMonkeyTests(args, devices) |
953 elif command == 'perf': | 963 elif command == 'perf': |
954 return _RunPerfTests(args) | 964 return _RunPerfTests(args, devices) |
955 elif command == 'python': | 965 elif command == 'python': |
956 return _RunPythonTests(args) | 966 return _RunPythonTests(args) |
957 else: | 967 else: |
958 raise Exception('Unknown test type.') | 968 raise Exception('Unknown test type.') |
959 | 969 |
960 | 970 |
961 _SUPPORTED_IN_PLATFORM_MODE = [ | 971 _SUPPORTED_IN_PLATFORM_MODE = [ |
962 # TODO(jbudorick): Add support for more test types. | 972 # TODO(jbudorick): Add support for more test types. |
963 'gtest', | 973 'gtest', |
964 'instrumentation', | 974 'instrumentation', |
(...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1055 if e.is_infra_error: | 1065 if e.is_infra_error: |
1056 return constants.INFRA_EXIT_CODE | 1066 return constants.INFRA_EXIT_CODE |
1057 return constants.ERROR_EXIT_CODE | 1067 return constants.ERROR_EXIT_CODE |
1058 except: # pylint: disable=W0702 | 1068 except: # pylint: disable=W0702 |
1059 logging.exception('Unrecognized error occurred.') | 1069 logging.exception('Unrecognized error occurred.') |
1060 return constants.ERROR_EXIT_CODE | 1070 return constants.ERROR_EXIT_CODE |
1061 | 1071 |
1062 | 1072 |
1063 if __name__ == '__main__': | 1073 if __name__ == '__main__': |
1064 sys.exit(main()) | 1074 sys.exit(main()) |
OLD | NEW |