| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # | 2 # |
| 3 # Copyright 2013 The Chromium Authors. All rights reserved. | 3 # Copyright 2013 The Chromium Authors. All rights reserved. |
| 4 # Use of this source code is governed by a BSD-style license that can be | 4 # Use of this source code is governed by a BSD-style license that can be |
| 5 # found in the LICENSE file. | 5 # found in the LICENSE file. |
| 6 | 6 |
| 7 """Runs all types of tests from one unified interface.""" | 7 """Runs all types of tests from one unified interface.""" |
| 8 | 8 |
| 9 import argparse | 9 import argparse |
| 10 import collections | 10 import collections |
| 11 import logging | 11 import logging |
| 12 import os | 12 import os |
| 13 import shutil | 13 import shutil |
| 14 import signal | 14 import signal |
| 15 import sys | 15 import sys |
| 16 import threading | 16 import threading |
| 17 import unittest | 17 import unittest |
| 18 | 18 |
| 19 from pylib import constants | 19 from pylib import constants |
| 20 from pylib import forwarder | 20 from pylib import forwarder |
| 21 from pylib import ports | 21 from pylib import ports |
| 22 from pylib.base import base_test_result | 22 from pylib.base import base_test_result |
| 23 from pylib.base import environment_factory | 23 from pylib.base import environment_factory |
| 24 from pylib.base import test_dispatcher | 24 from pylib.base import test_dispatcher |
| 25 from pylib.base import test_instance_factory | 25 from pylib.base import test_instance_factory |
| 26 from pylib.base import test_run_factory | 26 from pylib.base import test_run_factory |
| 27 from pylib.device import device_blacklist | |
| 28 from pylib.device import device_errors | 27 from pylib.device import device_errors |
| 29 from pylib.device import device_utils | 28 from pylib.device import device_utils |
| 30 from pylib.gtest import gtest_config | 29 from pylib.gtest import gtest_config |
| 31 # TODO(jbudorick): Remove this once we stop selectively enabling platform mode. | 30 # TODO(jbudorick): Remove this once we stop selectively enabling platform mode. |
| 32 from pylib.gtest import gtest_test_instance | 31 from pylib.gtest import gtest_test_instance |
| 33 from pylib.gtest import setup as gtest_setup | 32 from pylib.gtest import setup as gtest_setup |
| 34 from pylib.gtest import test_options as gtest_test_options | 33 from pylib.gtest import test_options as gtest_test_options |
| 35 from pylib.linker import setup as linker_setup | 34 from pylib.linker import setup as linker_setup |
| 36 from pylib.host_driven import setup as host_driven_setup | 35 from pylib.host_driven import setup as host_driven_setup |
| 37 from pylib.instrumentation import setup as instrumentation_setup | 36 from pylib.instrumentation import setup as instrumentation_setup |
| (...skipping 144 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 182 def AddDeviceOptions(parser): | 181 def AddDeviceOptions(parser): |
| 183 """Adds device options to |parser|.""" | 182 """Adds device options to |parser|.""" |
| 184 group = parser.add_argument_group(title='Device Options') | 183 group = parser.add_argument_group(title='Device Options') |
| 185 group.add_argument('--tool', | 184 group.add_argument('--tool', |
| 186 dest='tool', | 185 dest='tool', |
| 187 help=('Run the test under a tool ' | 186 help=('Run the test under a tool ' |
| 188 '(use --tool help to list them)')) | 187 '(use --tool help to list them)')) |
| 189 group.add_argument('-d', '--device', dest='test_device', | 188 group.add_argument('-d', '--device', dest='test_device', |
| 190 help=('Target device for the test suite ' | 189 help=('Target device for the test suite ' |
| 191 'to run on.')) | 190 'to run on.')) |
| 192 group.add_argument('--blacklist-file', help='Device blacklist file.') | |
| 193 | 191 |
| 194 | 192 |
| 195 def AddGTestOptions(parser): | 193 def AddGTestOptions(parser): |
| 196 """Adds gtest options to |parser|.""" | 194 """Adds gtest options to |parser|.""" |
| 197 | 195 |
| 198 gtest_suites = list(gtest_config.STABLE_TEST_SUITES | 196 gtest_suites = list(gtest_config.STABLE_TEST_SUITES |
| 199 + gtest_config.EXPERIMENTAL_TEST_SUITES) | 197 + gtest_config.EXPERIMENTAL_TEST_SUITES) |
| 200 | 198 |
| 201 group = parser.add_argument_group('GTest Options') | 199 group = parser.add_argument_group('GTest Options') |
| 202 group.add_argument('-s', '--suite', dest='suite_name', | 200 group.add_argument('-s', '--suite', dest='suite_name', |
| (...skipping 559 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 762 if args.json_results_file: | 760 if args.json_results_file: |
| 763 json_results.GenerateJsonResultsFile(results, args.json_results_file) | 761 json_results.GenerateJsonResultsFile(results, args.json_results_file) |
| 764 | 762 |
| 765 return exit_code | 763 return exit_code |
| 766 | 764 |
| 767 | 765 |
| 768 def _RunUIAutomatorTests(args, devices): | 766 def _RunUIAutomatorTests(args, devices): |
| 769 """Subcommand of RunTestsCommands which runs uiautomator tests.""" | 767 """Subcommand of RunTestsCommands which runs uiautomator tests.""" |
| 770 uiautomator_options = ProcessUIAutomatorOptions(args) | 768 uiautomator_options = ProcessUIAutomatorOptions(args) |
| 771 | 769 |
| 772 runner_factory, tests = uiautomator_setup.Setup(uiautomator_options, devices) | 770 runner_factory, tests = uiautomator_setup.Setup(uiautomator_options) |
| 773 | 771 |
| 774 results, exit_code = test_dispatcher.RunTests( | 772 results, exit_code = test_dispatcher.RunTests( |
| 775 tests, runner_factory, devices, shard=True, test_timeout=None, | 773 tests, runner_factory, devices, shard=True, test_timeout=None, |
| 776 num_retries=args.num_retries) | 774 num_retries=args.num_retries) |
| 777 | 775 |
| 778 report_results.LogFull( | 776 report_results.LogFull( |
| 779 results=results, | 777 results=results, |
| 780 test_type='UIAutomator', | 778 test_type='UIAutomator', |
| 781 test_package=os.path.basename(args.test_jar), | 779 test_package=os.path.basename(args.test_jar), |
| 782 annotation=args.annotations, | 780 annotation=args.annotations, |
| (...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 818 results=results, | 816 results=results, |
| 819 test_type='Monkey', | 817 test_type='Monkey', |
| 820 test_package='Monkey') | 818 test_package='Monkey') |
| 821 | 819 |
| 822 if args.json_results_file: | 820 if args.json_results_file: |
| 823 json_results.GenerateJsonResultsFile(results, args.json_results_file) | 821 json_results.GenerateJsonResultsFile(results, args.json_results_file) |
| 824 | 822 |
| 825 return exit_code | 823 return exit_code |
| 826 | 824 |
| 827 | 825 |
| 828 def _RunPerfTests(args, active_devices): | 826 def _RunPerfTests(args): |
| 829 """Subcommand of RunTestsCommands which runs perf tests.""" | 827 """Subcommand of RunTestsCommands which runs perf tests.""" |
| 830 perf_options = ProcessPerfTestOptions(args) | 828 perf_options = ProcessPerfTestOptions(args) |
| 831 | 829 |
| 832 # Just save a simple json with a list of test names. | 830 # Just save a simple json with a list of test names. |
| 833 if perf_options.output_json_list: | 831 if perf_options.output_json_list: |
| 834 return perf_test_runner.OutputJsonList( | 832 return perf_test_runner.OutputJsonList( |
| 835 perf_options.steps, perf_options.output_json_list) | 833 perf_options.steps, perf_options.output_json_list) |
| 836 | 834 |
| 837 # Just print the results from a single previously executed step. | 835 # Just print the results from a single previously executed step. |
| 838 if perf_options.print_step: | 836 if perf_options.print_step: |
| 839 return perf_test_runner.PrintTestOutput( | 837 return perf_test_runner.PrintTestOutput( |
| 840 perf_options.print_step, perf_options.output_chartjson_data) | 838 perf_options.print_step, perf_options.output_chartjson_data) |
| 841 | 839 |
| 842 runner_factory, tests, devices = perf_setup.Setup( | 840 runner_factory, tests, devices = perf_setup.Setup(perf_options) |
| 843 perf_options, active_devices) | |
| 844 | 841 |
| 845 # shard=False means that each device will get the full list of tests | 842 # shard=False means that each device will get the full list of tests |
| 846 # and then each one will decide their own affinity. | 843 # and then each one will decide their own affinity. |
| 847 # shard=True means each device will pop the next test available from a queue, | 844 # shard=True means each device will pop the next test available from a queue, |
| 848 # which increases throughput but have no affinity. | 845 # which increases throughput but have no affinity. |
| 849 results, _ = test_dispatcher.RunTests( | 846 results, _ = test_dispatcher.RunTests( |
| 850 tests, runner_factory, devices, shard=False, test_timeout=None, | 847 tests, runner_factory, devices, shard=False, test_timeout=None, |
| 851 num_retries=args.num_retries) | 848 num_retries=args.num_retries) |
| 852 | 849 |
| 853 report_results.LogFull( | 850 report_results.LogFull( |
| (...skipping 24 matching lines...) Expand all Loading... |
| 878 try: | 875 try: |
| 879 suite = unittest.TestSuite() | 876 suite = unittest.TestSuite() |
| 880 suite.addTests(unittest.defaultTestLoader.loadTestsFromName(m) | 877 suite.addTests(unittest.defaultTestLoader.loadTestsFromName(m) |
| 881 for m in suite_test_modules) | 878 for m in suite_test_modules) |
| 882 runner = unittest.TextTestRunner(verbosity=1+args.verbose_count) | 879 runner = unittest.TextTestRunner(verbosity=1+args.verbose_count) |
| 883 return 0 if runner.run(suite).wasSuccessful() else 1 | 880 return 0 if runner.run(suite).wasSuccessful() else 1 |
| 884 finally: | 881 finally: |
| 885 sys.path = sys.path[1:] | 882 sys.path = sys.path[1:] |
| 886 | 883 |
| 887 | 884 |
| 888 def _GetAttachedDevices(blacklist_file, test_device): | 885 def _GetAttachedDevices(test_device=None): |
| 889 """Get all attached devices. | 886 """Get all attached devices. |
| 890 | 887 |
| 891 Args: | 888 Args: |
| 892 test_device: Name of a specific device to use. | 889 test_device: Name of a specific device to use. |
| 893 | 890 |
| 894 Returns: | 891 Returns: |
| 895 A list of attached devices. | 892 A list of attached devices. |
| 896 """ | 893 """ |
| 897 if not blacklist_file: | 894 attached_devices = device_utils.DeviceUtils.HealthyDevices() |
| 898 # TODO(jbudorick): Remove this once bots pass the blacklist file. | |
| 899 blacklist_file = device_blacklist.BLACKLIST_JSON | |
| 900 logging.warning('Using default device blacklist %s', | |
| 901 device_blacklist.BLACKLIST_JSON) | |
| 902 | |
| 903 blacklist = device_blacklist.Blacklist(blacklist_file) | |
| 904 attached_devices = device_utils.DeviceUtils.HealthyDevices(blacklist) | |
| 905 if test_device: | 895 if test_device: |
| 906 test_device = [d for d in attached_devices if d == test_device] | 896 test_device = [d for d in attached_devices if d == test_device] |
| 907 if not test_device: | 897 if not test_device: |
| 908 raise device_errors.DeviceUnreachableError( | 898 raise device_errors.DeviceUnreachableError( |
| 909 'Did not find device %s among attached device. Attached devices: %s' | 899 'Did not find device %s among attached device. Attached devices: %s' |
| 910 % (test_device, ', '.join(attached_devices))) | 900 % (test_device, ', '.join(attached_devices))) |
| 911 return test_device | 901 return test_device |
| 912 | 902 |
| 913 else: | 903 else: |
| 914 if not attached_devices: | 904 if not attached_devices: |
| (...skipping 18 matching lines...) Expand all Loading... |
| 933 command = args.command | 923 command = args.command |
| 934 | 924 |
| 935 ProcessCommonOptions(args) | 925 ProcessCommonOptions(args) |
| 936 | 926 |
| 937 if args.enable_platform_mode: | 927 if args.enable_platform_mode: |
| 938 return RunTestsInPlatformMode(args, parser) | 928 return RunTestsInPlatformMode(args, parser) |
| 939 | 929 |
| 940 if command in constants.LOCAL_MACHINE_TESTS: | 930 if command in constants.LOCAL_MACHINE_TESTS: |
| 941 devices = [] | 931 devices = [] |
| 942 else: | 932 else: |
| 943 devices = _GetAttachedDevices(args.blacklist_file, args.test_device) | 933 devices = _GetAttachedDevices(args.test_device) |
| 944 | 934 |
| 945 forwarder.Forwarder.RemoveHostLog() | 935 forwarder.Forwarder.RemoveHostLog() |
| 946 if not ports.ResetTestServerPortAllocation(): | 936 if not ports.ResetTestServerPortAllocation(): |
| 947 raise Exception('Failed to reset test server port.') | 937 raise Exception('Failed to reset test server port.') |
| 948 | 938 |
| 949 if command == 'gtest': | 939 if command == 'gtest': |
| 950 if args.suite_name[0] in gtest_test_instance.BROWSER_TEST_SUITES: | 940 if args.suite_name[0] in gtest_test_instance.BROWSER_TEST_SUITES: |
| 951 return RunTestsInPlatformMode(args, parser) | 941 return RunTestsInPlatformMode(args, parser) |
| 952 return _RunGTests(args, devices) | 942 return _RunGTests(args, devices) |
| 953 elif command == 'linker': | 943 elif command == 'linker': |
| 954 return _RunLinkerTests(args, devices) | 944 return _RunLinkerTests(args, devices) |
| 955 elif command == 'instrumentation': | 945 elif command == 'instrumentation': |
| 956 return _RunInstrumentationTests(args, devices) | 946 return _RunInstrumentationTests(args, devices) |
| 957 elif command == 'uiautomator': | 947 elif command == 'uiautomator': |
| 958 return _RunUIAutomatorTests(args, devices) | 948 return _RunUIAutomatorTests(args, devices) |
| 959 elif command == 'junit': | 949 elif command == 'junit': |
| 960 return _RunJUnitTests(args) | 950 return _RunJUnitTests(args) |
| 961 elif command == 'monkey': | 951 elif command == 'monkey': |
| 962 return _RunMonkeyTests(args, devices) | 952 return _RunMonkeyTests(args, devices) |
| 963 elif command == 'perf': | 953 elif command == 'perf': |
| 964 return _RunPerfTests(args, devices) | 954 return _RunPerfTests(args) |
| 965 elif command == 'python': | 955 elif command == 'python': |
| 966 return _RunPythonTests(args) | 956 return _RunPythonTests(args) |
| 967 else: | 957 else: |
| 968 raise Exception('Unknown test type.') | 958 raise Exception('Unknown test type.') |
| 969 | 959 |
| 970 | 960 |
| 971 _SUPPORTED_IN_PLATFORM_MODE = [ | 961 _SUPPORTED_IN_PLATFORM_MODE = [ |
| 972 # TODO(jbudorick): Add support for more test types. | 962 # TODO(jbudorick): Add support for more test types. |
| 973 'gtest', | 963 'gtest', |
| 974 'instrumentation', | 964 'instrumentation', |
| (...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1065 if e.is_infra_error: | 1055 if e.is_infra_error: |
| 1066 return constants.INFRA_EXIT_CODE | 1056 return constants.INFRA_EXIT_CODE |
| 1067 return constants.ERROR_EXIT_CODE | 1057 return constants.ERROR_EXIT_CODE |
| 1068 except: # pylint: disable=W0702 | 1058 except: # pylint: disable=W0702 |
| 1069 logging.exception('Unrecognized error occurred.') | 1059 logging.exception('Unrecognized error occurred.') |
| 1070 return constants.ERROR_EXIT_CODE | 1060 return constants.ERROR_EXIT_CODE |
| 1071 | 1061 |
| 1072 | 1062 |
| 1073 if __name__ == '__main__': | 1063 if __name__ == '__main__': |
| 1074 sys.exit(main()) | 1064 sys.exit(main()) |
| OLD | NEW |