| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # | 2 # |
| 3 # Copyright 2013 The Chromium Authors. All rights reserved. | 3 # Copyright 2013 The Chromium Authors. All rights reserved. |
| 4 # Use of this source code is governed by a BSD-style license that can be | 4 # Use of this source code is governed by a BSD-style license that can be |
| 5 # found in the LICENSE file. | 5 # found in the LICENSE file. |
| 6 | 6 |
| 7 """Runs all types of tests from one unified interface.""" | 7 """Runs all types of tests from one unified interface.""" |
| 8 | 8 |
| 9 import argparse | 9 import argparse |
| 10 import collections | 10 import collections |
| 11 import logging | 11 import logging |
| 12 import os | 12 import os |
| 13 import shutil | 13 import shutil |
| 14 import signal | 14 import signal |
| 15 import sys | 15 import sys |
| 16 import threading | 16 import threading |
| 17 import unittest | 17 import unittest |
| 18 | 18 |
| 19 from pylib import constants | 19 from pylib import constants |
| 20 from pylib import forwarder | 20 from pylib import forwarder |
| 21 from pylib import ports | 21 from pylib import ports |
| 22 from pylib.base import base_test_result | 22 from pylib.base import base_test_result |
| 23 from pylib.base import environment_factory | 23 from pylib.base import environment_factory |
| 24 from pylib.base import test_dispatcher | 24 from pylib.base import test_dispatcher |
| 25 from pylib.base import test_instance_factory | 25 from pylib.base import test_instance_factory |
| 26 from pylib.base import test_run_factory | 26 from pylib.base import test_run_factory |
| 27 from pylib.device import device_blacklist |
| 27 from pylib.device import device_errors | 28 from pylib.device import device_errors |
| 28 from pylib.device import device_utils | 29 from pylib.device import device_utils |
| 29 from pylib.gtest import gtest_config | 30 from pylib.gtest import gtest_config |
| 30 # TODO(jbudorick): Remove this once we stop selectively enabling platform mode. | 31 # TODO(jbudorick): Remove this once we stop selectively enabling platform mode. |
| 31 from pylib.gtest import gtest_test_instance | 32 from pylib.gtest import gtest_test_instance |
| 32 from pylib.gtest import setup as gtest_setup | 33 from pylib.gtest import setup as gtest_setup |
| 33 from pylib.gtest import test_options as gtest_test_options | 34 from pylib.gtest import test_options as gtest_test_options |
| 34 from pylib.linker import setup as linker_setup | 35 from pylib.linker import setup as linker_setup |
| 35 from pylib.host_driven import setup as host_driven_setup | 36 from pylib.host_driven import setup as host_driven_setup |
| 36 from pylib.instrumentation import setup as instrumentation_setup | 37 from pylib.instrumentation import setup as instrumentation_setup |
| (...skipping 144 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 181 def AddDeviceOptions(parser): | 182 def AddDeviceOptions(parser): |
| 182 """Adds device options to |parser|.""" | 183 """Adds device options to |parser|.""" |
| 183 group = parser.add_argument_group(title='Device Options') | 184 group = parser.add_argument_group(title='Device Options') |
| 184 group.add_argument('--tool', | 185 group.add_argument('--tool', |
| 185 dest='tool', | 186 dest='tool', |
| 186 help=('Run the test under a tool ' | 187 help=('Run the test under a tool ' |
| 187 '(use --tool help to list them)')) | 188 '(use --tool help to list them)')) |
| 188 group.add_argument('-d', '--device', dest='test_device', | 189 group.add_argument('-d', '--device', dest='test_device', |
| 189 help=('Target device for the test suite ' | 190 help=('Target device for the test suite ' |
| 190 'to run on.')) | 191 'to run on.')) |
| 192 group.add_argument('--blacklist-file', help='Device blacklist file.') |
| 191 | 193 |
| 192 | 194 |
| 193 def AddGTestOptions(parser): | 195 def AddGTestOptions(parser): |
| 194 """Adds gtest options to |parser|.""" | 196 """Adds gtest options to |parser|.""" |
| 195 | 197 |
| 196 gtest_suites = list(gtest_config.STABLE_TEST_SUITES | 198 gtest_suites = list(gtest_config.STABLE_TEST_SUITES |
| 197 + gtest_config.EXPERIMENTAL_TEST_SUITES) | 199 + gtest_config.EXPERIMENTAL_TEST_SUITES) |
| 198 | 200 |
| 199 group = parser.add_argument_group('GTest Options') | 201 group = parser.add_argument_group('GTest Options') |
| 200 group.add_argument('-s', '--suite', dest='suite_name', | 202 group.add_argument('-s', '--suite', dest='suite_name', |
| (...skipping 559 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 760 if args.json_results_file: | 762 if args.json_results_file: |
| 761 json_results.GenerateJsonResultsFile(results, args.json_results_file) | 763 json_results.GenerateJsonResultsFile(results, args.json_results_file) |
| 762 | 764 |
| 763 return exit_code | 765 return exit_code |
| 764 | 766 |
| 765 | 767 |
| 766 def _RunUIAutomatorTests(args, devices): | 768 def _RunUIAutomatorTests(args, devices): |
| 767 """Subcommand of RunTestsCommands which runs uiautomator tests.""" | 769 """Subcommand of RunTestsCommands which runs uiautomator tests.""" |
| 768 uiautomator_options = ProcessUIAutomatorOptions(args) | 770 uiautomator_options = ProcessUIAutomatorOptions(args) |
| 769 | 771 |
| 770 runner_factory, tests = uiautomator_setup.Setup(uiautomator_options) | 772 runner_factory, tests = uiautomator_setup.Setup(uiautomator_options, devices) |
| 771 | 773 |
| 772 results, exit_code = test_dispatcher.RunTests( | 774 results, exit_code = test_dispatcher.RunTests( |
| 773 tests, runner_factory, devices, shard=True, test_timeout=None, | 775 tests, runner_factory, devices, shard=True, test_timeout=None, |
| 774 num_retries=args.num_retries) | 776 num_retries=args.num_retries) |
| 775 | 777 |
| 776 report_results.LogFull( | 778 report_results.LogFull( |
| 777 results=results, | 779 results=results, |
| 778 test_type='UIAutomator', | 780 test_type='UIAutomator', |
| 779 test_package=os.path.basename(args.test_jar), | 781 test_package=os.path.basename(args.test_jar), |
| 780 annotation=args.annotations, | 782 annotation=args.annotations, |
| (...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 816 results=results, | 818 results=results, |
| 817 test_type='Monkey', | 819 test_type='Monkey', |
| 818 test_package='Monkey') | 820 test_package='Monkey') |
| 819 | 821 |
| 820 if args.json_results_file: | 822 if args.json_results_file: |
| 821 json_results.GenerateJsonResultsFile(results, args.json_results_file) | 823 json_results.GenerateJsonResultsFile(results, args.json_results_file) |
| 822 | 824 |
| 823 return exit_code | 825 return exit_code |
| 824 | 826 |
| 825 | 827 |
| 826 def _RunPerfTests(args): | 828 def _RunPerfTests(args, active_devices): |
| 827 """Subcommand of RunTestsCommands which runs perf tests.""" | 829 """Subcommand of RunTestsCommands which runs perf tests.""" |
| 828 perf_options = ProcessPerfTestOptions(args) | 830 perf_options = ProcessPerfTestOptions(args) |
| 829 | 831 |
| 830 # Just save a simple json with a list of test names. | 832 # Just save a simple json with a list of test names. |
| 831 if perf_options.output_json_list: | 833 if perf_options.output_json_list: |
| 832 return perf_test_runner.OutputJsonList( | 834 return perf_test_runner.OutputJsonList( |
| 833 perf_options.steps, perf_options.output_json_list) | 835 perf_options.steps, perf_options.output_json_list) |
| 834 | 836 |
| 835 # Just print the results from a single previously executed step. | 837 # Just print the results from a single previously executed step. |
| 836 if perf_options.print_step: | 838 if perf_options.print_step: |
| 837 return perf_test_runner.PrintTestOutput( | 839 return perf_test_runner.PrintTestOutput( |
| 838 perf_options.print_step, perf_options.output_chartjson_data) | 840 perf_options.print_step, perf_options.output_chartjson_data) |
| 839 | 841 |
| 840 runner_factory, tests, devices = perf_setup.Setup(perf_options) | 842 runner_factory, tests, devices = perf_setup.Setup( |
| 843 perf_options, active_devices) |
| 841 | 844 |
| 842 # shard=False means that each device will get the full list of tests | 845 # shard=False means that each device will get the full list of tests |
| 843 # and then each one will decide their own affinity. | 846 # and then each one will decide their own affinity. |
| 844 # shard=True means each device will pop the next test available from a queue, | 847 # shard=True means each device will pop the next test available from a queue, |
| 845 # which increases throughput but have no affinity. | 848 # which increases throughput but have no affinity. |
| 846 results, _ = test_dispatcher.RunTests( | 849 results, _ = test_dispatcher.RunTests( |
| 847 tests, runner_factory, devices, shard=False, test_timeout=None, | 850 tests, runner_factory, devices, shard=False, test_timeout=None, |
| 848 num_retries=args.num_retries) | 851 num_retries=args.num_retries) |
| 849 | 852 |
| 850 report_results.LogFull( | 853 report_results.LogFull( |
| (...skipping 24 matching lines...) Expand all Loading... |
| 875 try: | 878 try: |
| 876 suite = unittest.TestSuite() | 879 suite = unittest.TestSuite() |
| 877 suite.addTests(unittest.defaultTestLoader.loadTestsFromName(m) | 880 suite.addTests(unittest.defaultTestLoader.loadTestsFromName(m) |
| 878 for m in suite_test_modules) | 881 for m in suite_test_modules) |
| 879 runner = unittest.TextTestRunner(verbosity=1+args.verbose_count) | 882 runner = unittest.TextTestRunner(verbosity=1+args.verbose_count) |
| 880 return 0 if runner.run(suite).wasSuccessful() else 1 | 883 return 0 if runner.run(suite).wasSuccessful() else 1 |
| 881 finally: | 884 finally: |
| 882 sys.path = sys.path[1:] | 885 sys.path = sys.path[1:] |
| 883 | 886 |
| 884 | 887 |
| 885 def _GetAttachedDevices(test_device=None): | 888 def _GetAttachedDevices(blacklist_file, test_device): |
| 886 """Get all attached devices. | 889 """Get all attached devices. |
| 887 | 890 |
| 888 Args: | 891 Args: |
| 889 test_device: Name of a specific device to use. | 892 test_device: Name of a specific device to use. |
| 890 | 893 |
| 891 Returns: | 894 Returns: |
| 892 A list of attached devices. | 895 A list of attached devices. |
| 893 """ | 896 """ |
| 894 attached_devices = device_utils.DeviceUtils.HealthyDevices() | 897 if not blacklist_file: |
| 898 # TODO(jbudorick): Remove this once bots pass the blacklist file. |
| 899 blacklist_file = device_blacklist.BLACKLIST_JSON |
| 900 logging.warning('Using default device blacklist %s', |
| 901 device_blacklist.BLACKLIST_JSON) |
| 902 |
| 903 blacklist = device_blacklist.Blacklist(blacklist_file) |
| 904 attached_devices = device_utils.DeviceUtils.HealthyDevices(blacklist) |
| 895 if test_device: | 905 if test_device: |
| 896 test_device = [d for d in attached_devices if d == test_device] | 906 test_device = [d for d in attached_devices if d == test_device] |
| 897 if not test_device: | 907 if not test_device: |
| 898 raise device_errors.DeviceUnreachableError( | 908 raise device_errors.DeviceUnreachableError( |
| 899 'Did not find device %s among attached device. Attached devices: %s' | 909 'Did not find device %s among attached device. Attached devices: %s' |
| 900 % (test_device, ', '.join(attached_devices))) | 910 % (test_device, ', '.join(attached_devices))) |
| 901 return test_device | 911 return test_device |
| 902 | 912 |
| 903 else: | 913 else: |
| 904 if not attached_devices: | 914 if not attached_devices: |
| (...skipping 18 matching lines...) Expand all Loading... |
| 923 command = args.command | 933 command = args.command |
| 924 | 934 |
| 925 ProcessCommonOptions(args) | 935 ProcessCommonOptions(args) |
| 926 | 936 |
| 927 if args.enable_platform_mode: | 937 if args.enable_platform_mode: |
| 928 return RunTestsInPlatformMode(args, parser) | 938 return RunTestsInPlatformMode(args, parser) |
| 929 | 939 |
| 930 if command in constants.LOCAL_MACHINE_TESTS: | 940 if command in constants.LOCAL_MACHINE_TESTS: |
| 931 devices = [] | 941 devices = [] |
| 932 else: | 942 else: |
| 933 devices = _GetAttachedDevices(args.test_device) | 943 devices = _GetAttachedDevices(args.blacklist_file, args.test_device) |
| 934 | 944 |
| 935 forwarder.Forwarder.RemoveHostLog() | 945 forwarder.Forwarder.RemoveHostLog() |
| 936 if not ports.ResetTestServerPortAllocation(): | 946 if not ports.ResetTestServerPortAllocation(): |
| 937 raise Exception('Failed to reset test server port.') | 947 raise Exception('Failed to reset test server port.') |
| 938 | 948 |
| 939 if command == 'gtest': | 949 if command == 'gtest': |
| 940 if args.suite_name[0] in gtest_test_instance.BROWSER_TEST_SUITES: | 950 if args.suite_name[0] in gtest_test_instance.BROWSER_TEST_SUITES: |
| 941 return RunTestsInPlatformMode(args, parser) | 951 return RunTestsInPlatformMode(args, parser) |
| 942 return _RunGTests(args, devices) | 952 return _RunGTests(args, devices) |
| 943 elif command == 'linker': | 953 elif command == 'linker': |
| 944 return _RunLinkerTests(args, devices) | 954 return _RunLinkerTests(args, devices) |
| 945 elif command == 'instrumentation': | 955 elif command == 'instrumentation': |
| 946 return _RunInstrumentationTests(args, devices) | 956 return _RunInstrumentationTests(args, devices) |
| 947 elif command == 'uiautomator': | 957 elif command == 'uiautomator': |
| 948 return _RunUIAutomatorTests(args, devices) | 958 return _RunUIAutomatorTests(args, devices) |
| 949 elif command == 'junit': | 959 elif command == 'junit': |
| 950 return _RunJUnitTests(args) | 960 return _RunJUnitTests(args) |
| 951 elif command == 'monkey': | 961 elif command == 'monkey': |
| 952 return _RunMonkeyTests(args, devices) | 962 return _RunMonkeyTests(args, devices) |
| 953 elif command == 'perf': | 963 elif command == 'perf': |
| 954 return _RunPerfTests(args) | 964 return _RunPerfTests(args, devices) |
| 955 elif command == 'python': | 965 elif command == 'python': |
| 956 return _RunPythonTests(args) | 966 return _RunPythonTests(args) |
| 957 else: | 967 else: |
| 958 raise Exception('Unknown test type.') | 968 raise Exception('Unknown test type.') |
| 959 | 969 |
| 960 | 970 |
| 961 _SUPPORTED_IN_PLATFORM_MODE = [ | 971 _SUPPORTED_IN_PLATFORM_MODE = [ |
| 962 # TODO(jbudorick): Add support for more test types. | 972 # TODO(jbudorick): Add support for more test types. |
| 963 'gtest', | 973 'gtest', |
| 964 'instrumentation', | 974 'instrumentation', |
| (...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1055 if e.is_infra_error: | 1065 if e.is_infra_error: |
| 1056 return constants.INFRA_EXIT_CODE | 1066 return constants.INFRA_EXIT_CODE |
| 1057 return constants.ERROR_EXIT_CODE | 1067 return constants.ERROR_EXIT_CODE |
| 1058 except: # pylint: disable=W0702 | 1068 except: # pylint: disable=W0702 |
| 1059 logging.exception('Unrecognized error occurred.') | 1069 logging.exception('Unrecognized error occurred.') |
| 1060 return constants.ERROR_EXIT_CODE | 1070 return constants.ERROR_EXIT_CODE |
| 1061 | 1071 |
| 1062 | 1072 |
| 1063 if __name__ == '__main__': | 1073 if __name__ == '__main__': |
| 1064 sys.exit(main()) | 1074 sys.exit(main()) |
| OLD | NEW |