Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(108)

Side by Side Diff: build/android/test_runner.py

Issue 1281923003: [Android] Add --blacklist-file as a command-line option. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: fix telemetry_unittests Created 5 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « build/android/screenshot.py ('k') | build/android/tombstones.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # 2 #
3 # Copyright 2013 The Chromium Authors. All rights reserved. 3 # Copyright 2013 The Chromium Authors. All rights reserved.
4 # Use of this source code is governed by a BSD-style license that can be 4 # Use of this source code is governed by a BSD-style license that can be
5 # found in the LICENSE file. 5 # found in the LICENSE file.
6 6
7 """Runs all types of tests from one unified interface.""" 7 """Runs all types of tests from one unified interface."""
8 8
9 import argparse 9 import argparse
10 import collections 10 import collections
11 import logging 11 import logging
12 import os 12 import os
13 import shutil 13 import shutil
14 import signal 14 import signal
15 import sys 15 import sys
16 import threading 16 import threading
17 import unittest 17 import unittest
18 18
19 from pylib import constants 19 from pylib import constants
20 from pylib import forwarder 20 from pylib import forwarder
21 from pylib import ports 21 from pylib import ports
22 from pylib.base import base_test_result 22 from pylib.base import base_test_result
23 from pylib.base import environment_factory 23 from pylib.base import environment_factory
24 from pylib.base import test_dispatcher 24 from pylib.base import test_dispatcher
25 from pylib.base import test_instance_factory 25 from pylib.base import test_instance_factory
26 from pylib.base import test_run_factory 26 from pylib.base import test_run_factory
27 from pylib.device import device_blacklist
27 from pylib.device import device_errors 28 from pylib.device import device_errors
28 from pylib.device import device_utils 29 from pylib.device import device_utils
29 from pylib.gtest import gtest_config 30 from pylib.gtest import gtest_config
30 # TODO(jbudorick): Remove this once we stop selectively enabling platform mode. 31 # TODO(jbudorick): Remove this once we stop selectively enabling platform mode.
31 from pylib.gtest import gtest_test_instance 32 from pylib.gtest import gtest_test_instance
32 from pylib.gtest import setup as gtest_setup 33 from pylib.gtest import setup as gtest_setup
33 from pylib.gtest import test_options as gtest_test_options 34 from pylib.gtest import test_options as gtest_test_options
34 from pylib.linker import setup as linker_setup 35 from pylib.linker import setup as linker_setup
35 from pylib.host_driven import setup as host_driven_setup 36 from pylib.host_driven import setup as host_driven_setup
36 from pylib.instrumentation import setup as instrumentation_setup 37 from pylib.instrumentation import setup as instrumentation_setup
(...skipping 144 matching lines...) Expand 10 before | Expand all | Expand 10 after
181 def AddDeviceOptions(parser): 182 def AddDeviceOptions(parser):
182 """Adds device options to |parser|.""" 183 """Adds device options to |parser|."""
183 group = parser.add_argument_group(title='Device Options') 184 group = parser.add_argument_group(title='Device Options')
184 group.add_argument('--tool', 185 group.add_argument('--tool',
185 dest='tool', 186 dest='tool',
186 help=('Run the test under a tool ' 187 help=('Run the test under a tool '
187 '(use --tool help to list them)')) 188 '(use --tool help to list them)'))
188 group.add_argument('-d', '--device', dest='test_device', 189 group.add_argument('-d', '--device', dest='test_device',
189 help=('Target device for the test suite ' 190 help=('Target device for the test suite '
190 'to run on.')) 191 'to run on.'))
192 group.add_argument('--blacklist-file', help='Device blacklist file.')
191 193
192 194
193 def AddGTestOptions(parser): 195 def AddGTestOptions(parser):
194 """Adds gtest options to |parser|.""" 196 """Adds gtest options to |parser|."""
195 197
196 gtest_suites = list(gtest_config.STABLE_TEST_SUITES 198 gtest_suites = list(gtest_config.STABLE_TEST_SUITES
197 + gtest_config.EXPERIMENTAL_TEST_SUITES) 199 + gtest_config.EXPERIMENTAL_TEST_SUITES)
198 200
199 group = parser.add_argument_group('GTest Options') 201 group = parser.add_argument_group('GTest Options')
200 group.add_argument('-s', '--suite', dest='suite_name', 202 group.add_argument('-s', '--suite', dest='suite_name',
(...skipping 559 matching lines...) Expand 10 before | Expand all | Expand 10 after
760 if args.json_results_file: 762 if args.json_results_file:
761 json_results.GenerateJsonResultsFile(results, args.json_results_file) 763 json_results.GenerateJsonResultsFile(results, args.json_results_file)
762 764
763 return exit_code 765 return exit_code
764 766
765 767
766 def _RunUIAutomatorTests(args, devices): 768 def _RunUIAutomatorTests(args, devices):
767 """Subcommand of RunTestsCommands which runs uiautomator tests.""" 769 """Subcommand of RunTestsCommands which runs uiautomator tests."""
768 uiautomator_options = ProcessUIAutomatorOptions(args) 770 uiautomator_options = ProcessUIAutomatorOptions(args)
769 771
770 runner_factory, tests = uiautomator_setup.Setup(uiautomator_options) 772 runner_factory, tests = uiautomator_setup.Setup(uiautomator_options, devices)
771 773
772 results, exit_code = test_dispatcher.RunTests( 774 results, exit_code = test_dispatcher.RunTests(
773 tests, runner_factory, devices, shard=True, test_timeout=None, 775 tests, runner_factory, devices, shard=True, test_timeout=None,
774 num_retries=args.num_retries) 776 num_retries=args.num_retries)
775 777
776 report_results.LogFull( 778 report_results.LogFull(
777 results=results, 779 results=results,
778 test_type='UIAutomator', 780 test_type='UIAutomator',
779 test_package=os.path.basename(args.test_jar), 781 test_package=os.path.basename(args.test_jar),
780 annotation=args.annotations, 782 annotation=args.annotations,
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after
816 results=results, 818 results=results,
817 test_type='Monkey', 819 test_type='Monkey',
818 test_package='Monkey') 820 test_package='Monkey')
819 821
820 if args.json_results_file: 822 if args.json_results_file:
821 json_results.GenerateJsonResultsFile(results, args.json_results_file) 823 json_results.GenerateJsonResultsFile(results, args.json_results_file)
822 824
823 return exit_code 825 return exit_code
824 826
825 827
826 def _RunPerfTests(args): 828 def _RunPerfTests(args, active_devices):
827 """Subcommand of RunTestsCommands which runs perf tests.""" 829 """Subcommand of RunTestsCommands which runs perf tests."""
828 perf_options = ProcessPerfTestOptions(args) 830 perf_options = ProcessPerfTestOptions(args)
829 831
830 # Just save a simple json with a list of test names. 832 # Just save a simple json with a list of test names.
831 if perf_options.output_json_list: 833 if perf_options.output_json_list:
832 return perf_test_runner.OutputJsonList( 834 return perf_test_runner.OutputJsonList(
833 perf_options.steps, perf_options.output_json_list) 835 perf_options.steps, perf_options.output_json_list)
834 836
835 # Just print the results from a single previously executed step. 837 # Just print the results from a single previously executed step.
836 if perf_options.print_step: 838 if perf_options.print_step:
837 return perf_test_runner.PrintTestOutput( 839 return perf_test_runner.PrintTestOutput(
838 perf_options.print_step, perf_options.output_chartjson_data) 840 perf_options.print_step, perf_options.output_chartjson_data)
839 841
840 runner_factory, tests, devices = perf_setup.Setup(perf_options) 842 runner_factory, tests, devices = perf_setup.Setup(
843 perf_options, active_devices)
841 844
842 # shard=False means that each device will get the full list of tests 845 # shard=False means that each device will get the full list of tests
843 # and then each one will decide their own affinity. 846 # and then each one will decide their own affinity.
844 # shard=True means each device will pop the next test available from a queue, 847 # shard=True means each device will pop the next test available from a queue,
845 # which increases throughput but have no affinity. 848 # which increases throughput but have no affinity.
846 results, _ = test_dispatcher.RunTests( 849 results, _ = test_dispatcher.RunTests(
847 tests, runner_factory, devices, shard=False, test_timeout=None, 850 tests, runner_factory, devices, shard=False, test_timeout=None,
848 num_retries=args.num_retries) 851 num_retries=args.num_retries)
849 852
850 report_results.LogFull( 853 report_results.LogFull(
(...skipping 24 matching lines...) Expand all
875 try: 878 try:
876 suite = unittest.TestSuite() 879 suite = unittest.TestSuite()
877 suite.addTests(unittest.defaultTestLoader.loadTestsFromName(m) 880 suite.addTests(unittest.defaultTestLoader.loadTestsFromName(m)
878 for m in suite_test_modules) 881 for m in suite_test_modules)
879 runner = unittest.TextTestRunner(verbosity=1+args.verbose_count) 882 runner = unittest.TextTestRunner(verbosity=1+args.verbose_count)
880 return 0 if runner.run(suite).wasSuccessful() else 1 883 return 0 if runner.run(suite).wasSuccessful() else 1
881 finally: 884 finally:
882 sys.path = sys.path[1:] 885 sys.path = sys.path[1:]
883 886
884 887
885 def _GetAttachedDevices(test_device=None): 888 def _GetAttachedDevices(blacklist_file, test_device):
886 """Get all attached devices. 889 """Get all attached devices.
887 890
888 Args: 891 Args:
889 test_device: Name of a specific device to use. 892 test_device: Name of a specific device to use.
890 893
891 Returns: 894 Returns:
892 A list of attached devices. 895 A list of attached devices.
893 """ 896 """
894 attached_devices = device_utils.DeviceUtils.HealthyDevices() 897 if not blacklist_file:
898 # TODO(jbudorick): Remove this once bots pass the blacklist file.
899 blacklist_file = device_blacklist.BLACKLIST_JSON
900 logging.warning('Using default device blacklist %s',
901 device_blacklist.BLACKLIST_JSON)
902
903 blacklist = device_blacklist.Blacklist(blacklist_file)
904 attached_devices = device_utils.DeviceUtils.HealthyDevices(blacklist)
895 if test_device: 905 if test_device:
896 test_device = [d for d in attached_devices if d == test_device] 906 test_device = [d for d in attached_devices if d == test_device]
897 if not test_device: 907 if not test_device:
898 raise device_errors.DeviceUnreachableError( 908 raise device_errors.DeviceUnreachableError(
899 'Did not find device %s among attached device. Attached devices: %s' 909 'Did not find device %s among attached device. Attached devices: %s'
900 % (test_device, ', '.join(attached_devices))) 910 % (test_device, ', '.join(attached_devices)))
901 return test_device 911 return test_device
902 912
903 else: 913 else:
904 if not attached_devices: 914 if not attached_devices:
(...skipping 18 matching lines...) Expand all
923 command = args.command 933 command = args.command
924 934
925 ProcessCommonOptions(args) 935 ProcessCommonOptions(args)
926 936
927 if args.enable_platform_mode: 937 if args.enable_platform_mode:
928 return RunTestsInPlatformMode(args, parser) 938 return RunTestsInPlatformMode(args, parser)
929 939
930 if command in constants.LOCAL_MACHINE_TESTS: 940 if command in constants.LOCAL_MACHINE_TESTS:
931 devices = [] 941 devices = []
932 else: 942 else:
933 devices = _GetAttachedDevices(args.test_device) 943 devices = _GetAttachedDevices(args.blacklist_file, args.test_device)
934 944
935 forwarder.Forwarder.RemoveHostLog() 945 forwarder.Forwarder.RemoveHostLog()
936 if not ports.ResetTestServerPortAllocation(): 946 if not ports.ResetTestServerPortAllocation():
937 raise Exception('Failed to reset test server port.') 947 raise Exception('Failed to reset test server port.')
938 948
939 if command == 'gtest': 949 if command == 'gtest':
940 if args.suite_name[0] in gtest_test_instance.BROWSER_TEST_SUITES: 950 if args.suite_name[0] in gtest_test_instance.BROWSER_TEST_SUITES:
941 return RunTestsInPlatformMode(args, parser) 951 return RunTestsInPlatformMode(args, parser)
942 return _RunGTests(args, devices) 952 return _RunGTests(args, devices)
943 elif command == 'linker': 953 elif command == 'linker':
944 return _RunLinkerTests(args, devices) 954 return _RunLinkerTests(args, devices)
945 elif command == 'instrumentation': 955 elif command == 'instrumentation':
946 return _RunInstrumentationTests(args, devices) 956 return _RunInstrumentationTests(args, devices)
947 elif command == 'uiautomator': 957 elif command == 'uiautomator':
948 return _RunUIAutomatorTests(args, devices) 958 return _RunUIAutomatorTests(args, devices)
949 elif command == 'junit': 959 elif command == 'junit':
950 return _RunJUnitTests(args) 960 return _RunJUnitTests(args)
951 elif command == 'monkey': 961 elif command == 'monkey':
952 return _RunMonkeyTests(args, devices) 962 return _RunMonkeyTests(args, devices)
953 elif command == 'perf': 963 elif command == 'perf':
954 return _RunPerfTests(args) 964 return _RunPerfTests(args, devices)
955 elif command == 'python': 965 elif command == 'python':
956 return _RunPythonTests(args) 966 return _RunPythonTests(args)
957 else: 967 else:
958 raise Exception('Unknown test type.') 968 raise Exception('Unknown test type.')
959 969
960 970
961 _SUPPORTED_IN_PLATFORM_MODE = [ 971 _SUPPORTED_IN_PLATFORM_MODE = [
962 # TODO(jbudorick): Add support for more test types. 972 # TODO(jbudorick): Add support for more test types.
963 'gtest', 973 'gtest',
964 'instrumentation', 974 'instrumentation',
(...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after
1055 if e.is_infra_error: 1065 if e.is_infra_error:
1056 return constants.INFRA_EXIT_CODE 1066 return constants.INFRA_EXIT_CODE
1057 return constants.ERROR_EXIT_CODE 1067 return constants.ERROR_EXIT_CODE
1058 except: # pylint: disable=W0702 1068 except: # pylint: disable=W0702
1059 logging.exception('Unrecognized error occurred.') 1069 logging.exception('Unrecognized error occurred.')
1060 return constants.ERROR_EXIT_CODE 1070 return constants.ERROR_EXIT_CODE
1061 1071
1062 1072
1063 if __name__ == '__main__': 1073 if __name__ == '__main__':
1064 sys.exit(main()) 1074 sys.exit(main())
OLDNEW
« no previous file with comments | « build/android/screenshot.py ('k') | build/android/tombstones.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698