OLD | NEW |
---|---|
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # | 2 # |
3 # Copyright 2013 The Chromium Authors. All rights reserved. | 3 # Copyright 2013 The Chromium Authors. All rights reserved. |
4 # Use of this source code is governed by a BSD-style license that can be | 4 # Use of this source code is governed by a BSD-style license that can be |
5 # found in the LICENSE file. | 5 # found in the LICENSE file. |
6 | 6 |
7 """Runs all types of tests from one unified interface.""" | 7 """Runs all types of tests from one unified interface.""" |
8 | 8 |
9 import argparse | 9 import argparse |
10 import collections | 10 import collections |
11 import contextlib | 11 import contextlib |
12 import itertools | 12 import itertools |
13 import logging | 13 import logging |
14 import os | 14 import os |
15 import shutil | 15 import shutil |
16 import signal | 16 import signal |
17 import sys | 17 import sys |
18 import tempfile | |
18 import threading | 19 import threading |
19 import traceback | 20 import traceback |
20 import unittest | 21 import unittest |
21 | 22 |
22 # Import _strptime before threaded code. datetime.datetime.strptime is | 23 # Import _strptime before threaded code. datetime.datetime.strptime is |
23 # threadsafe except for the initial import of the _strptime module. | 24 # threadsafe except for the initial import of the _strptime module. |
24 # See http://crbug.com/724524 and https://bugs.python.org/issue7980. | 25 # See http://crbug.com/724524 and https://bugs.python.org/issue7980. |
25 import _strptime # pylint: disable=unused-import | 26 import _strptime # pylint: disable=unused-import |
26 | 27 |
27 from pylib.constants import host_paths | 28 from pylib.constants import host_paths |
28 | 29 |
29 if host_paths.DEVIL_PATH not in sys.path: | 30 if host_paths.DEVIL_PATH not in sys.path: |
30 sys.path.append(host_paths.DEVIL_PATH) | 31 sys.path.append(host_paths.DEVIL_PATH) |
31 | 32 |
32 from devil import base_error | 33 from devil import base_error |
33 from devil.utils import reraiser_thread | 34 from devil.utils import reraiser_thread |
34 from devil.utils import run_tests_helper | 35 from devil.utils import run_tests_helper |
35 | 36 |
36 from pylib import constants | 37 from pylib import constants |
37 from pylib.base import base_test_result | 38 from pylib.base import base_test_result |
38 from pylib.base import environment_factory | 39 from pylib.base import environment_factory |
40 from pylib.base import output_manager | |
41 from pylib.base import output_manager_factory | |
39 from pylib.base import test_instance_factory | 42 from pylib.base import test_instance_factory |
40 from pylib.base import test_run_factory | 43 from pylib.base import test_run_factory |
41 from pylib.results import json_results | 44 from pylib.results import json_results |
42 from pylib.results import report_results | 45 from pylib.results import report_results |
46 from pylib.results.presentation import test_results_presentation | |
43 from pylib.utils import logdog_helper | 47 from pylib.utils import logdog_helper |
44 from pylib.utils import logging_utils | 48 from pylib.utils import logging_utils |
45 | 49 |
46 from py_utils import contextlib_ext | 50 from py_utils import contextlib_ext |
47 | 51 |
48 | 52 |
49 _DEVIL_STATIC_CONFIG_FILE = os.path.abspath(os.path.join( | 53 _DEVIL_STATIC_CONFIG_FILE = os.path.abspath(os.path.join( |
50 host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'devil_config.json')) | 54 host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'devil_config.json')) |
51 | 55 |
52 | 56 |
(...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
139 help='Run the test scripts in platform mode, which ' | 143 help='Run the test scripts in platform mode, which ' |
140 'conceptually separates the test runner from the ' | 144 'conceptually separates the test runner from the ' |
141 '"device" (local or remote, real or emulated) on ' | 145 '"device" (local or remote, real or emulated) on ' |
142 'which the tests are running. [experimental]') | 146 'which the tests are running. [experimental]') |
143 | 147 |
144 parser.add_argument( | 148 parser.add_argument( |
145 '-e', '--environment', | 149 '-e', '--environment', |
146 default='local', choices=constants.VALID_ENVIRONMENTS, | 150 default='local', choices=constants.VALID_ENVIRONMENTS, |
147 help='Test environment to run in (default: %(default)s).') | 151 help='Test environment to run in (default: %(default)s).') |
148 | 152 |
153 parser.add_argument( | |
154 '--local-output', action='store_true', | |
jbudorick
2017/08/10 16:27:38
nit: action on its own line
mikecase (-- gone --)
2017/08/23 04:28:20
Done
| |
155 help='Whether to archive test output locally and generate ' | |
156 'a local results detail page.') | |
157 | |
149 class FastLocalDevAction(argparse.Action): | 158 class FastLocalDevAction(argparse.Action): |
150 def __call__(self, parser, namespace, values, option_string=None): | 159 def __call__(self, parser, namespace, values, option_string=None): |
151 namespace.verbose_count = max(namespace.verbose_count, 1) | 160 namespace.verbose_count = max(namespace.verbose_count, 1) |
152 namespace.num_retries = 0 | 161 namespace.num_retries = 0 |
153 namespace.enable_device_cache = True | 162 namespace.enable_device_cache = True |
154 namespace.enable_concurrent_adb = True | 163 namespace.enable_concurrent_adb = True |
155 namespace.skip_clear_data = True | 164 namespace.skip_clear_data = True |
156 namespace.extract_test_list_from_filter = True | 165 namespace.extract_test_list_from_filter = True |
157 | 166 |
158 parser.add_argument( | 167 parser.add_argument( |
(...skipping 651 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
810 logging.critical( | 819 logging.critical( |
811 'Logcat: %s', logdog_helper.get_viewer_url('unified_logcats')) | 820 'Logcat: %s', logdog_helper.get_viewer_url('unified_logcats')) |
812 | 821 |
813 | 822 |
814 logcats_uploader = contextlib_ext.Optional( | 823 logcats_uploader = contextlib_ext.Optional( |
815 upload_logcats_file(), | 824 upload_logcats_file(), |
816 'upload_logcats_file' in args and args.upload_logcats_file) | 825 'upload_logcats_file' in args and args.upload_logcats_file) |
817 | 826 |
818 ### Set up test objects. | 827 ### Set up test objects. |
819 | 828 |
820 env = environment_factory.CreateEnvironment(args, infra_error) | 829 out_manager = output_manager_factory.CreateOutputManager(args) |
830 env = environment_factory.CreateEnvironment( | |
831 args, out_manager, infra_error) | |
821 test_instance = test_instance_factory.CreateTestInstance(args, infra_error) | 832 test_instance = test_instance_factory.CreateTestInstance(args, infra_error) |
822 test_run = test_run_factory.CreateTestRun( | 833 test_run = test_run_factory.CreateTestRun( |
823 args, env, test_instance, infra_error) | 834 args, env, test_instance, infra_error) |
824 | 835 |
825 ### Run. | 836 ### Run. |
826 | 837 |
827 with json_writer, logcats_uploader, env, test_instance, test_run: | 838 with out_manager: |
jbudorick
2017/08/10 16:27:38
Can this be part of the with line below?
| |
839 with json_writer, logcats_uploader, env, test_instance, test_run: | |
828 | 840 |
829 repetitions = (xrange(args.repeat + 1) if args.repeat >= 0 | 841 repetitions = (xrange(args.repeat + 1) if args.repeat >= 0 |
830 else itertools.count()) | 842 else itertools.count()) |
831 result_counts = collections.defaultdict( | 843 result_counts = collections.defaultdict( |
832 lambda: collections.defaultdict(int)) | 844 lambda: collections.defaultdict(int)) |
833 iteration_count = 0 | 845 iteration_count = 0 |
834 for _ in repetitions: | 846 for _ in repetitions: |
835 raw_results = test_run.RunTests() | 847 raw_results = test_run.RunTests() |
836 if not raw_results: | 848 if not raw_results: |
837 continue | 849 continue |
838 | 850 |
839 all_raw_results.append(raw_results) | 851 all_raw_results.append(raw_results) |
840 | 852 |
841 iteration_results = base_test_result.TestRunResults() | 853 iteration_results = base_test_result.TestRunResults() |
842 for r in reversed(raw_results): | 854 for r in reversed(raw_results): |
843 iteration_results.AddTestRunResults(r) | 855 iteration_results.AddTestRunResults(r) |
844 all_iteration_results.append(iteration_results) | 856 all_iteration_results.append(iteration_results) |
845 | 857 |
846 iteration_count += 1 | 858 iteration_count += 1 |
847 for r in iteration_results.GetAll(): | 859 for r in iteration_results.GetAll(): |
848 result_counts[r.GetName()][r.GetType()] += 1 | 860 result_counts[r.GetName()][r.GetType()] += 1 |
849 report_results.LogFull( | 861 report_results.LogFull( |
850 results=iteration_results, | 862 results=iteration_results, |
851 test_type=test_instance.TestType(), | 863 test_type=test_instance.TestType(), |
852 test_package=test_run.TestPackage(), | 864 test_package=test_run.TestPackage(), |
853 annotation=getattr(args, 'annotations', None), | 865 annotation=getattr(args, 'annotations', None), |
854 flakiness_server=getattr(args, 'flakiness_dashboard_server', | 866 flakiness_server=getattr(args, 'flakiness_dashboard_server', |
855 None)) | 867 None)) |
856 if args.break_on_failure and not iteration_results.DidRunPass(): | 868 if args.break_on_failure and not iteration_results.DidRunPass(): |
857 break | 869 break |
858 | 870 |
859 if iteration_count > 1: | 871 if iteration_count > 1: |
860 # display summary results | 872 # display summary results |
861 # only display results for a test if at least one test did not pass | 873 # only display results for a test if at least one test did not pass |
862 all_pass = 0 | 874 all_pass = 0 |
863 tot_tests = 0 | 875 tot_tests = 0 |
864 for test_name in result_counts: | 876 for test_name in result_counts: |
865 tot_tests += 1 | 877 tot_tests += 1 |
866 if any(result_counts[test_name][x] for x in ( | 878 if any(result_counts[test_name][x] for x in ( |
867 base_test_result.ResultType.FAIL, | 879 base_test_result.ResultType.FAIL, |
868 base_test_result.ResultType.CRASH, | 880 base_test_result.ResultType.CRASH, |
869 base_test_result.ResultType.TIMEOUT, | 881 base_test_result.ResultType.TIMEOUT, |
870 base_test_result.ResultType.UNKNOWN)): | 882 base_test_result.ResultType.UNKNOWN)): |
871 logging.critical( | 883 logging.critical( |
872 '%s: %s', | 884 '%s: %s', |
873 test_name, | 885 test_name, |
874 ', '.join('%s %s' % (str(result_counts[test_name][i]), i) | 886 ', '.join('%s %s' % (str(result_counts[test_name][i]), i) |
875 for i in base_test_result.ResultType.GetTypes())) | 887 for i in base_test_result.ResultType.GetTypes())) |
876 else: | 888 else: |
877 all_pass += 1 | 889 all_pass += 1 |
878 | 890 |
879 logging.critical('%s of %s tests passed in all %s runs', | 891 logging.critical('%s of %s tests passed in all %s runs', |
880 str(all_pass), | 892 str(all_pass), |
881 str(tot_tests), | 893 str(tot_tests), |
882 str(iteration_count)) | 894 str(iteration_count)) |
895 | |
896 if args.local_output and args.json_results_file: | |
897 try: | |
898 results_detail_file = tempfile.NamedTemporaryFile(delete=False) | |
899 result_html_string, _, _ = test_results_presentation.result_details( | |
900 json_path=args.json_results_file, | |
901 test_name=args.command, | |
902 cs_base_url='http://cs.chromium.org', | |
903 local_output=True) | |
904 results_detail_file.write(result_html_string) | |
905 results_detail_file.flush() | |
906 finally: | |
907 results_detail_link = out_manager.ArchiveAndDeleteFile( | |
908 results_detail_file.name, | |
909 'test_results_presentation.html', | |
910 'test_results_presentation', | |
911 output_manager.Datatype.HTML) | |
912 logging.critical('TEST RESULTS: %s', results_detail_link) | |
883 | 913 |
884 if args.command == 'perf' and (args.steps or args.single_step): | 914 if args.command == 'perf' and (args.steps or args.single_step): |
885 return 0 | 915 return 0 |
886 | 916 |
887 return (0 if all(r.DidRunPass() for r in all_iteration_results) | 917 return (0 if all(r.DidRunPass() for r in all_iteration_results) |
888 else constants.ERROR_EXIT_CODE) | 918 else constants.ERROR_EXIT_CODE) |
889 | 919 |
890 | 920 |
891 def DumpThreadStacks(_signal, _frame): | 921 def DumpThreadStacks(_signal, _frame): |
892 for thread in threading.enumerate(): | 922 for thread in threading.enumerate(): |
(...skipping 81 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
974 if e.is_infra_error: | 1004 if e.is_infra_error: |
975 return constants.INFRA_EXIT_CODE | 1005 return constants.INFRA_EXIT_CODE |
976 return constants.ERROR_EXIT_CODE | 1006 return constants.ERROR_EXIT_CODE |
977 except: # pylint: disable=W0702 | 1007 except: # pylint: disable=W0702 |
978 logging.exception('Unrecognized error occurred.') | 1008 logging.exception('Unrecognized error occurred.') |
979 return constants.ERROR_EXIT_CODE | 1009 return constants.ERROR_EXIT_CODE |
980 | 1010 |
981 | 1011 |
982 if __name__ == '__main__': | 1012 if __name__ == '__main__': |
983 sys.exit(main()) | 1013 sys.exit(main()) |
OLD | NEW |