| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # | 2 # |
| 3 # Copyright 2013 The Chromium Authors. All rights reserved. | 3 # Copyright 2013 The Chromium Authors. All rights reserved. |
| 4 # Use of this source code is governed by a BSD-style license that can be | 4 # Use of this source code is governed by a BSD-style license that can be |
| 5 # found in the LICENSE file. | 5 # found in the LICENSE file. |
| 6 | 6 |
| 7 """Runs all types of tests from one unified interface.""" | 7 """Runs all types of tests from one unified interface.""" |
| 8 | 8 |
| 9 import argparse | 9 import argparse |
| 10 import collections | 10 import collections |
| 11 import contextlib | |
| 12 import itertools | 11 import itertools |
| 13 import logging | 12 import logging |
| 14 import os | 13 import os |
| 15 import signal | 14 import signal |
| 16 import sys | 15 import sys |
| 17 import threading | 16 import threading |
| 18 import unittest | 17 import unittest |
| 19 | 18 |
| 20 import devil_chromium | 19 import devil_chromium |
| 21 from devil import base_error | 20 from devil import base_error |
| 22 from devil.android import device_blacklist | 21 from devil.android import device_blacklist |
| 23 from devil.android import device_errors | 22 from devil.android import device_errors |
| 24 from devil.android import device_utils | 23 from devil.android import device_utils |
| 25 from devil.android import forwarder | 24 from devil.android import forwarder |
| 26 from devil.android import ports | 25 from devil.android import ports |
| 27 from devil.utils import reraiser_thread | 26 from devil.utils import reraiser_thread |
| 28 from devil.utils import run_tests_helper | 27 from devil.utils import run_tests_helper |
| 29 from devil.utils import signal_handler | |
| 30 | 28 |
| 31 from pylib import constants | 29 from pylib import constants |
| 32 from pylib.base import base_test_result | 30 from pylib.base import base_test_result |
| 33 from pylib.base import environment_factory | 31 from pylib.base import environment_factory |
| 34 from pylib.base import test_dispatcher | 32 from pylib.base import test_dispatcher |
| 35 from pylib.base import test_instance_factory | 33 from pylib.base import test_instance_factory |
| 36 from pylib.base import test_run_factory | 34 from pylib.base import test_run_factory |
| 37 from pylib.constants import host_paths | 35 from pylib.constants import host_paths |
| 38 from pylib.linker import setup as linker_setup | 36 from pylib.linker import setup as linker_setup |
| 39 from pylib.junit import setup as junit_setup | 37 from pylib.junit import setup as junit_setup |
| (...skipping 654 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 694 | 692 |
| 695 def RunTestsInPlatformMode(args): | 693 def RunTestsInPlatformMode(args): |
| 696 | 694 |
| 697 def infra_error(message): | 695 def infra_error(message): |
| 698 logging.fatal(message) | 696 logging.fatal(message) |
| 699 sys.exit(constants.INFRA_EXIT_CODE) | 697 sys.exit(constants.INFRA_EXIT_CODE) |
| 700 | 698 |
| 701 if args.command not in _SUPPORTED_IN_PLATFORM_MODE: | 699 if args.command not in _SUPPORTED_IN_PLATFORM_MODE: |
| 702 infra_error('%s is not yet supported in platform mode' % args.command) | 700 infra_error('%s is not yet supported in platform mode' % args.command) |
| 703 | 701 |
| 704 ### Set up sigterm handler. | 702 with environment_factory.CreateEnvironment(args, infra_error) as env: |
| 703 with test_instance_factory.CreateTestInstance(args, infra_error) as test: |
| 704 with test_run_factory.CreateTestRun( |
| 705 args, env, test, infra_error) as test_run: |
| 705 | 706 |
| 706 def unexpected_sigterm(_signum, _frame): | 707 # TODO(jbudorick): Rewrite results handling. |
| 707 infra_error('Received SIGTERM. Shutting down.') | |
| 708 | 708 |
| 709 sigterm_handler = signal_handler.SignalHandler( | 709 # all_raw_results is a list of lists of base_test_result.TestRunResults |
| 710 signal.SIGTERM, unexpected_sigterm) | 710 # objects. Each instance of TestRunResults contains all test results |
| 711 # produced by a single try, while each list of TestRunResults contains |
| 712 # all tries in a single iteration. |
| 713 all_raw_results = [] |
| 714 # all_iteration_results is a list of base_test_result.TestRunResults |
| 715 # objects. Each instance of TestRunResults contains the last test result |
| 716 # for each test run in that iteration. |
| 717 all_iteration_results = [] |
| 711 | 718 |
| 712 ### Set up results handling. | 719 repetitions = (xrange(args.repeat + 1) if args.repeat >= 0 |
| 713 # TODO(jbudorick): Rewrite results handling. | 720 else itertools.count()) |
| 721 result_counts = collections.defaultdict( |
| 722 lambda: collections.defaultdict(int)) |
| 723 iteration_count = 0 |
| 724 for _ in repetitions: |
| 725 raw_results = test_run.RunTests() |
| 726 if not raw_results: |
| 727 continue |
| 714 | 728 |
| 715 # all_raw_results is a list of lists of | 729 all_raw_results.append(raw_results) |
| 716 # base_test_result.TestRunResults objects. Each instance of | |
| 717 # TestRunResults contains all test results produced by a single try, | |
| 718 # while each list of TestRunResults contains all tries in a single | |
| 719 # iteration. | |
| 720 all_raw_results = [] | |
| 721 | 730 |
| 722 # all_iteration_results is a list of base_test_result.TestRunResults | 731 iteration_results = base_test_result.TestRunResults() |
| 723 # objects. Each instance of TestRunResults contains the last test | 732 for r in reversed(raw_results): |
| 724 # result for each test run in that iteration. | 733 iteration_results.AddTestRunResults(r) |
| 725 all_iteration_results = [] | 734 all_iteration_results.append(iteration_results) |
| 726 | 735 |
| 727 @contextlib.contextmanager | 736 iteration_count += 1 |
| 728 def noop(): | 737 for r in iteration_results.GetAll(): |
| 729 yield | 738 result_counts[r.GetName()][r.GetType()] += 1 |
| 739 report_results.LogFull( |
| 740 results=iteration_results, |
| 741 test_type=test.TestType(), |
| 742 test_package=test_run.TestPackage(), |
| 743 annotation=getattr(args, 'annotations', None), |
| 744 flakiness_server=getattr(args, 'flakiness_dashboard_server', |
| 745 None)) |
| 746 if args.break_on_failure and not iteration_results.DidRunPass(): |
| 747 break |
| 730 | 748 |
| 731 json_writer = noop() | 749 if iteration_count > 1: |
| 732 if args.json_results_file: | 750 # display summary results |
| 733 @contextlib.contextmanager | 751 # only display results for a test if at least one test did not pass |
| 734 def write_json_file(): | 752 all_pass = 0 |
| 735 try: | 753 tot_tests = 0 |
| 736 yield | 754 for test_name in result_counts: |
| 737 finally: | 755 tot_tests += 1 |
| 738 json_results.GenerateJsonResultsFile( | 756 if any(result_counts[test_name][x] for x in ( |
| 739 all_raw_results, args.json_results_file) | 757 base_test_result.ResultType.FAIL, |
| 758 base_test_result.ResultType.CRASH, |
| 759 base_test_result.ResultType.TIMEOUT, |
| 760 base_test_result.ResultType.UNKNOWN)): |
| 761 logging.critical( |
| 762 '%s: %s', |
| 763 test_name, |
| 764 ', '.join('%s %s' % (str(result_counts[test_name][i]), i) |
| 765 for i in base_test_result.ResultType.GetTypes())) |
| 766 else: |
| 767 all_pass += 1 |
| 740 | 768 |
| 741 json_writer = write_json_file() | 769 logging.critical('%s of %s tests passed in all %s runs', |
| 770 str(all_pass), |
| 771 str(tot_tests), |
| 772 str(iteration_count)) |
| 742 | 773 |
| 743 ### Set up test objects. | 774 if args.json_results_file: |
| 744 | 775 json_results.GenerateJsonResultsFile( |
| 745 env = environment_factory.CreateEnvironment(args, infra_error) | 776 all_raw_results, args.json_results_file) |
| 746 test_instance = test_instance_factory.CreateTestInstance(args, infra_error) | |
| 747 test_run = test_run_factory.CreateTestRun( | |
| 748 args, env, test_instance, infra_error) | |
| 749 | |
| 750 ### Run. | |
| 751 | |
| 752 with sigterm_handler, json_writer, env, test_instance, test_run: | |
| 753 | |
| 754 repetitions = (xrange(args.repeat + 1) if args.repeat >= 0 | |
| 755 else itertools.count()) | |
| 756 result_counts = collections.defaultdict( | |
| 757 lambda: collections.defaultdict(int)) | |
| 758 iteration_count = 0 | |
| 759 for _ in repetitions: | |
| 760 raw_results = test_run.RunTests() | |
| 761 if not raw_results: | |
| 762 continue | |
| 763 | |
| 764 all_raw_results.append(raw_results) | |
| 765 | |
| 766 iteration_results = base_test_result.TestRunResults() | |
| 767 for r in reversed(raw_results): | |
| 768 iteration_results.AddTestRunResults(r) | |
| 769 all_iteration_results.append(iteration_results) | |
| 770 | |
| 771 iteration_count += 1 | |
| 772 for r in iteration_results.GetAll(): | |
| 773 result_counts[r.GetName()][r.GetType()] += 1 | |
| 774 report_results.LogFull( | |
| 775 results=iteration_results, | |
| 776 test_type=test_instance.TestType(), | |
| 777 test_package=test_run.TestPackage(), | |
| 778 annotation=getattr(args, 'annotations', None), | |
| 779 flakiness_server=getattr(args, 'flakiness_dashboard_server', | |
| 780 None)) | |
| 781 if args.break_on_failure and not iteration_results.DidRunPass(): | |
| 782 break | |
| 783 | |
| 784 if iteration_count > 1: | |
| 785 # display summary results | |
| 786 # only display results for a test if at least one test did not pass | |
| 787 all_pass = 0 | |
| 788 tot_tests = 0 | |
| 789 for test_name in result_counts: | |
| 790 tot_tests += 1 | |
| 791 if any(result_counts[test_name][x] for x in ( | |
| 792 base_test_result.ResultType.FAIL, | |
| 793 base_test_result.ResultType.CRASH, | |
| 794 base_test_result.ResultType.TIMEOUT, | |
| 795 base_test_result.ResultType.UNKNOWN)): | |
| 796 logging.critical( | |
| 797 '%s: %s', | |
| 798 test_name, | |
| 799 ', '.join('%s %s' % (str(result_counts[test_name][i]), i) | |
| 800 for i in base_test_result.ResultType.GetTypes())) | |
| 801 else: | |
| 802 all_pass += 1 | |
| 803 | |
| 804 logging.critical('%s of %s tests passed in all %s runs', | |
| 805 str(all_pass), | |
| 806 str(tot_tests), | |
| 807 str(iteration_count)) | |
| 808 | 777 |
| 809 if args.command == 'perf' and (args.steps or args.single_step): | 778 if args.command == 'perf' and (args.steps or args.single_step): |
| 810 return 0 | 779 return 0 |
| 811 | 780 |
| 812 return (0 if all(r.DidRunPass() for r in all_iteration_results) | 781 return (0 if all(r.DidRunPass() for r in all_iteration_results) |
| 813 else constants.ERROR_EXIT_CODE) | 782 else constants.ERROR_EXIT_CODE) |
| 814 | 783 |
| 815 | 784 |
| 816 CommandConfigTuple = collections.namedtuple( | 785 CommandConfigTuple = collections.namedtuple( |
| 817 'CommandConfigTuple', | 786 'CommandConfigTuple', |
| (...skipping 50 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 868 if e.is_infra_error: | 837 if e.is_infra_error: |
| 869 return constants.INFRA_EXIT_CODE | 838 return constants.INFRA_EXIT_CODE |
| 870 return constants.ERROR_EXIT_CODE | 839 return constants.ERROR_EXIT_CODE |
| 871 except: # pylint: disable=W0702 | 840 except: # pylint: disable=W0702 |
| 872 logging.exception('Unrecognized error occurred.') | 841 logging.exception('Unrecognized error occurred.') |
| 873 return constants.ERROR_EXIT_CODE | 842 return constants.ERROR_EXIT_CODE |
| 874 | 843 |
| 875 | 844 |
| 876 if __name__ == '__main__': | 845 if __name__ == '__main__': |
| 877 sys.exit(main()) | 846 sys.exit(main()) |
| OLD | NEW |