| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # | 2 # |
| 3 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 3 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 4 # Use of this source code is governed by a BSD-style license that can be | 4 # Use of this source code is governed by a BSD-style license that can be |
| 5 # found in the LICENSE file. | 5 # found in the LICENSE file. |
| 6 | 6 |
| 7 """Runs both the Python and Java tests.""" | 7 """Runs both the Python and Java tests.""" |
| 8 | 8 |
| 9 import optparse | 9 import optparse |
| 10 import os | 10 import os |
| 11 import sys | 11 import sys |
| 12 import time | 12 import time |
| 13 | 13 |
| 14 from pylib import apk_info | 14 from pylib import apk_info |
| 15 from pylib import buildbot_report | 15 from pylib import buildbot_report |
| 16 from pylib import constants | 16 from pylib import constants |
| 17 from pylib import flakiness_dashboard_results_uploader | |
| 18 from pylib import ports | 17 from pylib import ports |
| 19 from pylib import run_java_tests | 18 from pylib import run_java_tests |
| 20 from pylib import run_python_tests | 19 from pylib import run_python_tests |
| 21 from pylib import run_tests_helper | 20 from pylib import run_tests_helper |
| 22 from pylib import test_options_parser | 21 from pylib import test_options_parser |
| 23 from pylib.test_result import TestResults | 22 from pylib.test_result import TestResults |
| 24 | 23 |
| 25 | 24 |
| 26 def SummarizeResults(java_results, python_results, annotation, build_type): | |
| 27 """Summarize the results from the various test types. | |
| 28 | |
| 29 Args: | |
| 30 java_results: a TestResults object with java test case results. | |
| 31 python_results: a TestResults object with python test case results. | |
| 32 annotation: the annotation used for these results. | |
| 33 build_type: 'Release' or 'Debug'. | |
| 34 | |
| 35 Returns: | |
| 36 A tuple (all_results, summary_string, num_failing) | |
| 37 """ | |
| 38 all_results = TestResults.FromTestResults([java_results, python_results]) | |
| 39 summary_string = all_results.LogFull('Instrumentation', annotation, | |
| 40 build_type, []) | |
| 41 num_failing = (len(all_results.failed) + len(all_results.crashed) + | |
| 42 len(all_results.unknown)) | |
| 43 return all_results, summary_string, num_failing | |
| 44 | |
| 45 | |
| 46 def DispatchInstrumentationTests(options): | 25 def DispatchInstrumentationTests(options): |
| 47 """Dispatches the Java and Python instrumentation tests, sharding if possible. | 26 """Dispatches the Java and Python instrumentation tests, sharding if possible. |
| 48 | 27 |
| 49 Uses the logging module to print the combined final results and | 28 Uses the logging module to print the combined final results and |
| 50 summary of the Java and Python tests. If the java_only option is set, only | 29 summary of the Java and Python tests. If the java_only option is set, only |
| 51 the Java tests run. If the python_only option is set, only the python tests | 30 the Java tests run. If the python_only option is set, only the python tests |
| 52 run. If neither are set, run both Java and Python tests. | 31 run. If neither are set, run both Java and Python tests. |
| 53 | 32 |
| 54 Args: | 33 Args: |
| 55 options: command-line options for running the Java and Python tests. | 34 options: command-line options for running the Java and Python tests. |
| 56 | 35 |
| 57 Returns: | 36 Returns: |
| 58 An integer representing the number of failing tests. | 37 An integer representing the number of broken tests. |
| 59 """ | 38 """ |
| 60 if not options.keep_test_server_ports: | 39 if not options.keep_test_server_ports: |
| 61 # Reset the test port allocation. It's important to do it before starting | 40 # Reset the test port allocation. It's important to do it before starting |
| 62 # to dispatch any tests. | 41 # to dispatch any tests. |
| 63 if not ports.ResetTestServerPortAllocation(): | 42 if not ports.ResetTestServerPortAllocation(): |
| 64 raise Exception('Failed to reset test server port.') | 43 raise Exception('Failed to reset test server port.') |
| 65 | 44 |
| 66 start_date = int(time.time() * 1000) | 45 start_date = int(time.time() * 1000) |
| 67 java_results = TestResults() | 46 java_results = TestResults() |
| 68 python_results = TestResults() | 47 python_results = TestResults() |
| 69 | 48 |
| 70 if options.run_java_tests: | 49 if options.run_java_tests: |
| 71 java_results = run_java_tests.DispatchJavaTests( | 50 java_results = run_java_tests.DispatchJavaTests( |
| 72 options, | 51 options, |
| 73 [apk_info.ApkInfo(options.test_apk_path, options.test_apk_jar_path)]) | 52 [apk_info.ApkInfo(options.test_apk_path, options.test_apk_jar_path)]) |
| 74 if options.run_python_tests: | 53 if options.run_python_tests: |
| 75 python_results = run_python_tests.DispatchPythonTests(options) | 54 python_results = run_python_tests.DispatchPythonTests(options) |
| 76 | 55 |
| 77 all_results, summary_string, num_failing = SummarizeResults( | 56 all_results = TestResults.FromTestResults([java_results, python_results]) |
| 78 java_results, python_results, options.annotation, options.build_type) | |
| 79 | 57 |
| 80 if options.flakiness_dashboard_server: | 58 all_results.LogFull( |
| 81 flakiness_dashboard_results_uploader.Upload( | 59 test_type='Instrumentation', |
| 82 options.flakiness_dashboard_server, 'Chromium_Android_Instrumentation', | 60 test_package=options.test_apk, |
| 83 TestResults.FromTestResults([java_results, python_results])) | 61 annotation=options.annotation, |
| 62 build_type=options.build_type, |
| 63 flakiness_server=options.flakiness_dashboard_server) |
| 84 | 64 |
| 85 return num_failing | 65 return len(all_results.GetAllBroken()) |
| 86 | 66 |
| 87 | 67 |
| 88 def main(argv): | 68 def main(argv): |
| 89 option_parser = optparse.OptionParser() | 69 option_parser = optparse.OptionParser() |
| 90 test_options_parser.AddInstrumentationOptions(option_parser) | 70 test_options_parser.AddInstrumentationOptions(option_parser) |
| 91 options, args = option_parser.parse_args(argv) | 71 options, args = option_parser.parse_args(argv) |
| 92 test_options_parser.ValidateInstrumentationOptions(option_parser, options, | 72 test_options_parser.ValidateInstrumentationOptions(option_parser, options, |
| 93 args) | 73 args) |
| 94 | 74 |
| 95 run_tests_helper.SetLogLevel(options.verbose_count) | 75 run_tests_helper.SetLogLevel(options.verbose_count) |
| 96 buildbot_report.PrintNamedStep( | 76 buildbot_report.PrintNamedStep( |
| 97 'Instrumentation tests: %s - %s' % (', '.join(options.annotation), | 77 'Instrumentation tests: %s - %s' % (', '.join(options.annotation), |
| 98 options.test_apk)) | 78 options.test_apk)) |
| 99 ret = 1 | 79 ret = 1 |
| 100 try: | 80 try: |
| 101 ret = DispatchInstrumentationTests(options) | 81 ret = DispatchInstrumentationTests(options) |
| 102 finally: | 82 finally: |
| 103 buildbot_report.PrintStepResultIfNeeded(options, ret) | 83 buildbot_report.PrintStepResultIfNeeded(options, ret) |
| 104 return ret | 84 return ret |
| 105 | 85 |
| 106 | 86 |
| 107 if __name__ == '__main__': | 87 if __name__ == '__main__': |
| 108 sys.exit(main(sys.argv)) | 88 sys.exit(main(sys.argv)) |
| OLD | NEW |