| OLD | NEW |
| 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """Takes care of sharding the python-drive tests in multiple devices.""" | 5 """Takes care of sharding the python-drive tests in multiple devices.""" |
| 6 | 6 |
| 7 import copy | 7 import copy |
| 8 import logging | 8 import logging |
| 9 import multiprocessing | 9 import multiprocessing |
| 10 | 10 |
| 11 from pylib.base import base_test_result |
| 11 from pylib.base import sharded_tests_queue | 12 from pylib.base import sharded_tests_queue |
| 12 from pylib.base.test_result import TestResults | |
| 13 | 13 |
| 14 from python_test_caller import CallPythonTest | 14 from python_test_caller import CallPythonTest |
| 15 | 15 |
| 16 | 16 |
| 17 def SetTestsContainer(tests_container): | 17 def SetTestsContainer(tests_container): |
| 18 """Sets PythonTestSharder as a top-level field. | 18 """Sets PythonTestSharder as a top-level field. |
| 19 | 19 |
| 20 PythonTestSharder uses multiprocessing.Pool, which creates a pool of | 20 PythonTestSharder uses multiprocessing.Pool, which creates a pool of |
| 21 processes. This is used to initialize each worker in the pool, ensuring that | 21 processes. This is used to initialize each worker in the pool, ensuring that |
| 22 each worker has access to this shared pool of tests. | 22 each worker has access to this shared pool of tests. |
| (...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 60 self.options = options | 60 self.options = options |
| 61 | 61 |
| 62 def RunTests(self): | 62 def RunTests(self): |
| 63 """Runs tests from the shared pool of tests, aggregating results. | 63 """Runs tests from the shared pool of tests, aggregating results. |
| 64 | 64 |
| 65 Returns: | 65 Returns: |
| 66 A list of test results for all of the tests which this runner executed. | 66 A list of test results for all of the tests which this runner executed. |
| 67 """ | 67 """ |
| 68 tests = PythonTestSharder.tests_container | 68 tests = PythonTestSharder.tests_container |
| 69 | 69 |
| 70 results = [] | 70 results = base_test_result.TestRunResults() |
| 71 for t in tests: | 71 for t in tests: |
| 72 res = CallPythonTest(t, self.options) | 72 results.AddTestRunResults(CallPythonTest(t, self.options)) |
| 73 results.append(res) | 73 return results |
| 74 | |
| 75 return TestResults.FromTestResults(results) | |
| 76 | 74 |
| 77 | 75 |
| 78 class PythonTestSharder(object): | 76 class PythonTestSharder(object): |
| 79 """Runs Python tests in parallel on multiple devices. | 77 """Runs Python tests in parallel on multiple devices. |
| 80 | 78 |
| 81 This is lifted more or less wholesale from BaseTestRunner. | 79 This is lifted more or less wholesale from BaseTestRunner. |
| 82 | 80 |
| 83 Under the covers, it creates a pool of long-lived PythonTestRunners, which | 81 Under the covers, it creates a pool of long-lived PythonTestRunners, which |
| 84 execute tests from the pool of tests. | 82 execute tests from the pool of tests. |
| 85 | 83 |
| (...skipping 27 matching lines...) Expand all Loading... |
| 113 | 111 |
| 114 Returns: | 112 Returns: |
| 115 A list of test results aggregated from all test runs. | 113 A list of test results aggregated from all test runs. |
| 116 """ | 114 """ |
| 117 logging.warning('*' * 80) | 115 logging.warning('*' * 80) |
| 118 logging.warning('Sharding in ' + str(len(self.attached_devices)) + | 116 logging.warning('Sharding in ' + str(len(self.attached_devices)) + |
| 119 ' devices.') | 117 ' devices.') |
| 120 logging.warning('Note that the output is not synchronized.') | 118 logging.warning('Note that the output is not synchronized.') |
| 121 logging.warning('Look for the "Final result" banner in the end.') | 119 logging.warning('Look for the "Final result" banner in the end.') |
| 122 logging.warning('*' * 80) | 120 logging.warning('*' * 80) |
| 123 all_passed = [] | 121 final_results = base_test_result.TestRunResults() |
| 124 test_results = TestResults() | |
| 125 tests_to_run = self.tests | 122 tests_to_run = self.tests |
| 126 for retry in xrange(self.retries): | 123 for retry in xrange(self.retries): |
| 127 logging.warning('Try %d of %d', retry + 1, self.retries) | 124 logging.warning('Try %d of %d', retry + 1, self.retries) |
| 128 self._SetupSharding(self.tests) | 125 self._SetupSharding(self.tests) |
| 129 test_runners = self._MakeTestRunners(self.attached_devices) | 126 test_runners = self._MakeTestRunners(self.attached_devices) |
| 130 logging.warning('Starting...') | 127 logging.warning('Starting...') |
| 131 pool = multiprocessing.Pool(len(self.attached_devices), | 128 pool = multiprocessing.Pool(len(self.attached_devices), |
| 132 SetTestsContainer, | 129 SetTestsContainer, |
| 133 [PythonTestSharder.tests_container]) | 130 [PythonTestSharder.tests_container]) |
| 134 | 131 |
| 135 # List of TestResults objects from each test execution. | 132 # List of TestRunResults objects from each test execution. |
| 136 try: | 133 try: |
| 137 results_lists = pool.map(_DefaultRunnable, test_runners) | 134 results_lists = pool.map(_DefaultRunnable, test_runners) |
| 138 except Exception: | 135 except Exception: |
| 139 logging.exception('Unable to run tests. Something with the ' | 136 logging.exception('Unable to run tests. Something with the ' |
| 140 'PythonTestRunners has gone wrong.') | 137 'PythonTestRunners has gone wrong.') |
| 141 raise Exception('PythonTestRunners were unable to run tests.') | 138 raise Exception('PythonTestRunners were unable to run tests.') |
| 142 | 139 |
| 143 test_results = TestResults.FromTestResults(results_lists) | 140 test_results = base_test_result.TestRunResults() |
| 141 for t in results_lists: |
| 142 test_results.AddTestRunResults(t) |
| 144 # Accumulate passing results. | 143 # Accumulate passing results. |
| 145 all_passed += test_results.ok | 144 final_results.AddResults(test_results.GetPass()) |
| 146 # If we have failed tests, map them to tests to retry. | 145 # If we have failed tests, map them to tests to retry. |
| 147 failed_tests = test_results.GetAllBroken() | 146 failed_tests = [t.GetName() for t in test_results.GetNotPass()] |
| 148 tests_to_run = self._GetTestsToRetry(self.tests, | 147 tests_to_run = self._GetTestsToRetry(self.tests, failed_tests) |
| 149 failed_tests) | |
| 150 | 148 |
| 151 # Bail out early if we have no more tests. This can happen if all tests | 149 # Bail out early if we have no more tests. This can happen if all tests |
| 152 # pass before we're out of retries, for example. | 150 # pass before we're out of retries, for example. |
| 153 if not tests_to_run: | 151 if not tests_to_run: |
| 154 break | 152 break |
| 155 | 153 |
| 156 final_results = TestResults() | |
| 157 # all_passed has accumulated all passing test results. | 154 # all_passed has accumulated all passing test results. |
| 158 # test_results will have the results from the most recent run, which could | 155 # test_results will have the results from the most recent run, which could |
| 159 # include a variety of failure modes (unknown, crashed, failed, etc). | 156 # include a variety of failure modes (unknown, crashed, failed, etc). |
| 157 test_results.AddResults(final_results.GetPass()) |
| 160 final_results = test_results | 158 final_results = test_results |
| 161 final_results.ok = all_passed | |
| 162 | 159 |
| 163 return final_results | 160 return final_results |
| 164 | 161 |
| 165 def _MakeTestRunners(self, attached_devices): | 162 def _MakeTestRunners(self, attached_devices): |
| 166 """Initialize and return a list of PythonTestRunners. | 163 """Initialize and return a list of PythonTestRunners. |
| 167 | 164 |
| 168 Args: | 165 Args: |
| 169 attached_devices: list of device IDs attached to host. | 166 attached_devices: list of device IDs attached to host. |
| 170 | 167 |
| 171 Returns: | 168 Returns: |
| 172 A list of PythonTestRunners, one for each device. | 169 A list of PythonTestRunners, one for each device. |
| 173 """ | 170 """ |
| 174 test_runners = [] | 171 test_runners = [] |
| 175 for index, device in enumerate(attached_devices): | 172 for index, device in enumerate(attached_devices): |
| 176 logging.warning('*' * 80) | 173 logging.warning('*' * 80) |
| 177 logging.warning('Creating shard %d for %s', index, device) | 174 logging.warning('Creating shard %d for %s', index, device) |
| 178 logging.warning('*' * 80) | 175 logging.warning('*' * 80) |
| 179 # Bind the PythonTestRunner to a device & shard index. Give it the | 176 # Bind the PythonTestRunner to a device & shard index. Give it the |
| 180 # runnable which it will use to actually execute the tests. | 177 # runnable which it will use to actually execute the tests. |
| 181 test_options = copy.deepcopy(self.options) | 178 test_options = copy.deepcopy(self.options) |
| 182 test_options.ensure_value('device_id', device) | 179 test_options.ensure_value('device_id', device) |
| 183 test_options.ensure_value('shard_index', index) | 180 test_options.ensure_value('shard_index', index) |
| 184 test_runner = PythonTestRunner(test_options) | 181 test_runner = PythonTestRunner(test_options) |
| 185 test_runners.append(test_runner) | 182 test_runners.append(test_runner) |
| 186 | 183 |
| 187 return test_runners | 184 return test_runners |
| 188 | 185 |
| 189 def _GetTestsToRetry(self, available_tests, failed_tests): | 186 def _GetTestsToRetry(self, available_tests, failed_test_names): |
| 190 """Infers a list of tests to retry from failed tests and available tests. | 187 """Infers a list of tests to retry from failed tests and available tests. |
| 191 | 188 |
| 192 Args: | 189 Args: |
| 193 available_tests: a list of tests which subclass PythonTestBase. | 190 available_tests: a list of tests which subclass PythonTestBase. |
| 194 failed_tests: a list of SingleTestResults representing failed tests. | 191 failed_test_names: a list of failed test names. |
| 195 | 192 |
| 196 Returns: | 193 Returns: |
| 197 A list of test objects which correspond to test names found in | 194 A list of test objects which correspond to test names found in |
| 198 failed_tests, or an empty list if there is no correspondence. | 195 failed_test_names, or an empty list if there is no correspondence. |
| 199 """ | 196 """ |
| 200 failed_test_names = map(lambda t: t.test_name, failed_tests) | |
| 201 tests_to_retry = [t for t in available_tests | 197 tests_to_retry = [t for t in available_tests |
| 202 if t.qualified_name in failed_test_names] | 198 if t.qualified_name in failed_test_names] |
| 203 return tests_to_retry | 199 return tests_to_retry |
| OLD | NEW |