OLD | NEW |
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """Takes care of sharding the python-drive tests in multiple devices.""" | 5 """Takes care of sharding the python-drive tests in multiple devices.""" |
6 | 6 |
7 import copy | 7 import copy |
8 import logging | 8 import logging |
9 import multiprocessing | 9 import multiprocessing |
10 | 10 |
| 11 from pylib.base import base_test_result |
11 from pylib.base import sharded_tests_queue | 12 from pylib.base import sharded_tests_queue |
12 from pylib.base.test_result import TestResults | |
13 | 13 |
14 from python_test_caller import CallPythonTest | 14 from python_test_caller import CallPythonTest |
15 | 15 |
16 | 16 |
17 def SetTestsContainer(tests_container): | 17 def SetTestsContainer(tests_container): |
18 """Sets PythonTestSharder as a top-level field. | 18 """Sets PythonTestSharder as a top-level field. |
19 | 19 |
20 PythonTestSharder uses multiprocessing.Pool, which creates a pool of | 20 PythonTestSharder uses multiprocessing.Pool, which creates a pool of |
21 processes. This is used to initialize each worker in the pool, ensuring that | 21 processes. This is used to initialize each worker in the pool, ensuring that |
22 each worker has access to this shared pool of tests. | 22 each worker has access to this shared pool of tests. |
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
60 self.options = options | 60 self.options = options |
61 | 61 |
62 def RunTests(self): | 62 def RunTests(self): |
63 """Runs tests from the shared pool of tests, aggregating results. | 63 """Runs tests from the shared pool of tests, aggregating results. |
64 | 64 |
65 Returns: | 65 Returns: |
66 A list of test results for all of the tests which this runner executed. | 66 A list of test results for all of the tests which this runner executed. |
67 """ | 67 """ |
68 tests = PythonTestSharder.tests_container | 68 tests = PythonTestSharder.tests_container |
69 | 69 |
70 results = [] | 70 results = base_test_result.TestRunResults() |
71 for t in tests: | 71 for t in tests: |
72 res = CallPythonTest(t, self.options) | 72 results.AddTestRunResults(CallPythonTest(t, self.options)) |
73 results.append(res) | 73 return results |
74 | |
75 return TestResults.FromTestResults(results) | |
76 | 74 |
77 | 75 |
78 class PythonTestSharder(object): | 76 class PythonTestSharder(object): |
79 """Runs Python tests in parallel on multiple devices. | 77 """Runs Python tests in parallel on multiple devices. |
80 | 78 |
81 This is lifted more or less wholesale from BaseTestRunner. | 79 This is lifted more or less wholesale from BaseTestRunner. |
82 | 80 |
83 Under the covers, it creates a pool of long-lived PythonTestRunners, which | 81 Under the covers, it creates a pool of long-lived PythonTestRunners, which |
84 execute tests from the pool of tests. | 82 execute tests from the pool of tests. |
85 | 83 |
(...skipping 27 matching lines...) Expand all Loading... |
113 | 111 |
114 Returns: | 112 Returns: |
115 A list of test results aggregated from all test runs. | 113 A list of test results aggregated from all test runs. |
116 """ | 114 """ |
117 logging.warning('*' * 80) | 115 logging.warning('*' * 80) |
118 logging.warning('Sharding in ' + str(len(self.attached_devices)) + | 116 logging.warning('Sharding in ' + str(len(self.attached_devices)) + |
119 ' devices.') | 117 ' devices.') |
120 logging.warning('Note that the output is not synchronized.') | 118 logging.warning('Note that the output is not synchronized.') |
121 logging.warning('Look for the "Final result" banner in the end.') | 119 logging.warning('Look for the "Final result" banner in the end.') |
122 logging.warning('*' * 80) | 120 logging.warning('*' * 80) |
123 all_passed = [] | 121 final_results = base_test_result.TestRunResults() |
124 test_results = TestResults() | |
125 tests_to_run = self.tests | 122 tests_to_run = self.tests |
126 for retry in xrange(self.retries): | 123 for retry in xrange(self.retries): |
127 logging.warning('Try %d of %d', retry + 1, self.retries) | 124 logging.warning('Try %d of %d', retry + 1, self.retries) |
128 self._SetupSharding(self.tests) | 125 self._SetupSharding(self.tests) |
129 test_runners = self._MakeTestRunners(self.attached_devices) | 126 test_runners = self._MakeTestRunners(self.attached_devices) |
130 logging.warning('Starting...') | 127 logging.warning('Starting...') |
131 pool = multiprocessing.Pool(len(self.attached_devices), | 128 pool = multiprocessing.Pool(len(self.attached_devices), |
132 SetTestsContainer, | 129 SetTestsContainer, |
133 [PythonTestSharder.tests_container]) | 130 [PythonTestSharder.tests_container]) |
134 | 131 |
135 # List of TestResults objects from each test execution. | 132 # List of TestRunResults objects from each test execution. |
136 try: | 133 try: |
137 results_lists = pool.map(_DefaultRunnable, test_runners) | 134 results_lists = pool.map(_DefaultRunnable, test_runners) |
138 except Exception: | 135 except Exception: |
139 logging.exception('Unable to run tests. Something with the ' | 136 logging.exception('Unable to run tests. Something with the ' |
140 'PythonTestRunners has gone wrong.') | 137 'PythonTestRunners has gone wrong.') |
141 raise Exception('PythonTestRunners were unable to run tests.') | 138 raise Exception('PythonTestRunners were unable to run tests.') |
142 | 139 |
143 test_results = TestResults.FromTestResults(results_lists) | 140 test_results = base_test_result.TestRunResults() |
| 141 for t in results_lists: |
| 142 test_results.AddTestRunResults(t) |
144 # Accumulate passing results. | 143 # Accumulate passing results. |
145 all_passed += test_results.ok | 144 final_results.AddResults(test_results.GetPass()) |
146 # If we have failed tests, map them to tests to retry. | 145 # If we have failed tests, map them to tests to retry. |
147 failed_tests = test_results.GetAllBroken() | 146 failed_tests = [t.GetName() for t in test_results.GetNotPass()] |
148 tests_to_run = self._GetTestsToRetry(self.tests, | 147 tests_to_run = self._GetTestsToRetry(self.tests, failed_tests) |
149 failed_tests) | |
150 | 148 |
151 # Bail out early if we have no more tests. This can happen if all tests | 149 # Bail out early if we have no more tests. This can happen if all tests |
152 # pass before we're out of retries, for example. | 150 # pass before we're out of retries, for example. |
153 if not tests_to_run: | 151 if not tests_to_run: |
154 break | 152 break |
155 | 153 |
156 final_results = TestResults() | |
157 # all_passed has accumulated all passing test results. | 154 # all_passed has accumulated all passing test results. |
158 # test_results will have the results from the most recent run, which could | 155 # test_results will have the results from the most recent run, which could |
159 # include a variety of failure modes (unknown, crashed, failed, etc). | 156 # include a variety of failure modes (unknown, crashed, failed, etc). |
| 157 test_results.AddResults(final_results.GetPass()) |
160 final_results = test_results | 158 final_results = test_results |
161 final_results.ok = all_passed | |
162 | 159 |
163 return final_results | 160 return final_results |
164 | 161 |
165 def _MakeTestRunners(self, attached_devices): | 162 def _MakeTestRunners(self, attached_devices): |
166 """Initialize and return a list of PythonTestRunners. | 163 """Initialize and return a list of PythonTestRunners. |
167 | 164 |
168 Args: | 165 Args: |
169 attached_devices: list of device IDs attached to host. | 166 attached_devices: list of device IDs attached to host. |
170 | 167 |
171 Returns: | 168 Returns: |
172 A list of PythonTestRunners, one for each device. | 169 A list of PythonTestRunners, one for each device. |
173 """ | 170 """ |
174 test_runners = [] | 171 test_runners = [] |
175 for index, device in enumerate(attached_devices): | 172 for index, device in enumerate(attached_devices): |
176 logging.warning('*' * 80) | 173 logging.warning('*' * 80) |
177 logging.warning('Creating shard %d for %s', index, device) | 174 logging.warning('Creating shard %d for %s', index, device) |
178 logging.warning('*' * 80) | 175 logging.warning('*' * 80) |
179 # Bind the PythonTestRunner to a device & shard index. Give it the | 176 # Bind the PythonTestRunner to a device & shard index. Give it the |
180 # runnable which it will use to actually execute the tests. | 177 # runnable which it will use to actually execute the tests. |
181 test_options = copy.deepcopy(self.options) | 178 test_options = copy.deepcopy(self.options) |
182 test_options.ensure_value('device_id', device) | 179 test_options.ensure_value('device_id', device) |
183 test_options.ensure_value('shard_index', index) | 180 test_options.ensure_value('shard_index', index) |
184 test_runner = PythonTestRunner(test_options) | 181 test_runner = PythonTestRunner(test_options) |
185 test_runners.append(test_runner) | 182 test_runners.append(test_runner) |
186 | 183 |
187 return test_runners | 184 return test_runners |
188 | 185 |
189 def _GetTestsToRetry(self, available_tests, failed_tests): | 186 def _GetTestsToRetry(self, available_tests, failed_test_names): |
190 """Infers a list of tests to retry from failed tests and available tests. | 187 """Infers a list of tests to retry from failed tests and available tests. |
191 | 188 |
192 Args: | 189 Args: |
193 available_tests: a list of tests which subclass PythonTestBase. | 190 available_tests: a list of tests which subclass PythonTestBase. |
194 failed_tests: a list of SingleTestResults representing failed tests. | 191 failed_test_names: a list of failed test names. |
195 | 192 |
196 Returns: | 193 Returns: |
197 A list of test objects which correspond to test names found in | 194 A list of test objects which correspond to test names found in |
198 failed_tests, or an empty list if there is no correspondence. | 195 failed_test_names, or an empty list if there is no correspondence. |
199 """ | 196 """ |
200 failed_test_names = map(lambda t: t.test_name, failed_tests) | |
201 tests_to_retry = [t for t in available_tests | 197 tests_to_retry = [t for t in available_tests |
202 if t.qualified_name in failed_test_names] | 198 if t.qualified_name in failed_test_names] |
203 return tests_to_retry | 199 return tests_to_retry |
OLD | NEW |