OLD | NEW |
1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """Implements test sharding logic.""" | 5 """Implements test sharding logic.""" |
6 | 6 |
7 import logging | 7 import logging |
8 import threading | 8 import threading |
9 | 9 |
10 from pylib import android_commands | 10 from pylib import android_commands |
| 11 from pylib import constants |
| 12 from pylib import exit_code |
11 from pylib import forwarder | 13 from pylib import forwarder |
12 from pylib.utils import reraiser_thread | 14 from pylib.utils import reraiser_thread |
13 from pylib.utils import watchdog_timer | 15 from pylib.utils import watchdog_timer |
14 | 16 |
15 import base_test_result | 17 import base_test_result |
16 | 18 |
17 | 19 |
18 DEFAULT_TIMEOUT = 7 * 60 # seven minutes | 20 DEFAULT_TIMEOUT = 7 * 60 # seven minutes |
19 | 21 |
20 | 22 |
(...skipping 64 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
85 try: | 87 try: |
86 return self._tests.pop(0) | 88 return self._tests.pop(0) |
87 except IndexError: | 89 except IndexError: |
88 # Another thread beat us to the avaliable test, wait again. | 90 # Another thread beat us to the avaliable test, wait again. |
89 self._item_avaliable_or_all_done.clear() | 91 self._item_avaliable_or_all_done.clear() |
90 | 92 |
91 def add(self, test): | 93 def add(self, test): |
92 """Add an test to the collection. | 94 """Add an test to the collection. |
93 | 95 |
94 Args: | 96 Args: |
95 item: A test to add. | 97 test: A test to add. |
96 """ | 98 """ |
97 with self._lock: | 99 with self._lock: |
98 self._tests.append(test) | 100 self._tests.append(test) |
99 self._item_avaliable_or_all_done.set() | 101 self._item_avaliable_or_all_done.set() |
100 self._tests_in_progress += 1 | 102 self._tests_in_progress += 1 |
101 | 103 |
102 def test_completed(self): | 104 def test_completed(self): |
103 """Indicate that a test has been fully handled.""" | 105 """Indicate that a test has been fully handled.""" |
104 with self._lock: | 106 with self._lock: |
105 self._tests_in_progress -= 1 | 107 self._tests_in_progress -= 1 |
106 if self._tests_in_progress == 0: | 108 if self._tests_in_progress == 0: |
107 # All tests have been handled, signal all waiting threads. | 109 # All tests have been handled, signal all waiting threads. |
108 self._item_avaliable_or_all_done.set() | 110 self._item_avaliable_or_all_done.set() |
109 | 111 |
110 def __iter__(self): | 112 def __iter__(self): |
111 """Iterate through tests in the collection until all have been handled.""" | 113 """Iterate through tests in the collection until all have been handled.""" |
112 while True: | 114 while True: |
113 r = self._pop() | 115 r = self._pop() |
114 if r is None: | 116 if r is None: |
115 break | 117 break |
116 yield r | 118 yield r |
117 | 119 |
118 | 120 |
119 def _RunTestsFromQueue(runner, test_collection, out_results, watcher, | 121 def _RunTestsFromQueue(runner, test_collection, out_results, watcher, |
120 num_retries): | 122 num_retries): |
121 """Runs tests from the test_collection until empty using the given runner. | 123 """Runs tests from the test_collection until empty using the given runner. |
122 | 124 |
123 Adds TestRunResults objects to the out_results list and may add tests to the | 125 Adds TestRunResults objects to the out_results list and may add tests to the |
124 out_retry list. | 126 out_retry list. |
125 | 127 |
126 Args: | 128 Args: |
127 runner: A TestRunner object used to run the tests. | 129 runner: A TestRunner object used to run the tests. |
128 test_collection: A _TestCollection from which to get _Test objects to run. | 130 test_collection: A _TestCollection from which to get _Test objects to run. |
129 out_results: A list to add TestRunResults to. | 131 out_results: A list to add TestRunResults to. |
130 watcher: A watchdog_timer.WatchdogTimer object, used as a shared timeout. | 132 watcher: A watchdog_timer.WatchdogTimer object, used as a shared timeout. |
(...skipping 12 matching lines...) Expand all Loading... |
143 if retry and test.tries <= num_retries: | 145 if retry and test.tries <= num_retries: |
144 # Retry non-passing results, only record passing results. | 146 # Retry non-passing results, only record passing results. |
145 pass_results = base_test_result.TestRunResults() | 147 pass_results = base_test_result.TestRunResults() |
146 pass_results.AddResults(result.GetPass()) | 148 pass_results.AddResults(result.GetPass()) |
147 out_results.append(pass_results) | 149 out_results.append(pass_results) |
148 logging.warning('Will retry test, try #%s.' % test.tries) | 150 logging.warning('Will retry test, try #%s.' % test.tries) |
149 test_collection.add(_Test(test=retry, tries=test.tries)) | 151 test_collection.add(_Test(test=retry, tries=test.tries)) |
150 else: | 152 else: |
151 # All tests passed or retry limit reached. Either way, record results. | 153 # All tests passed or retry limit reached. Either way, record results. |
152 out_results.append(result) | 154 out_results.append(result) |
153 except android_commands.errors.DeviceUnresponsiveError: | |
154 # Device is unresponsive, stop handling tests on this device and ensure | |
155 # current test gets runs by another device. Don't reraise this exception | |
156 # on the main thread. | |
157 test_collection.add(test) | |
158 return | |
159 except: | 155 except: |
160 # An unhandleable exception, ensure tests get run by another device and | 156 # An unhandleable exception, ensure tests get run by another device and |
161 # reraise this exception on the main thread. | 157 # reraise this exception on the main thread. |
162 test_collection.add(test) | 158 test_collection.add(test) |
163 raise | 159 raise |
164 finally: | 160 finally: |
165 # Retries count as separate tasks so always mark the popped test as done. | 161 # Retries count as separate tasks so always mark the popped test as done. |
166 test_collection.test_completed() | 162 test_collection.test_completed() |
167 | 163 |
168 | 164 |
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
205 (len(tests), len(runners))) | 201 (len(tests), len(runners))) |
206 tests_collection = _TestCollection([_Test(t) for t in tests]) | 202 tests_collection = _TestCollection([_Test(t) for t in tests]) |
207 results = [] | 203 results = [] |
208 watcher = watchdog_timer.WatchdogTimer(timeout) | 204 watcher = watchdog_timer.WatchdogTimer(timeout) |
209 workers = reraiser_thread.ReraiserThreadGroup( | 205 workers = reraiser_thread.ReraiserThreadGroup( |
210 [reraiser_thread.ReraiserThread( | 206 [reraiser_thread.ReraiserThread( |
211 _RunTestsFromQueue, | 207 _RunTestsFromQueue, |
212 [r, tests_collection, results, watcher, num_retries], | 208 [r, tests_collection, results, watcher, num_retries], |
213 name=r.device[-4:]) | 209 name=r.device[-4:]) |
214 for r in runners]) | 210 for r in runners]) |
| 211 run_results = base_test_result.TestRunResults() |
215 workers.StartAll() | 212 workers.StartAll() |
216 workers.JoinAll(watcher) | 213 |
217 run_results = base_test_result.TestRunResults() | 214 # Catch DeviceUnresponsiveErrors and set a warning exit code |
| 215 try: |
| 216 workers.JoinAll(watcher) |
| 217 except android_commands.errors.DeviceUnresponsiveError as e: |
| 218 logging.error(e) |
| 219 exit_code.UpdateExitCode(constants.WARNING_EXIT_CODE) |
| 220 |
218 for r in results: | 221 for r in results: |
219 run_results.AddTestRunResults(r) | 222 run_results.AddTestRunResults(r) |
220 return run_results | 223 return run_results |
221 | 224 |
222 | 225 |
223 def _CreateRunners(runner_factory, devices, timeout=None): | 226 def _CreateRunners(runner_factory, devices, timeout=None): |
224 """Creates a test runner for each device and calls SetUp() in parallel. | 227 """Creates a test runner for each device and calls SetUp() in parallel. |
225 | 228 |
226 Note: if a device is unresponsive the corresponding TestRunner will not be | 229 Note: if a device is unresponsive the corresponding TestRunner will not be |
227 included in the returned list. | 230 included in the returned list. |
(...skipping 15 matching lines...) Expand all Loading... |
243 [runner_factory, d, runners, counter], | 246 [runner_factory, d, runners, counter], |
244 name=d[-4:]) | 247 name=d[-4:]) |
245 for d in devices]) | 248 for d in devices]) |
246 threads.StartAll() | 249 threads.StartAll() |
247 threads.JoinAll(watchdog_timer.WatchdogTimer(timeout)) | 250 threads.JoinAll(watchdog_timer.WatchdogTimer(timeout)) |
248 return runners | 251 return runners |
249 | 252 |
250 | 253 |
251 def _TearDownRunners(runners, timeout=None): | 254 def _TearDownRunners(runners, timeout=None): |
252 """Calls TearDown() for each test runner in parallel. | 255 """Calls TearDown() for each test runner in parallel. |
| 256 |
253 Args: | 257 Args: |
254 runners: a list of TestRunner objects. | 258 runners: a list of TestRunner objects. |
255 timeout: watchdog timeout in seconds, defaults to the default timeout. | 259 timeout: watchdog timeout in seconds, defaults to the default timeout. |
256 """ | 260 """ |
257 threads = reraiser_thread.ReraiserThreadGroup( | 261 threads = reraiser_thread.ReraiserThreadGroup( |
258 [reraiser_thread.ReraiserThread(r.TearDown, name=r.device[-4:]) | 262 [reraiser_thread.ReraiserThread(r.TearDown, name=r.device[-4:]) |
259 for r in runners]) | 263 for r in runners]) |
260 threads.StartAll() | 264 threads.StartAll() |
261 threads.JoinAll(watchdog_timer.WatchdogTimer(timeout)) | 265 threads.JoinAll(watchdog_timer.WatchdogTimer(timeout)) |
262 | 266 |
(...skipping 28 matching lines...) Expand all Loading... |
291 runners = _CreateRunners(runner_factory, devices, setup_timeout) | 295 runners = _CreateRunners(runner_factory, devices, setup_timeout) |
292 try: | 296 try: |
293 return _RunAllTests(runners, tests, num_retries, test_timeout) | 297 return _RunAllTests(runners, tests, num_retries, test_timeout) |
294 finally: | 298 finally: |
295 try: | 299 try: |
296 _TearDownRunners(runners, setup_timeout) | 300 _TearDownRunners(runners, setup_timeout) |
297 except android_commands.errors.DeviceUnresponsiveError as e: | 301 except android_commands.errors.DeviceUnresponsiveError as e: |
298 logging.warning('Device unresponsive during TearDown: [%s]', e) | 302 logging.warning('Device unresponsive during TearDown: [%s]', e) |
299 finally: | 303 finally: |
300 forwarder.Forwarder.KillHost(build_type) | 304 forwarder.Forwarder.KillHost(build_type) |
OLD | NEW |