OLD | NEW |
1 # Copyright 2013 The Chromium Authors. All rights reserved. | 1 # Copyright 2013 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """Dispatches tests, either sharding or replicating them. | 5 """Dispatches tests, either sharding or replicating them. |
6 | 6 |
7 Performs the following steps: | 7 Performs the following steps: |
8 * Create a test collection factory, using the given tests | 8 * Create a test collection factory, using the given tests |
9 - If sharding: test collection factory returns the same shared test collection | 9 - If sharding: test collection factory returns the same shared test collection |
10 to all test runners | 10 to all test runners |
11 - If replciating: test collection factory returns a unique test collection to | 11 - If replciating: test collection factory returns a unique test collection to |
12 each test runner, with the same set of tests in each. | 12 each test runner, with the same set of tests in each. |
13 * Create a test runner for each device. | 13 * Create a test runner for each device. |
14 * Run each test runner in its own thread, grabbing tests from the test | 14 * Run each test runner in its own thread, grabbing tests from the test |
15 collection until there are no tests left. | 15 collection until there are no tests left. |
16 """ | 16 """ |
17 | 17 |
18 import logging | 18 import logging |
19 import threading | 19 import threading |
20 | 20 |
21 from pylib import android_commands | 21 from pylib import android_commands |
22 from pylib import constants | 22 from pylib import constants |
23 from pylib.base import base_test_result | 23 from pylib.base import base_test_result |
| 24 from pylib.device import adb_wrapper |
24 from pylib.utils import reraiser_thread | 25 from pylib.utils import reraiser_thread |
25 from pylib.utils import watchdog_timer | 26 from pylib.utils import watchdog_timer |
26 | 27 |
27 | 28 |
28 DEFAULT_TIMEOUT = 7 * 60 # seven minutes | 29 DEFAULT_TIMEOUT = 7 * 60 # seven minutes |
29 | 30 |
30 | 31 |
31 class _ThreadSafeCounter(object): | 32 class _ThreadSafeCounter(object): |
32 """A threadsafe counter.""" | 33 """A threadsafe counter.""" |
33 | 34 |
(...skipping 119 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
153 | 154 |
154 def TagTestRunResults(test_run_results): | 155 def TagTestRunResults(test_run_results): |
155 """Tags all results with the last 4 digits of the device id. | 156 """Tags all results with the last 4 digits of the device id. |
156 | 157 |
157 Used when replicating tests to distinguish the same tests run on different | 158 Used when replicating tests to distinguish the same tests run on different |
158 devices. We use a set to store test results, so the hash (generated from | 159 devices. We use a set to store test results, so the hash (generated from |
159 name and tag) must be unique to be considered different results. | 160 name and tag) must be unique to be considered different results. |
160 """ | 161 """ |
161 new_test_run_results = base_test_result.TestRunResults() | 162 new_test_run_results = base_test_result.TestRunResults() |
162 for test_result in test_run_results.GetAll(): | 163 for test_result in test_run_results.GetAll(): |
163 test_result.SetName('%s_%s' % (runner.device[-4:], test_result.GetName())) | 164 test_result.SetName('%s_%s' % (runner.device_serial[-4:], |
| 165 test_result.GetName())) |
164 new_test_run_results.AddResult(test_result) | 166 new_test_run_results.AddResult(test_result) |
165 return new_test_run_results | 167 return new_test_run_results |
166 | 168 |
167 for test in test_collection: | 169 for test in test_collection: |
168 watcher.Reset() | 170 watcher.Reset() |
169 try: | 171 try: |
170 if not android_commands.IsDeviceAttached(runner.device): | 172 if runner.device_serial not in android_commands.GetAttachedDevices(): |
171 # Device is unresponsive, stop handling tests on this device. | 173 # Device is unresponsive, stop handling tests on this device. |
172 msg = 'Device %s is unresponsive.' % runner.device | 174 msg = 'Device %s is unresponsive.' % runner.device_serial |
173 logging.warning(msg) | 175 logging.warning(msg) |
174 raise android_commands.errors.DeviceUnresponsiveError(msg) | 176 raise adb_wrapper.DeviceUnreachableError(msg) |
175 result, retry = runner.RunTest(test.test) | 177 result, retry = runner.RunTest(test.test) |
176 if tag_results_with_device: | 178 if tag_results_with_device: |
177 result = TagTestRunResults(result) | 179 result = TagTestRunResults(result) |
178 test.tries += 1 | 180 test.tries += 1 |
179 if retry and test.tries <= num_retries: | 181 if retry and test.tries <= num_retries: |
180 # Retry non-passing results, only record passing results. | 182 # Retry non-passing results, only record passing results. |
181 pass_results = base_test_result.TestRunResults() | 183 pass_results = base_test_result.TestRunResults() |
182 pass_results.AddResults(result.GetPass()) | 184 pass_results.AddResults(result.GetPass()) |
183 out_results.append(pass_results) | 185 out_results.append(pass_results) |
184 logging.warning('Will retry test, try #%s.' % test.tries) | 186 logging.warning('Will retry test, try #%s.' % test.tries) |
(...skipping 23 matching lines...) Expand all Loading... |
208 device: The device serial number to set up. | 210 device: The device serial number to set up. |
209 out_runners: List to add the successfully set up TestRunner object. | 211 out_runners: List to add the successfully set up TestRunner object. |
210 threadsafe_counter: A _ThreadSafeCounter object used to get shard indices. | 212 threadsafe_counter: A _ThreadSafeCounter object used to get shard indices. |
211 """ | 213 """ |
212 try: | 214 try: |
213 index = threadsafe_counter.GetAndIncrement() | 215 index = threadsafe_counter.GetAndIncrement() |
214 logging.warning('Creating shard %s for device %s.', index, device) | 216 logging.warning('Creating shard %s for device %s.', index, device) |
215 runner = runner_factory(device, index) | 217 runner = runner_factory(device, index) |
216 runner.SetUp() | 218 runner.SetUp() |
217 out_runners.append(runner) | 219 out_runners.append(runner) |
218 except android_commands.errors.DeviceUnresponsiveError as e: | 220 except adb_wrapper.DeviceUnreachableError as e: |
219 logging.warning('Failed to create shard for %s: [%s]', device, e) | 221 logging.warning('Failed to create shard for %s: [%s]', device, e) |
220 | 222 |
221 | 223 |
222 def _RunAllTests(runners, test_collection_factory, num_retries, timeout=None, | 224 def _RunAllTests(runners, test_collection_factory, num_retries, timeout=None, |
223 tag_results_with_device=False): | 225 tag_results_with_device=False): |
224 """Run all tests using the given TestRunners. | 226 """Run all tests using the given TestRunners. |
225 | 227 |
226 Args: | 228 Args: |
227 runners: A list of TestRunner objects. | 229 runners: A list of TestRunner objects. |
228 test_collection_factory: A callable to generate a _TestCollection object for | 230 test_collection_factory: A callable to generate a _TestCollection object for |
(...skipping 12 matching lines...) Expand all Loading... |
241 results = [] | 243 results = [] |
242 exit_code = 0 | 244 exit_code = 0 |
243 run_results = base_test_result.TestRunResults() | 245 run_results = base_test_result.TestRunResults() |
244 watcher = watchdog_timer.WatchdogTimer(timeout) | 246 watcher = watchdog_timer.WatchdogTimer(timeout) |
245 test_collections = [test_collection_factory() for _ in runners] | 247 test_collections = [test_collection_factory() for _ in runners] |
246 | 248 |
247 threads = [ | 249 threads = [ |
248 reraiser_thread.ReraiserThread( | 250 reraiser_thread.ReraiserThread( |
249 _RunTestsFromQueue, | 251 _RunTestsFromQueue, |
250 [r, tc, results, watcher, num_retries, tag_results_with_device], | 252 [r, tc, results, watcher, num_retries, tag_results_with_device], |
251 name=r.device[-4:]) | 253 name=r.device_serial[-4:]) |
252 for r, tc in zip(runners, test_collections)] | 254 for r, tc in zip(runners, test_collections)] |
253 | 255 |
254 workers = reraiser_thread.ReraiserThreadGroup(threads) | 256 workers = reraiser_thread.ReraiserThreadGroup(threads) |
255 workers.StartAll() | 257 workers.StartAll() |
256 | 258 |
257 # Catch DeviceUnresponsiveErrors and set a warning exit code | 259 # Catch DeviceUnreachableErrors and set a warning exit code |
258 try: | 260 try: |
259 workers.JoinAll(watcher) | 261 workers.JoinAll(watcher) |
260 except android_commands.errors.DeviceUnresponsiveError as e: | 262 except adb_wrapper.DeviceUnreachableError as e: |
261 logging.error(e) | 263 logging.error(e) |
262 exit_code = constants.WARNING_EXIT_CODE | 264 exit_code = constants.WARNING_EXIT_CODE |
263 | 265 |
264 assert all([len(tc) == 0 for tc in test_collections]), ( | 266 assert all([len(tc) == 0 for tc in test_collections]), ( |
265 'Some tests were not run, all devices are likely offline (ran %d tests)' % | 267 'Some tests were not run, all devices are likely offline (ran %d tests)' % |
266 len(run_results.GetAll())) | 268 len(run_results.GetAll())) |
267 | 269 |
268 for r in results: | 270 for r in results: |
269 run_results.AddTestRunResults(r) | 271 run_results.AddTestRunResults(r) |
270 if not run_results.DidRunPass(): | 272 if not run_results.DidRunPass(): |
(...skipping 30 matching lines...) Expand all Loading... |
301 | 303 |
302 | 304 |
303 def _TearDownRunners(runners, timeout=None): | 305 def _TearDownRunners(runners, timeout=None): |
304 """Calls TearDown() for each test runner in parallel. | 306 """Calls TearDown() for each test runner in parallel. |
305 | 307 |
306 Args: | 308 Args: |
307 runners: A list of TestRunner objects. | 309 runners: A list of TestRunner objects. |
308 timeout: Watchdog timeout in seconds, defaults to the default timeout. | 310 timeout: Watchdog timeout in seconds, defaults to the default timeout. |
309 """ | 311 """ |
310 threads = reraiser_thread.ReraiserThreadGroup( | 312 threads = reraiser_thread.ReraiserThreadGroup( |
311 [reraiser_thread.ReraiserThread(r.TearDown, name=r.device[-4:]) | 313 [reraiser_thread.ReraiserThread(r.TearDown, name=r.device_serial[-4:]) |
312 for r in runners]) | 314 for r in runners]) |
313 threads.StartAll() | 315 threads.StartAll() |
314 threads.JoinAll(watchdog_timer.WatchdogTimer(timeout)) | 316 threads.JoinAll(watchdog_timer.WatchdogTimer(timeout)) |
315 | 317 |
316 | 318 |
317 def RunTests(tests, runner_factory, devices, shard=True, | 319 def RunTests(tests, runner_factory, devices, shard=True, |
318 test_timeout=DEFAULT_TIMEOUT, setup_timeout=DEFAULT_TIMEOUT, | 320 test_timeout=DEFAULT_TIMEOUT, setup_timeout=DEFAULT_TIMEOUT, |
319 num_retries=2): | 321 num_retries=2): |
320 """Run all tests on attached devices, retrying tests that don't pass. | 322 """Run all tests on attached devices, retrying tests that don't pass. |
321 | 323 |
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
356 log_string = 'replicated on each device' | 358 log_string = 'replicated on each device' |
357 | 359 |
358 logging.info('Will run %d tests (%s): %s', len(tests), log_string, str(tests)) | 360 logging.info('Will run %d tests (%s): %s', len(tests), log_string, str(tests)) |
359 runners = _CreateRunners(runner_factory, devices, setup_timeout) | 361 runners = _CreateRunners(runner_factory, devices, setup_timeout) |
360 try: | 362 try: |
361 return _RunAllTests(runners, test_collection_factory, | 363 return _RunAllTests(runners, test_collection_factory, |
362 num_retries, test_timeout, tag_results_with_device) | 364 num_retries, test_timeout, tag_results_with_device) |
363 finally: | 365 finally: |
364 try: | 366 try: |
365 _TearDownRunners(runners, setup_timeout) | 367 _TearDownRunners(runners, setup_timeout) |
366 except android_commands.errors.DeviceUnresponsiveError as e: | 368 except adb_wrapper.DeviceUnreachableError as e: |
367 logging.warning('Device unresponsive during TearDown: [%s]', e) | 369 logging.warning('Device unresponsive during TearDown: [%s]', e) |
OLD | NEW |