| OLD | NEW |
| 1 # Copyright 2013 The Chromium Authors. All rights reserved. | 1 # Copyright 2013 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """Dispatches tests, either sharding or replicating them. | 5 """Dispatches tests, either sharding or replicating them. |
| 6 | 6 |
| 7 Performs the following steps: | 7 Performs the following steps: |
| 8 * Create a test collection factory, using the given tests | 8 * Create a test collection factory, using the given tests |
| 9 - If sharding: test collection factory returns the same shared test collection | 9 - If sharding: test collection factory returns the same shared test collection |
| 10 to all test runners | 10 to all test runners |
| 11 - If replciating: test collection factory returns a unique test collection to | 11 - If replciating: test collection factory returns a unique test collection to |
| 12 each test runner, with the same set of tests in each. | 12 each test runner, with the same set of tests in each. |
| 13 * Create a test runner for each device. | 13 * Create a test runner for each device. |
| 14 * Run each test runner in its own thread, grabbing tests from the test | 14 * Run each test runner in its own thread, grabbing tests from the test |
| 15 collection until there are no tests left. | 15 collection until there are no tests left. |
| 16 """ | 16 """ |
| 17 | 17 |
| 18 import logging | 18 import logging |
| 19 import threading | 19 import threading |
| 20 | 20 |
| 21 from pylib import android_commands | 21 from pylib import android_commands |
| 22 from pylib import constants | 22 from pylib import constants |
| 23 from pylib.base import base_test_result | 23 from pylib.base import base_test_result |
| 24 from pylib.device import adb_wrapper |
| 24 from pylib.utils import reraiser_thread | 25 from pylib.utils import reraiser_thread |
| 25 from pylib.utils import watchdog_timer | 26 from pylib.utils import watchdog_timer |
| 26 | 27 |
| 27 | 28 |
| 28 DEFAULT_TIMEOUT = 7 * 60 # seven minutes | 29 DEFAULT_TIMEOUT = 7 * 60 # seven minutes |
| 29 | 30 |
| 30 | 31 |
| 31 class _ThreadSafeCounter(object): | 32 class _ThreadSafeCounter(object): |
| 32 """A threadsafe counter.""" | 33 """A threadsafe counter.""" |
| 33 | 34 |
| (...skipping 119 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 153 | 154 |
| 154 def TagTestRunResults(test_run_results): | 155 def TagTestRunResults(test_run_results): |
| 155 """Tags all results with the last 4 digits of the device id. | 156 """Tags all results with the last 4 digits of the device id. |
| 156 | 157 |
| 157 Used when replicating tests to distinguish the same tests run on different | 158 Used when replicating tests to distinguish the same tests run on different |
| 158 devices. We use a set to store test results, so the hash (generated from | 159 devices. We use a set to store test results, so the hash (generated from |
| 159 name and tag) must be unique to be considered different results. | 160 name and tag) must be unique to be considered different results. |
| 160 """ | 161 """ |
| 161 new_test_run_results = base_test_result.TestRunResults() | 162 new_test_run_results = base_test_result.TestRunResults() |
| 162 for test_result in test_run_results.GetAll(): | 163 for test_result in test_run_results.GetAll(): |
| 163 test_result.SetName('%s_%s' % (runner.device[-4:], test_result.GetName())) | 164 test_result.SetName('%s_%s' % (runner.device_serial[-4:], |
| 165 test_result.GetName())) |
| 164 new_test_run_results.AddResult(test_result) | 166 new_test_run_results.AddResult(test_result) |
| 165 return new_test_run_results | 167 return new_test_run_results |
| 166 | 168 |
| 167 for test in test_collection: | 169 for test in test_collection: |
| 168 watcher.Reset() | 170 watcher.Reset() |
| 169 try: | 171 try: |
| 170 if not android_commands.IsDeviceAttached(runner.device): | 172 if runner.device_serial not in android_commands.GetAttachedDevices(): |
| 171 # Device is unresponsive, stop handling tests on this device. | 173 # Device is unresponsive, stop handling tests on this device. |
| 172 msg = 'Device %s is unresponsive.' % runner.device | 174 msg = 'Device %s is unresponsive.' % runner.device_serial |
| 173 logging.warning(msg) | 175 logging.warning(msg) |
| 174 raise android_commands.errors.DeviceUnresponsiveError(msg) | 176 raise adb_wrapper.DeviceUnreachableError(msg) |
| 175 result, retry = runner.RunTest(test.test) | 177 result, retry = runner.RunTest(test.test) |
| 176 if tag_results_with_device: | 178 if tag_results_with_device: |
| 177 result = TagTestRunResults(result) | 179 result = TagTestRunResults(result) |
| 178 test.tries += 1 | 180 test.tries += 1 |
| 179 if retry and test.tries <= num_retries: | 181 if retry and test.tries <= num_retries: |
| 180 # Retry non-passing results, only record passing results. | 182 # Retry non-passing results, only record passing results. |
| 181 pass_results = base_test_result.TestRunResults() | 183 pass_results = base_test_result.TestRunResults() |
| 182 pass_results.AddResults(result.GetPass()) | 184 pass_results.AddResults(result.GetPass()) |
| 183 out_results.append(pass_results) | 185 out_results.append(pass_results) |
| 184 logging.warning('Will retry test, try #%s.' % test.tries) | 186 logging.warning('Will retry test, try #%s.' % test.tries) |
| (...skipping 23 matching lines...) Expand all Loading... |
| 208 device: The device serial number to set up. | 210 device: The device serial number to set up. |
| 209 out_runners: List to add the successfully set up TestRunner object. | 211 out_runners: List to add the successfully set up TestRunner object. |
| 210 threadsafe_counter: A _ThreadSafeCounter object used to get shard indices. | 212 threadsafe_counter: A _ThreadSafeCounter object used to get shard indices. |
| 211 """ | 213 """ |
| 212 try: | 214 try: |
| 213 index = threadsafe_counter.GetAndIncrement() | 215 index = threadsafe_counter.GetAndIncrement() |
| 214 logging.warning('Creating shard %s for device %s.', index, device) | 216 logging.warning('Creating shard %s for device %s.', index, device) |
| 215 runner = runner_factory(device, index) | 217 runner = runner_factory(device, index) |
| 216 runner.SetUp() | 218 runner.SetUp() |
| 217 out_runners.append(runner) | 219 out_runners.append(runner) |
| 218 except android_commands.errors.DeviceUnresponsiveError as e: | 220 except adb_wrapper.DeviceUnreachableError as e: |
| 219 logging.warning('Failed to create shard for %s: [%s]', device, e) | 221 logging.warning('Failed to create shard for %s: [%s]', device, e) |
| 220 | 222 |
| 221 | 223 |
| 222 def _RunAllTests(runners, test_collection_factory, num_retries, timeout=None, | 224 def _RunAllTests(runners, test_collection_factory, num_retries, timeout=None, |
| 223 tag_results_with_device=False): | 225 tag_results_with_device=False): |
| 224 """Run all tests using the given TestRunners. | 226 """Run all tests using the given TestRunners. |
| 225 | 227 |
| 226 Args: | 228 Args: |
| 227 runners: A list of TestRunner objects. | 229 runners: A list of TestRunner objects. |
| 228 test_collection_factory: A callable to generate a _TestCollection object for | 230 test_collection_factory: A callable to generate a _TestCollection object for |
| (...skipping 12 matching lines...) Expand all Loading... |
| 241 results = [] | 243 results = [] |
| 242 exit_code = 0 | 244 exit_code = 0 |
| 243 run_results = base_test_result.TestRunResults() | 245 run_results = base_test_result.TestRunResults() |
| 244 watcher = watchdog_timer.WatchdogTimer(timeout) | 246 watcher = watchdog_timer.WatchdogTimer(timeout) |
| 245 test_collections = [test_collection_factory() for _ in runners] | 247 test_collections = [test_collection_factory() for _ in runners] |
| 246 | 248 |
| 247 threads = [ | 249 threads = [ |
| 248 reraiser_thread.ReraiserThread( | 250 reraiser_thread.ReraiserThread( |
| 249 _RunTestsFromQueue, | 251 _RunTestsFromQueue, |
| 250 [r, tc, results, watcher, num_retries, tag_results_with_device], | 252 [r, tc, results, watcher, num_retries, tag_results_with_device], |
| 251 name=r.device[-4:]) | 253 name=r.device_serial[-4:]) |
| 252 for r, tc in zip(runners, test_collections)] | 254 for r, tc in zip(runners, test_collections)] |
| 253 | 255 |
| 254 workers = reraiser_thread.ReraiserThreadGroup(threads) | 256 workers = reraiser_thread.ReraiserThreadGroup(threads) |
| 255 workers.StartAll() | 257 workers.StartAll() |
| 256 | 258 |
| 257 # Catch DeviceUnresponsiveErrors and set a warning exit code | 259 # Catch DeviceUnreachableErrors and set a warning exit code |
| 258 try: | 260 try: |
| 259 workers.JoinAll(watcher) | 261 workers.JoinAll(watcher) |
| 260 except android_commands.errors.DeviceUnresponsiveError as e: | 262 except adb_wrapper.DeviceUnreachableError as e: |
| 261 logging.error(e) | 263 logging.error(e) |
| 262 exit_code = constants.WARNING_EXIT_CODE | 264 exit_code = constants.WARNING_EXIT_CODE |
| 263 | 265 |
| 264 assert all([len(tc) == 0 for tc in test_collections]), ( | 266 assert all([len(tc) == 0 for tc in test_collections]), ( |
| 265 'Some tests were not run, all devices are likely offline (ran %d tests)' % | 267 'Some tests were not run, all devices are likely offline (ran %d tests)' % |
| 266 len(run_results.GetAll())) | 268 len(run_results.GetAll())) |
| 267 | 269 |
| 268 for r in results: | 270 for r in results: |
| 269 run_results.AddTestRunResults(r) | 271 run_results.AddTestRunResults(r) |
| 270 if not run_results.DidRunPass(): | 272 if not run_results.DidRunPass(): |
| (...skipping 30 matching lines...) Expand all Loading... |
| 301 | 303 |
| 302 | 304 |
| 303 def _TearDownRunners(runners, timeout=None): | 305 def _TearDownRunners(runners, timeout=None): |
| 304 """Calls TearDown() for each test runner in parallel. | 306 """Calls TearDown() for each test runner in parallel. |
| 305 | 307 |
| 306 Args: | 308 Args: |
| 307 runners: A list of TestRunner objects. | 309 runners: A list of TestRunner objects. |
| 308 timeout: Watchdog timeout in seconds, defaults to the default timeout. | 310 timeout: Watchdog timeout in seconds, defaults to the default timeout. |
| 309 """ | 311 """ |
| 310 threads = reraiser_thread.ReraiserThreadGroup( | 312 threads = reraiser_thread.ReraiserThreadGroup( |
| 311 [reraiser_thread.ReraiserThread(r.TearDown, name=r.device[-4:]) | 313 [reraiser_thread.ReraiserThread(r.TearDown, name=r.device_serial[-4:]) |
| 312 for r in runners]) | 314 for r in runners]) |
| 313 threads.StartAll() | 315 threads.StartAll() |
| 314 threads.JoinAll(watchdog_timer.WatchdogTimer(timeout)) | 316 threads.JoinAll(watchdog_timer.WatchdogTimer(timeout)) |
| 315 | 317 |
| 316 | 318 |
| 317 def RunTests(tests, runner_factory, devices, shard=True, | 319 def RunTests(tests, runner_factory, devices, shard=True, |
| 318 test_timeout=DEFAULT_TIMEOUT, setup_timeout=DEFAULT_TIMEOUT, | 320 test_timeout=DEFAULT_TIMEOUT, setup_timeout=DEFAULT_TIMEOUT, |
| 319 num_retries=2): | 321 num_retries=2): |
| 320 """Run all tests on attached devices, retrying tests that don't pass. | 322 """Run all tests on attached devices, retrying tests that don't pass. |
| 321 | 323 |
| (...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 356 log_string = 'replicated on each device' | 358 log_string = 'replicated on each device' |
| 357 | 359 |
| 358 logging.info('Will run %d tests (%s): %s', len(tests), log_string, str(tests)) | 360 logging.info('Will run %d tests (%s): %s', len(tests), log_string, str(tests)) |
| 359 runners = _CreateRunners(runner_factory, devices, setup_timeout) | 361 runners = _CreateRunners(runner_factory, devices, setup_timeout) |
| 360 try: | 362 try: |
| 361 return _RunAllTests(runners, test_collection_factory, | 363 return _RunAllTests(runners, test_collection_factory, |
| 362 num_retries, test_timeout, tag_results_with_device) | 364 num_retries, test_timeout, tag_results_with_device) |
| 363 finally: | 365 finally: |
| 364 try: | 366 try: |
| 365 _TearDownRunners(runners, setup_timeout) | 367 _TearDownRunners(runners, setup_timeout) |
| 366 except android_commands.errors.DeviceUnresponsiveError as e: | 368 except adb_wrapper.DeviceUnreachableError as e: |
| 367 logging.warning('Device unresponsive during TearDown: [%s]', e) | 369 logging.warning('Device unresponsive during TearDown: [%s]', e) |
| OLD | NEW |