[android] Fail if all devices drop offline while running tests.

build/android/pylib/base/test_dispatcher.py

 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Dispatches tests, either sharding or replicating them.
 
 Performs the following steps:
 * Create a test collection factory, using the given tests
   - If sharding: test collection factory returns the same shared test collection
     to all test runners
@@ skipping 108 matching lines @@
         self._item_avaliable_or_all_done.set()
 
   def __iter__(self):
     """Iterate through tests in the collection until all have been handled."""
     while True:
       r = self._pop()
       if r is None:
         break
       yield r
 
+  def __len__(self):
+    """Return the number of tests currently in the collection."""
+    return len(self._tests)
+
 
 def _RunTestsFromQueue(runner, test_collection, out_results, watcher,
                        num_retries, tag_results_with_device=False):
   """Runs tests from the test_collection until empty using the given runner.
 
   Adds TestRunResults objects to the out_results list and may add tests to the
   out_retry list.
 
   Args:
     runner: A TestRunner object used to run the tests.
@@ skipping 89 matching lines @@
         the test was run to the test name. Used when replicating to identify
         which device ran each copy of the test, and to ensure each copy of the
         test is recorded separately.
 
   Returns:
     A tuple of (TestRunResults object, exit code)
   """
   logging.warning('Running tests with %s test runners.' % (len(runners)))
   results = []
   exit_code = 0
+  run_results = base_test_result.TestRunResults()
   watcher = watchdog_timer.WatchdogTimer(timeout)
+  test_collections = [test_collection_factory() for _ in runners]
 
-  workers = reraiser_thread.ReraiserThreadGroup(
-      [reraiser_thread.ReraiserThread(
+  threads = [
+      reraiser_thread.ReraiserThread(
           _RunTestsFromQueue,
-          [r, test_collection_factory(), results, watcher, num_retries,
-           tag_results_with_device],
+          [r, tc, results, watcher, num_retries, tag_results_with_device],
           name=r.device[-4:])
-       for r in runners])
-  run_results = base_test_result.TestRunResults()
+      for r, tc in zip(runners, test_collections)]
+
+  workers = reraiser_thread.ReraiserThreadGroup(threads)
   workers.StartAll()
 
   # Catch DeviceUnresponsiveErrors and set a warning exit code
   try:
     workers.JoinAll(watcher)
   except android_commands.errors.DeviceUnresponsiveError as e:
     logging.error(e)
     exit_code = constants.WARNING_EXIT_CODE
 
+  assert all([len(tc) == 0 for tc in test_collections]), (
+      'Some tests were not run, all devices are likely offline (ran %d tests)' %
+      len(run_results.GetAll()))
+
   for r in results:
     run_results.AddTestRunResults(r)
   if not run_results.DidRunPass():
     exit_code = constants.ERROR_EXIT_CODE
   return (run_results, exit_code)
 
 
 def _CreateRunners(runner_factory, devices, timeout=None):
   """Creates a test runner for each device and calls SetUp() in parallel.
 
@@ skipping 80 matching lines @@
   logging.info('Will run %d tests (%s): %s', len(tests), log_string, str(tests))
   runners = _CreateRunners(runner_factory, devices, setup_timeout)
   try:
     return _RunAllTests(runners, test_collection_factory,
                         num_retries, test_timeout, tag_results_with_device)
   finally:
     try:
       _TearDownRunners(runners, setup_timeout)
     except android_commands.errors.DeviceUnresponsiveError as e:
       logging.warning('Device unresponsive during TearDown: [%s]', e)