[android] Fail if all devices drop offline while running tests.

build/android/pylib/base/test_dispatcher.py

 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Dispatches tests, either sharding or replicating them.
 
 Performs the following steps:
 * Create a test collection factory, using the given tests
   - If sharding: test collection factory returns the same shared test collection
     to all test runners
@@ skipping 108 matching lines @@
         self._item_avaliable_or_all_done.set()
 
   def __iter__(self):
     """Iterate through tests in the collection until all have been handled."""
     while True:
       r = self._pop()
       if r is None:
         break
       yield r
 
+  def __len__(self):
+    """Return the number of tests currently in the collection."""
+    return len(self._tests)
+
 
 def _RunTestsFromQueue(runner, test_collection, out_results, watcher,
                        num_retries, tag_results_with_device=False):
   """Runs tests from the test_collection until empty using the given runner.
 
   Adds TestRunResults objects to the out_results list and may add tests to the
   out_retry list.
 
   Args:
     runner: A TestRunner object used to run the tests.
@@ skipping 90 matching lines @@
         which device ran each copy of the test, and to ensure each copy of the
         test is recorded separately.
 
   Returns:
     A tuple of (TestRunResults object, exit code)
   """
   logging.warning('Running tests with %s test runners.' % (len(runners)))
   results = []
   exit_code = 0
   watcher = watchdog_timer.WatchdogTimer(timeout)
+  test_collections = [test_collection_factory() for _ in runners]
 
   workers = reraiser_thread.ReraiserThreadGroup(
       [reraiser_thread.ReraiserThread(
           _RunTestsFromQueue,
-          [r, test_collection_factory(), results, watcher, num_retries,
-           tag_results_with_device],
+          [r, tc, results, watcher, num_retries, tag_results_with_device],
           name=r.device[-4:])
-       for r in runners])
+       for r, tc in zip(runners, test_collections)])
   run_results = base_test_result.TestRunResults()
   workers.StartAll()
 
   # Catch DeviceUnresponsiveErrors and set a warning exit code
   try:
     workers.JoinAll(watcher)
   except android_commands.errors.DeviceUnresponsiveError as e:
     logging.error(e)
     exit_code = constants.WARNING_EXIT_CODE
 
+  assert all([len(tc) == 0 for tc in test_collections]), (
+      'Some tests were not run, all devices are likely offline (ran %d tests)' %
+      len(run_results.GetAll()))
+
   for r in results:
     run_results.AddTestRunResults(r)
   if not run_results.DidRunPass():
     exit_code = constants.ERROR_EXIT_CODE
   return (run_results, exit_code)
 
 
 def _CreateRunners(runner_factory, devices, timeout=None):
   """Creates a test runner for each device and calls SetUp() in parallel.
 
@@ skipping 80 matching lines @@
   logging.info('Will run %d tests (%s): %s', len(tests), log_string, str(tests))
   runners = _CreateRunners(runner_factory, devices, setup_timeout)
   try:
     return _RunAllTests(runners, test_collection_factory,
                         num_retries, test_timeout, tag_results_with_device)
   finally:
     try:
       _TearDownRunners(runners, setup_timeout)
     except android_commands.errors.DeviceUnresponsiveError as e:
       logging.warning('Device unresponsive during TearDown: [%s]', e)