Android: improves test sharding reliability.

build/android/pylib/base_test_sharder.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 
 import android_commands
 import logging
 import multiprocessing
 
+from android_commands import errors
 from test_result import TestResults
 
 
 def _ShardedTestRunnable(test):
   """Standalone function needed by multiprocessing.Pool."""
   log_format = '[' + test.device + '] # %(asctime)-15s: %(message)s'
   if logging.getLogger().handlers:
     logging.getLogger().handlers[0].setFormatter(logging.Formatter(log_format))
   else:
     logging.basicConfig(format=log_format)
@@ skipping 16 matching lines @@
   """Base class for sharding tests across multiple devices.
 
   Args:
     attached_devices: A list of attached devices.
   """
   # See more in SetTestsContainer.
   tests_container = None
 
   def __init__(self, attached_devices):
     self.attached_devices = attached_devices
-    self.retries = 1
+    # Worst case scenario: a device will drop offline per run, so we need
+    # to retry until we're out of devices.
+    self.retries = len(self.attached_devices)
     self.tests = []
 
   def CreateShardedTestRunner(self, device, index):
     """Factory function to create a suite-specific test runner.
 
     Args:
       device: Device serial where this shard will run
       index: Index of this device in the pool.
 
     Returns:
@@ skipping 19 matching lines @@
     logging.warning('Sharding in ' + str(len(self.attached_devices)) +
                     ' devices.')
     logging.warning('Note that the output is not synchronized.')
     logging.warning('Look for the "Final result" banner in the end.')
     logging.warning('*' * 80)
     final_results = TestResults()
     for retry in xrange(self.retries):
       logging.warning('Try %d of %d', retry + 1, self.retries)
       self.SetupSharding(self.tests)
       test_runners = []
-      for index, device in enumerate(self.attached_devices):
-        logging.warning('*' * 80)
-        logging.warning('Creating shard %d for %s', index, device)
-        logging.warning('*' * 80)
-        test_runner = self.CreateShardedTestRunner(device, index)
-        test_runners += [test_runner]
+
+      # Try to create N shards, and retrying on failure.
+      try:
+        for index, device in enumerate(self.attached_devices):
+          logging.warning('*' * 80)
+          logging.warning('Creating shard %d for %s', index, device)
+          logging.warning('*' * 80)
+          test_runner = self.CreateShardedTestRunner(device, index)
+          test_runners += [test_runner]
+      except errors.DeviceUnresponsiveError as e:
+        logging.critical('****Failed to create a shard: [%s]', e)
+        self.attached_devices.remove(device)
+        continue
+
       logging.warning('Starting...')
       pool = multiprocessing.Pool(len(self.attached_devices),
                                   SetTestsContainer,
                                   [BaseTestSharder.tests_container])
       # map can't handle KeyboardInterrupt exception. It's a python bug.
       # So use map_async instead.
       async_results = pool.map_async(_ShardedTestRunnable, test_runners)
-      results_lists = async_results.get(999999)
-
+      try:
+        results_lists = async_results.get(999999)
+      except errors.DeviceUnresponsiveError as e:

[Isaac (away), 2012/10/31 19:18:14]

Is it possible to get other exceptions besides DeviceUnresponsive?

[bulach, 2012/10/31 19:27:31]

quite possibly. :) but trying to recover from them by just retrying on other
devices would probably be wrong..
let's see how this behaves and we can fine tune from there.

+        logging.critical('****Failed to run test: [%s]', e)
+        self.attached_devices = android_commands.GetAttachedDevices()
+        continue
       test_results = TestResults.FromTestResults(results_lists)
       # Re-check the attached devices for some devices may
       # become offline
       retry_devices = set(android_commands.GetAttachedDevices())
       # Remove devices that had exceptions.
       retry_devices -= TestResults.DeviceExceptions(results_lists)
       # Retry on devices that didn't have any exception.
       self.attached_devices = list(retry_devices)
       if (retry == self.retries - 1 or
           len(self.attached_devices) == 0):
         all_passed = final_results.ok + test_results.ok
         final_results = test_results
         final_results.ok = all_passed
         break
       else:
         final_results.ok += test_results.ok
         self.tests = []
         for t in test_results.GetAllBroken():
           self.tests += [t.name]
         if not self.tests:
           break
     self.OnTestsCompleted(test_runners, final_results)
     return final_results