[Android] Reland 'Keep track of unknown test results at the TestRunner layer.'

build/android/pylib/base/base_test_sharder.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 
 import logging
 import multiprocessing
 
 from pylib import android_commands
 from pylib.base.test_result import TestResults
@@ skipping 29 matching lines @@
   Args:
     attached_devices: A list of attached devices.
   """
   # See more in SetTestsContainer.
   tests_container = None
 
   def __init__(self, attached_devices, build_type='Debug'):
     self.attached_devices = attached_devices
     # Worst case scenario: a device will drop offline per run, so we need
     # to retry until we're out of devices.
+
+    # TODO(frankf): There are two sources of flakiness:
+    # 1. Device flakiness
+    # 2. Test/product flakiness
+    # We should differentiate between these. Otherwise, blindly retrying tests
+    # might mask test/product flakiness. For type 2, we should follow the
+    # general chrome best practices.
     self.retries = len(self.attached_devices)
     self.tests = []
     self.build_type = build_type
 
   def CreateShardedTestRunner(self, device, index):
     """Factory function to create a suite-specific test runner.
 
     Args:
       device: Device serial where this shard will run
       index: Index of this device in the pool.
@@ skipping 23 matching lines @@
     logging.warning('*' * 80)
     logging.warning('Sharding in ' + str(len(self.attached_devices)) +
                     ' devices.')
     logging.warning('Note that the output is not synchronized.')
     logging.warning('Look for the "Final result" banner in the end.')
     logging.warning('*' * 80)
     final_results = TestResults()
     self._KillHostForwarder()
     for retry in xrange(self.retries):
       logging.warning('Try %d of %d', retry + 1, self.retries)
+      logging.warning('Attempting to run %d tests: %s'
+                      % (len(self.tests), self.tests))
       self.SetupSharding(self.tests)
       test_runners = []
 
       # Try to create N shards, and retrying on failure.
       try:
         for index, device in enumerate(self.attached_devices):
           logging.warning('*' * 80)
           logging.warning('Creating shard %d for %s', index, device)
           logging.warning('*' * 80)
           test_runner = self.CreateShardedTestRunner(device, index)
@@ skipping 18 matching lines @@
         continue
       test_results = TestResults.FromTestResults(results_lists)
       # Re-check the attached devices for some devices may
       # become offline
       retry_devices = set(android_commands.GetAttachedDevices())
       # Remove devices that had exceptions.
       retry_devices -= TestResults.DeviceExceptions(results_lists)
       # Retry on devices that didn't have any exception.
       self.attached_devices = list(retry_devices)
 
-      # TODO(frankf): Fix the retry logic:
-      # - GetAllBroken() should report all tests not ran.
-      # - Do not break TestResults encapsulation.
+      # TODO(frankf): Do not break TestResults encapsulation.
       if (retry == self.retries - 1 or
           len(self.attached_devices) == 0):
         all_passed = final_results.ok + test_results.ok
         final_results = test_results
         final_results.ok = all_passed
         break
       else:
         final_results.ok += test_results.ok
-        final_results.overall_timed_out = (final_results.overall_timed_out or
-                                           test_results.overall_timed_out)
-        final_results.overall_fail = (final_results.overall_fail or
-                                      test_results.overall_fail)
 
         self.tests = []
         for t in test_results.GetAllBroken():
           self.tests += [t.name]
         if not self.tests:
           break
     else:
       # We ran out retries, possibly out of healthy devices.
       # There's no recovery at this point.
       raise Exception('Unrecoverable error while retrying test runs.')
     self.OnTestsCompleted(test_runners, final_results)
     self._KillHostForwarder()
     return final_results