Created separate shards for suspicious testcases.

build/android/pylib/local/device/local_device_test_run.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import fnmatch
 import functools
 import imp
 import logging
 
 from devil import base_error
@@ skipping 58 matching lines @@
         logging.exception('Shard failed: %s(%s)', f.__name__,
                           str(dev))
       if on_failure:
         on_failure(dev, f.__name__)
       return None
 
     return wrapper
 
   return decorator
 
+def find_suspects(unfinished_tests, shards):

[jbudorick, 2016/02/18 23:31:56]

This is gtest-specific and should be handled as such.

+  # The first unfinished test in each shard is a suspicious testcase that may
+  # have crashed and have caused all the following testcases in that shard not
+  # run.
+  suspects = []
+  for shard in shards:
+    for test in shard:
+      if (test in unfinished_tests):
+        suspects += test
+        break
+  return suspects
 
 class LocalDeviceTestRun(test_run.TestRun):
 
   def __init__(self, env, test_instance):
     super(LocalDeviceTestRun, self).__init__(env, test_instance)
     self._tools = {}
 
   #override
   def RunTests(self):
     tests = self._GetTests()
@@ skipping 18 matching lines @@
         finally:
           if isinstance(tests, test_collection.TestCollection):
             tests.test_completed()
 
 
       logging.info('Finished running tests on this device.')
 
     tries = 0
     results = base_test_result.TestRunResults()
     all_fail_results = {}
+    suspects = []
     while tries < self._env.max_tries and tests:
       logging.info('STARTING TRY #%d/%d', tries + 1, self._env.max_tries)
       logging.info('Will run %d tests on %d devices: %s',
                    len(tests), len(self._env.devices),
                    ', '.join(str(d) for d in self._env.devices))
       for t in tests:
         logging.debug('  %s', t)
 
       try_results = base_test_result.TestRunResults()
       if self._ShouldShard():
-        tc = test_collection.TestCollection(self._CreateShards(tests))
+        shards = self._CreateShards(tests, suspects)

[jbudorick, 2016/02/18 23:31:56]

Shard creation is going to have to be result-aware, but this isn't exactly what
I had in mind. What if we actually used the existing results in shard creation &
better detected crashed tests?

(also, as is, this call will break instrumentation tests)

+        tc = test_collection.TestCollection(shards)
         self._env.parallel_devices.pMap(
             run_tests_on_device, tc, try_results).pGet(None)
       else:
         self._env.parallel_devices.pMap(
             run_tests_on_device, tests, try_results).pGet(None)
 
       for result in try_results.GetAll():
         if result.GetType() in (base_test_result.ResultType.PASS,
                                 base_test_result.ResultType.SKIP):
           results.AddResult(result)
         else:
           all_fail_results[result.GetName()] = result
 
       results_names = set(r.GetName() for r in results.GetAll())
 
       def has_test_result(name):
         # When specifying a test filter, names can contain trailing wildcards.
         # See local_device_gtest_run._ExtractTestsFromFilter()
         if name.endswith('*'):
           return any(fnmatch.fnmatch(n, name) for n in results_names)
         return name in results_names
-
       tests = [t for t in tests if not has_test_result(self._GetTestName(t))]
+      suspects = find_suspects(tests, shards)
       tries += 1
       logging.info('FINISHED TRY #%d/%d', tries, self._env.max_tries)
       if tests:
         logging.info('%d failed tests remain.', len(tests))
       else:
         logging.info('All tests completed.')
 
     all_unknown_test_names = set(self._GetTestName(t) for t in tests)
     all_failed_test_names = set(all_fail_results.iterkeys())
 
@@ skipping 24 matching lines @@
     return test
 
   def _GetTests(self):
     raise NotImplementedError
 
   def _RunTest(self, device, test):
     raise NotImplementedError
 
   def _ShouldShard(self):
     raise NotImplementedError