Created separate shards for suspicious testcases.

build/android/pylib/local/device/local_device_test_run.py

Index: build/android/pylib/local/device/local_device_test_run.py
diff --git a/build/android/pylib/local/device/local_device_test_run.py b/build/android/pylib/local/device/local_device_test_run.py
index 4660b8e4ab368312fd313cf1f8d320ff9dd28fd2..303eece8f6a84672c4ac5d7d5e5d169b73f34914 100644
--- a/build/android/pylib/local/device/local_device_test_run.py
+++ b/build/android/pylib/local/device/local_device_test_run.py
@@ -76,6 +76,17 @@ def handle_shard_failures_with(on_failure):
 
   return decorator
 
+def find_suspects(unfinished_tests, shards):

[jbudorick, 2016/02/18 23:31:56]

This is gtest-specific and should be handled as such.

+  # The first unfinished test in each shard is a suspicious testcase that may
+  # have crashed and have caused all the following testcases in that shard not
+  # run.
+  suspects = []
+  for shard in shards:
+    for test in shard:
+      if (test in unfinished_tests):
+        suspects += test
+        break
+  return suspects
 
 class LocalDeviceTestRun(test_run.TestRun):
 
@@ -114,6 +125,7 @@ class LocalDeviceTestRun(test_run.TestRun):
     tries = 0
     results = base_test_result.TestRunResults()
     all_fail_results = {}
+    suspects = []
     while tries < self._env.max_tries and tests:
       logging.info('STARTING TRY #%d/%d', tries + 1, self._env.max_tries)
       logging.info('Will run %d tests on %d devices: %s',
@@ -124,7 +136,8 @@ class LocalDeviceTestRun(test_run.TestRun):
 
       try_results = base_test_result.TestRunResults()
       if self._ShouldShard():
-        tc = test_collection.TestCollection(self._CreateShards(tests))
+        shards = self._CreateShards(tests, suspects)

[jbudorick, 2016/02/18 23:31:56]

Shard creation is going to have to be result-aware, but this isn't exactly what
I had in mind. What if we actually used the existing results in shard creation &
better detected crashed tests?

(also, as is, this call will break instrumentation tests)

+        tc = test_collection.TestCollection(shards)
         self._env.parallel_devices.pMap(
             run_tests_on_device, tc, try_results).pGet(None)
       else:
@@ -146,8 +159,8 @@ class LocalDeviceTestRun(test_run.TestRun):
         if name.endswith('*'):
           return any(fnmatch.fnmatch(n, name) for n in results_names)
         return name in results_names
-
       tests = [t for t in tests if not has_test_result(self._GetTestName(t))]
+      suspects = find_suspects(tests, shards)
       tries += 1
       logging.info('FINISHED TRY #%d/%d', tries, self._env.max_tries)
       if tests: