ABANDONED [Android] Expose each try result in test results JSON.

build/android/pylib/local/device/local_device_test_run.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import fnmatch
 import functools
 import imp
 import logging
 
 from devil import base_error
@@ skipping 95 matching lines @@
             tests.add(test)
           raise
         finally:
           if isinstance(tests, test_collection.TestCollection):
             tests.test_completed()
 
 
       logging.info('Finished running tests on this device.')
 
     tries = 0
-    results = base_test_result.TestRunResults()
-    all_fail_results = {}
+    results = []
     while tries < self._env.max_tries and tests:
       logging.info('STARTING TRY #%d/%d', tries + 1, self._env.max_tries)
       logging.info('Will run %d tests on %d devices: %s',
                    len(tests), len(self._env.devices),
                    ', '.join(str(d) for d in self._env.devices))
       for t in tests:
         logging.debug('  %s', t)
 
       try_results = base_test_result.TestRunResults()
       if self._ShouldShard():
         tc = test_collection.TestCollection(self._CreateShards(tests))
         self._env.parallel_devices.pMap(
             run_tests_on_device, tc, try_results).pGet(None)
       else:
         self._env.parallel_devices.pMap(
             run_tests_on_device, tests, try_results).pGet(None)
 
-      for result in try_results.GetAll():
-        if result.GetType() in (base_test_result.ResultType.PASS,
-                                base_test_result.ResultType.SKIP):
-          results.AddResult(result)
-        else:
-          all_fail_results[result.GetName()] = result
+      results.append(try_results)

[mikecase (-- gone --), 2016/05/13 17:32:13]

Should this be results.extend(try_results)? Im kinda pretty confused how all
these results are structured actually. I think it would be clearer if we kept
all the results for each single test in a separate list like....

results['testname'].append(try_results['testname'])

[jbudorick, 2016/05/13 17:37:05]

No, try_results is a single instance of base_test_result.TestRunResults
Something like that would be neat as part of a larger rewrite of how we handle
results (and, probably, pending tests). I don't have the time to do that at the
moment.

+      tries += 1
+      tests = self._GetTestsToRetry(tests, try_results)
 
-      results_names = set(r.GetName() for r in results.GetAll())
-
-      def has_test_result(name):
-        # When specifying a test filter, names can contain trailing wildcards.
-        # See local_device_gtest_run._ExtractTestsFromFilter()
-        if name.endswith('*'):
-          return any(fnmatch.fnmatch(n, name) for n in results_names)
-        return name in results_names
-
-      tests = [t for t in tests if not has_test_result(self._GetTestName(t))]
-      tries += 1
       logging.info('FINISHED TRY #%d/%d', tries, self._env.max_tries)
       if tests:
         logging.info('%d failed tests remain.', len(tests))
       else:
         logging.info('All tests completed.')
 
-    all_unknown_test_names = set(self._GetTestName(t) for t in tests)
-    all_failed_test_names = set(all_fail_results.iterkeys())
+    return results
 
-    unknown_tests = all_unknown_test_names.difference(all_failed_test_names)
-    failed_tests = all_failed_test_names.intersection(all_unknown_test_names)
+  def _GetTestsToRetry(self, tests, try_results):
+    failed_test_names = set(
+        r.GetName() for r in try_results.GetAll()
+        if r.GetType() not in (base_test_result.ResultType.PASS,
+                               base_test_result.ResultType.SKIP))
 
-    if unknown_tests:
-      results.AddResults(
-          base_test_result.BaseTestResult(
-              u, base_test_result.ResultType.UNKNOWN)
-          for u in unknown_tests)
-    if failed_tests:
-      results.AddResults(all_fail_results[f] for f in failed_tests)
+    all_test_names = set(r.GetName() for r in try_results.GetAll())
 
-    return results
+    def has_test_result(name):
+      # When specifying a test filter, names can contain trailing wildcards.
+      # See local_device_gtest_run._ExtractTestsFromFilter()
+      if name.endswith('*'):
+        return any(fnmatch.fnmatch(n, name) for n in all_test_names)
+      return name in all_test_names
+
+    unknown_test_names = set(
+        t for t in tests if not has_test_result(self._GetTestName(t)))
+
+    return failed_test_names.union(unknown_test_names)
 
   def GetTool(self, device):
     if not str(device) in self._tools:
       self._tools[str(device)] = valgrind_tools.CreateTool(
           self._env.tool, device)
     return self._tools[str(device)]
 
   def _CreateShards(self, tests):
     raise NotImplementedError
 
   # pylint: disable=no-self-use
   def _GetTestName(self, test):
     return test
 
   def _GetTests(self):
     raise NotImplementedError
 
   def _RunTest(self, device, test):
     raise NotImplementedError
 
   def _ShouldShard(self):
     raise NotImplementedError