[Android] Expose each try result in test results JSON.

build/android/test_runner.py

 #!/usr/bin/env python
 #
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Runs all types of tests from one unified interface."""
 
 import argparse
 import collections
@@ skipping 10 matching lines @@
 from devil import devil_env
 from devil.android import device_blacklist
 from devil.android import device_errors
 from devil.android import device_utils
 from devil.android import forwarder
 from devil.android import ports
 from devil.utils import reraiser_thread
 from devil.utils import run_tests_helper
 
 from pylib import constants
-from pylib.constants import host_paths
 from pylib.base import base_test_result
 from pylib.base import environment_factory
 from pylib.base import test_dispatcher
 from pylib.base import test_instance_factory
 from pylib.base import test_run_factory
+from pylib.constants import host_paths
 from pylib.linker import setup as linker_setup
 from pylib.junit import setup as junit_setup
 from pylib.junit import test_dispatcher as junit_dispatcher
 from pylib.monkey import setup as monkey_setup
 from pylib.monkey import test_options as monkey_test_options
 from pylib.perf import setup as perf_setup
 from pylib.perf import test_options as perf_test_options
 from pylib.perf import test_runner as perf_test_runner
 from pylib.results import json_results
 from pylib.results import report_results
@@ skipping 798 matching lines @@
     logging.fatal(message)
     sys.exit(constants.INFRA_EXIT_CODE)
 
   if args.command not in _SUPPORTED_IN_PLATFORM_MODE:
     infra_error('%s is not yet supported in platform mode' % args.command)
 
   with environment_factory.CreateEnvironment(args, infra_error) as env:
     with test_instance_factory.CreateTestInstance(args, infra_error) as test:
       with test_run_factory.CreateTestRun(
           args, env, test, infra_error) as test_run:
-        results = []
+
+        # TODO(jbudorick): Rewrite results handling.
+
+        # all_raw_results is a list of lists of base_test_result.TestRunResults
+        # objects. Each instance of TestRunResults contains all test results
+        # produced by a single try, while each list of TestRunResults contains
+        # all tries in a single iteration.
+        all_raw_results = []
+        # all_iteration_results is a list of base_test_result.TestRunResults
+        # objects. Each instance of TestRunResults contains the last test result
+        # for each test run in that iteration.
+        all_iteration_results = []
+
         repetitions = (xrange(args.repeat + 1) if args.repeat >= 0
                        else itertools.count())
         result_counts = collections.defaultdict(
             lambda: collections.defaultdict(int))
         iteration_count = 0
         for _ in repetitions:
-          iteration_results = test_run.RunTests()
-          if iteration_results is not None:
-            iteration_count += 1
-            results.append(iteration_results)
-            for r in iteration_results.GetAll():
-              result_counts[r.GetName()][r.GetType()] += 1
-            report_results.LogFull(
-                results=iteration_results,
-                test_type=test.TestType(),
-                test_package=test_run.TestPackage(),
-                annotation=getattr(args, 'annotations', None),
-                flakiness_server=getattr(args, 'flakiness_dashboard_server',
-                                         None))
-            if args.break_on_failure and not iteration_results.DidRunPass():
-              break
+          raw_results = test_run.RunTests()
+          if not raw_results:
+            continue
+
+          all_raw_results.append(raw_results)
+
+          iteration_results = base_test_result.TestRunResults()
+          for r in reversed(raw_results):
+            iteration_results.AddTestRunResults(r)
+          all_iteration_results.append(iteration_results)
+
+          iteration_count += 1
+          for r in iteration_results.GetAll():
+            result_counts[r.GetName()][r.GetType()] += 1
+          report_results.LogFull(
+              results=iteration_results,
+              test_type=test.TestType(),
+              test_package=test_run.TestPackage(),
+              annotation=getattr(args, 'annotations', None),
+              flakiness_server=getattr(args, 'flakiness_dashboard_server',
+                                       None))
+          if args.break_on_failure and not iteration_results.DidRunPass():
+            break
 
         if iteration_count > 1:
           # display summary results
           # only display results for a test if at least one test did not pass
           all_pass = 0
           tot_tests = 0
           for test_name in result_counts:
             tot_tests += 1
             if any(result_counts[test_name][x] for x in (
                 base_test_result.ResultType.FAIL,
                 base_test_result.ResultType.CRASH,
                 base_test_result.ResultType.TIMEOUT,
                 base_test_result.ResultType.UNKNOWN)):
               logging.critical(
                   '%s: %s',
                   test_name,
                   ', '.join('%s %s' % (str(result_counts[test_name][i]), i)
                             for i in base_test_result.ResultType.GetTypes()))
             else:
               all_pass += 1
 
           logging.critical('%s of %s tests passed in all %s runs',
                            str(all_pass),
                            str(tot_tests),
                            str(iteration_count))
 
         if args.json_results_file:
           json_results.GenerateJsonResultsFile(
-              results, args.json_results_file)
+              all_raw_results, args.json_results_file)
 
-  return (0 if all(r.DidRunPass() for r in results)
+  return (0 if all(r.DidRunPass() for r in all_iteration_results)
           else constants.ERROR_EXIT_CODE)
 
 
 CommandConfigTuple = collections.namedtuple(
     'CommandConfigTuple',
     ['add_options_func', 'help_txt'])
 VALID_COMMANDS = {
     'gtest': CommandConfigTuple(
         AddGTestOptions,
         'googletest-based C++ tests'),
@@ skipping 48 matching lines @@
     if e.is_infra_error:
       return constants.INFRA_EXIT_CODE
     return constants.ERROR_EXIT_CODE
   except: # pylint: disable=W0702
     logging.exception('Unrecognized error occurred.')
     return constants.ERROR_EXIT_CODE
 
 
 if __name__ == '__main__':
   sys.exit(main())