ABANDONED [Android] Expose each try result in test results JSON.

build/android/test_runner.py

 #!/usr/bin/env python
 #
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Runs all types of tests from one unified interface."""
 
 import argparse
 import collections
@@ skipping 838 matching lines @@
     with test_instance_factory.CreateTestInstance(args, infra_error) as test:
       with test_run_factory.CreateTestRun(
           args, env, test, infra_error) as test_run:
         results = []
         repetitions = (xrange(args.repeat + 1) if args.repeat >= 0
                        else itertools.count())
         result_counts = collections.defaultdict(
             lambda: collections.defaultdict(int))
         iteration_count = 0
         for _ in repetitions:
-          iteration_results = test_run.RunTests()
-          if iteration_results is not None:
-            iteration_count += 1
-            results.append(iteration_results)
-            for r in iteration_results.GetAll():
-              result_counts[r.GetName()][r.GetType()] += 1
-            report_results.LogFull(
-                results=iteration_results,
-                test_type=test.TestType(),
-                test_package=test_run.TestPackage(),
-                annotation=getattr(args, 'annotations', None),
-                flakiness_server=getattr(args, 'flakiness_dashboard_server',
-                                         None))
-            if args.break_on_failure and not iteration_results.DidRunPass():
-              break
+          raw_results = test_run.RunTests()
+          if not raw_results:
+            continue
+
+          results.extend(raw_results)

[mikecase (-- gone --), 2016/05/13 17:45:34]

My only question I guess would be will the GenerateJson function put each
repetition in a separate 'iteration_results' dict (I assume we want that)?

Since you are just extending the results list here for all the repetitions my
suspicion would be that it won't do that. 

https://code.google.com/p/chromium/codesearch#chromium/src/build/android/pyli...

[jbudorick, 2016/05/13 17:59:10]

It should:
https://code.google.com/p/chromium/codesearch#chromium/src/build/android/pyli...

+
+          iteration_results = base_test_result.TestRunResults()
+          for r in raw_results:
+            iteration_results.AddTestRunResults(r)

[mikecase (-- gone --), 2016/05/13 17:32:13]

I find this pretty confusing. So, just to make sure I am getting it right....

raw_results is a list of list of results gotten from retries. So like...
raw_results = [['Test1 Try1', 'Test2 Try1'], ['Test2 Try 2']]

Then results is a just a super extended list of these lists like....

results = [['Test1 Try1', 'Test2 Try1'], ['Test2 Try 2'], //repition 1
           ['Test1 Try1', 'Test2 Try1'], ['Test1 Try2', 'Test2 Try 2']]
//repition 2

If this is the case, maybe add some more comments. And maybe some names like...
repetition_results instead of results or something.


Also, would it make more sense to store it more like how the JSON results is
stored

{
  'repitition1': {
    'test1': ['try1', 'try2'],
    'test2': ['try1']
  }
  'repitition2': {
    'test1': ['try1', 'try2'],
    'test2': ['try1']
  }
}

[jbudorick, 2016/05/13 17:37:05]

No, raw_results is a list of base_test_result.TestRunResults objects.
wrong for the same reason as above

+
+          iteration_count += 1
+          for r in iteration_results.GetAll():
+            result_counts[r.GetName()][r.GetType()] += 1
+          report_results.LogFull(
+              results=iteration_results,
+              test_type=test.TestType(),
+              test_package=test_run.TestPackage(),
+              annotation=getattr(args, 'annotations', None),
+              flakiness_server=getattr(args, 'flakiness_dashboard_server',
+                                       None))
+          if args.break_on_failure and not iteration_results.DidRunPass():
+            break
 
         if iteration_count > 1:
           # display summary results
           # only display results for a test if at least one test did not pass
           all_pass = 0
           tot_tests = 0
           for test_name in result_counts:
             tot_tests += 1
             if any(result_counts[test_name][x] for x in (
                 base_test_result.ResultType.FAIL,
@@ skipping 79 matching lines @@
     if e.is_infra_error:
       return constants.INFRA_EXIT_CODE
     return constants.ERROR_EXIT_CODE
   except: # pylint: disable=W0702
     logging.exception('Unrecognized error occurred.')
     return constants.ERROR_EXIT_CODE
 
 
 if __name__ == '__main__':
   sys.exit(main())