[Android] Add --repeat for gtests and instrumentation tests.

build/android/test_runner.py

 #!/usr/bin/env python
 #
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Runs all types of tests from one unified interface."""
 
 import argparse
 import collections
+import itertools
 import logging
 import os
 import signal
 import sys
 import threading
 import unittest
 
 from devil import base_error
 from devil.android import apk_helper
 from devil.android import device_blacklist
@@ skipping 190 matching lines @@
                           'path')
   group.add_argument('--app-data-file', action='append', dest='app_data_files',
                      help='A file path relative to the app data directory '
                           'that should be saved to the host.')
   group.add_argument('--app-data-file-dir',
                      help='Host directory to which app data files will be'
                           ' saved. Used with --app-data-file.')
   group.add_argument('--delete-stale-data', dest='delete_stale_data',
                      action='store_true',
                      help='Delete stale test data on the device.')
+  group.add_argument('--repeat', '--gtest_repeat', '--gtest-repeat',
+                     dest='repeat', type=int, default=0,
+                     help='Number of times to repeat the specified set of '
+                          'tests.')
 
   filter_group = group.add_mutually_exclusive_group()
   filter_group.add_argument('-f', '--gtest_filter', '--gtest-filter',
                             dest='test_filter',
                             help='googletest-style filter string.')
   filter_group.add_argument('--gtest-filter-file', dest='test_filter_file',
                             help='Path to file that contains googletest-style '
                                   'filter strings. (Lines will be joined with '
                                   '":" to create a single filter string.)')
 
@@ skipping 10 matching lines @@
   AddDeviceOptions(parser)
 
 
 def AddJavaTestOptions(argument_group):
   """Adds the Java test options to |option_parser|."""
 
   argument_group.add_argument(
       '-f', '--test-filter', dest='test_filter',
       help=('Test filter (if not fully qualified, will run all matches).'))
   argument_group.add_argument(
+      '--repeat', dest='repeat', type=int, default=0,
+      help='Number of times to repeat the specified set of tests.')
+  argument_group.add_argument(
       '-A', '--annotation', dest='annotation_str',
       help=('Comma-separated list of annotations. Run only tests with any of '
             'the given annotations. An annotation can be either a key or a '
             'key-values pair. A test that has no annotation is considered '
             '"SmallTest".'))
   argument_group.add_argument(
       '-E', '--exclude-annotation', dest='exclude_annotation_str',
       help=('Comma-separated list of annotations. Exclude tests with these '
             'annotations.'))
   argument_group.add_argument(
@@ skipping 388 matching lines @@
   results, exit_code = test_dispatcher.RunTests(
       tests, runner_factory, devices, shard=True, test_timeout=60,
       num_retries=args.num_retries)
 
   report_results.LogFull(
       results=results,
       test_type='Linker test',
       test_package='ChromiumLinkerTest')
 
   if args.json_results_file:
-    json_results.GenerateJsonResultsFile(results, args.json_results_file)
+    json_results.GenerateJsonResultsFile([results], args.json_results_file)
 
   return exit_code
 
 
 def _RunInstrumentationTests(args, devices):
   """Subcommand of RunTestsCommands which runs instrumentation tests."""
   logging.info('_RunInstrumentationTests(%s, %s)', str(args), str(devices))
 
   instrumentation_options = ProcessInstrumentationOptions(args)
 
   if len(devices) > 1 and args.wait_for_debugger:
     logging.warning('Debugger can not be sharded, using first available device')
     devices = devices[:1]
 
   results = base_test_result.TestRunResults()
   exit_code = 0

[mikecase (-- gone --), 2015/09/28 14:00:12]

nit: This don't think this exit_code var is really used anymore in this
function. Its just set to 0 here and then used in the line

if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:

which I guess is just really

if test_exit_code and constants.ERROR_EXIT_CODE != 0:




Previously, it looks like the python_tests would set the test_exit_code variable
and the java_tests would set the exit_code variable. hmm...

[jbudorick, 2015/09/28 14:03:21]

It gets modified in the body of that if statement, then returned at the end of
this function.
the if statement should actually be after both the java tests and the python
tests

 
   if args.run_java_tests:
-    runner_factory, tests = instrumentation_setup.Setup(
+    java_runner_factory, java_tests = instrumentation_setup.Setup(
         instrumentation_options, devices)
-
-    test_results, exit_code = test_dispatcher.RunTests(
-        tests, runner_factory, devices, shard=True, test_timeout=None,
-        num_retries=args.num_retries)
-
-    results.AddTestRunResults(test_results)
+  else:
+    java_runner_factory = None
+    java_tests = None
 
   if args.run_python_tests:
-    runner_factory, tests = host_driven_setup.InstrumentationSetup(
+    py_runner_factory, py_tests = host_driven_setup.InstrumentationSetup(
         args.host_driven_root, args.official_build,
         instrumentation_options)
+  else:
+    py_runner_factory = None
+    py_tests = None
 
-    if tests:
+  results = []
+  repetitions = (xrange(args.repeat + 1) if args.repeat >= 0
+                 else itertools.count())
+  for _ in repetitions:
+    iteration_results = base_test_result.TestRunResults()
+    if java_tests:
       test_results, test_exit_code = test_dispatcher.RunTests(
-          tests, runner_factory, devices, shard=True, test_timeout=None,
-          num_retries=args.num_retries)
-
-      results.AddTestRunResults(test_results)
+          java_tests, java_runner_factory, devices, shard=True,
+          test_timeout=None, num_retries=args.num_retries)
+      iteration_results.AddTestRunResults(test_results)
 
       # Only allow exit code escalation
       if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
         exit_code = test_exit_code
 
-  if args.device_flags:
-    args.device_flags = os.path.join(constants.DIR_SOURCE_ROOT,
-                                     args.device_flags)
+    if py_tests:
+      test_results, test_exit_code = test_dispatcher.RunTests(
+          py_tests, py_runner_factory, devices, shard=True, test_timeout=None,
+          num_retries=args.num_retries)
+      iteration_results.AddTestRunResults(test_results)
 
-  report_results.LogFull(
-      results=results,
-      test_type='Instrumentation',
-      test_package=os.path.basename(args.test_apk),
-      annotation=args.annotations,
-      flakiness_server=args.flakiness_dashboard_server)
+    results.append(iteration_results)
+    report_results.LogFull(
+        results=iteration_results,
+        test_type='Instrumentation',
+        test_package=os.path.basename(args.test_apk),
+        annotation=args.annotations,
+        flakiness_server=args.flakiness_dashboard_server)
 
   if args.json_results_file:
     json_results.GenerateJsonResultsFile(results, args.json_results_file)
 
   return exit_code
 
 
 def _RunUIAutomatorTests(args, devices):
   """Subcommand of RunTestsCommands which runs uiautomator tests."""
   uiautomator_options = ProcessUIAutomatorOptions(args)
 
   runner_factory, tests = uiautomator_setup.Setup(uiautomator_options, devices)
 
   results, exit_code = test_dispatcher.RunTests(
       tests, runner_factory, devices, shard=True, test_timeout=None,
       num_retries=args.num_retries)
 
   report_results.LogFull(
       results=results,
       test_type='UIAutomator',
       test_package=os.path.basename(args.test_jar),
       annotation=args.annotations,
       flakiness_server=args.flakiness_dashboard_server)
 
   if args.json_results_file:
-    json_results.GenerateJsonResultsFile(results, args.json_results_file)
+    json_results.GenerateJsonResultsFile([results], args.json_results_file)
 
   return exit_code
 
 
 def _RunJUnitTests(args):
   """Subcommand of RunTestsCommand which runs junit tests."""
   runner_factory, tests = junit_setup.Setup(args)
   results, exit_code = junit_dispatcher.RunTests(tests, runner_factory)
 
   report_results.LogFull(
       results=results,
       test_type='JUnit',
       test_package=args.test_suite)
 
   if args.json_results_file:
-    json_results.GenerateJsonResultsFile(results, args.json_results_file)
+    json_results.GenerateJsonResultsFile([results], args.json_results_file)
 
   return exit_code
 
 
 def _RunMonkeyTests(args, devices):
   """Subcommand of RunTestsCommands which runs monkey tests."""
   monkey_options = ProcessMonkeyTestOptions(args)
 
   runner_factory, tests = monkey_setup.Setup(monkey_options)
 
   results, exit_code = test_dispatcher.RunTests(
       tests, runner_factory, devices, shard=False, test_timeout=None,
       num_retries=args.num_retries)
 
   report_results.LogFull(
       results=results,
       test_type='Monkey',
       test_package='Monkey')
 
   if args.json_results_file:
-    json_results.GenerateJsonResultsFile(results, args.json_results_file)
+    json_results.GenerateJsonResultsFile([results], args.json_results_file)
 
   return exit_code
 
 
 def _RunPerfTests(args, active_devices):
   """Subcommand of RunTestsCommands which runs perf tests."""
   perf_options = ProcessPerfTestOptions(args)
 
   # Just save a simple json with a list of test names.
   if perf_options.output_json_list:
@@ skipping 15 matching lines @@
   results, _ = test_dispatcher.RunTests(
       tests, runner_factory, devices, shard=False, test_timeout=None,
       num_retries=args.num_retries)
 
   report_results.LogFull(
       results=results,
       test_type='Perf',
       test_package='Perf')
 
   if args.json_results_file:
-    json_results.GenerateJsonResultsFile(results, args.json_results_file)
+    json_results.GenerateJsonResultsFile([results], args.json_results_file)
 
   if perf_options.single_step:
     return perf_test_runner.PrintTestOutput('single_step')
 
   perf_test_runner.PrintSummary(tests)
 
   # Always return 0 on the sharding stage. Individual tests exit_code
   # will be returned on the print_step stage.
   return 0
 
@@ skipping 106 matching lines @@
   def infra_error(message):
     parser.exit(status=constants.INFRA_EXIT_CODE, message=message)
 
   if args.command not in _SUPPORTED_IN_PLATFORM_MODE:
     infra_error('%s is not yet supported in platform mode' % args.command)
 
   with environment_factory.CreateEnvironment(args, infra_error) as env:
     with test_instance_factory.CreateTestInstance(args, infra_error) as test:
       with test_run_factory.CreateTestRun(
           args, env, test, infra_error) as test_run:
-        results = test_run.RunTests()
+        results = []
+        repetitions = (xrange(args.repeat + 1) if args.repeat >= 0
+                       else itertools.count())

[mikecase (-- gone --), 2015/09/28 13:37:58]

From what I see online, itertools.count() will just keep counting forever.

When would someone set args.repeat to be < 0?

[jbudorick, 2015/09/28 13:40:16]

For infinite repetition, of course:
https://code.google.com/p/googletest/wiki/AdvancedGuide#Repeating_the_Tests

[mikecase (-- gone --), 2015/09/28 14:00:12]

Ah, this makes sense then.

+        for _ in repetitions:
+          iteration_results = test_run.RunTests()
+          results.append(iteration_results)
 
-        if args.environment == 'remote_device' and args.trigger:
-          return 0 # Not returning results, only triggering.
-
-        report_results.LogFull(
-            results=results,
-            test_type=test.TestType(),
-            test_package=test_run.TestPackage(),
-            annotation=getattr(args, 'annotations', None),
-            flakiness_server=getattr(args, 'flakiness_dashboard_server', None))
+          if iteration_results is not None:
+            report_results.LogFull(
+                results=iteration_results,
+                test_type=test.TestType(),
+                test_package=test_run.TestPackage(),
+                annotation=getattr(args, 'annotations', None),
+                flakiness_server=getattr(args, 'flakiness_dashboard_server',
+                                         None))
 
         if args.json_results_file:
           json_results.GenerateJsonResultsFile(
               results, args.json_results_file)
 
-  return 0 if results.DidRunPass() else constants.ERROR_EXIT_CODE
+  return (0 if all(r.DidRunPass() for r in results)
+          else constants.ERROR_EXIT_CODE)
 
 
 CommandConfigTuple = collections.namedtuple(
     'CommandConfigTuple',
     ['add_options_func', 'help_txt'])
 VALID_COMMANDS = {
     'gtest': CommandConfigTuple(
         AddGTestOptions,
         'googletest-based C++ tests'),
     'instrumentation': CommandConfigTuple(
@@ skipping 50 matching lines @@
     if e.is_infra_error:
       return constants.INFRA_EXIT_CODE
     return constants.ERROR_EXIT_CODE
   except: # pylint: disable=W0702
     logging.exception('Unrecognized error occurred.')
     return constants.ERROR_EXIT_CODE
 
 
 if __name__ == '__main__':
   sys.exit(main())