[Android] Add --repeat for gtests and instrumentation tests.

build/android/test_runner.py

 #!/usr/bin/env python
 #
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Runs all types of tests from one unified interface."""
 
 import argparse
 import collections
+import itertools
 import logging
 import os
 import signal
 import sys
 import threading
 import unittest
 
 from devil import base_error
 from devil.android import apk_helper
 from devil.android import device_blacklist
@@ skipping 190 matching lines @@
                           'path')
   group.add_argument('--app-data-file', action='append', dest='app_data_files',
                      help='A file path relative to the app data directory '
                           'that should be saved to the host.')
   group.add_argument('--app-data-file-dir',
                      help='Host directory to which app data files will be'
                           ' saved. Used with --app-data-file.')
   group.add_argument('--delete-stale-data', dest='delete_stale_data',
                      action='store_true',
                      help='Delete stale test data on the device.')
+  group.add_argument('--repeat', '--gtest_repeat', '--gtest-repeat',
+                     dest='repeat', type=int, default=0,
+                     help='Number of times to repeat the specified set of '
+                          'tests.')
 
   filter_group = group.add_mutually_exclusive_group()
   filter_group.add_argument('-f', '--gtest_filter', '--gtest-filter',
                             dest='test_filter',
                             help='googletest-style filter string.')
   filter_group.add_argument('--gtest-filter-file', dest='test_filter_file',
                             help='Path to file that contains googletest-style '
                                   'filter strings. (Lines will be joined with '
                                   '":" to create a single filter string.)')
 
@@ skipping 10 matching lines @@
   AddDeviceOptions(parser)
 
 
 def AddJavaTestOptions(argument_group):
   """Adds the Java test options to |option_parser|."""
 
   argument_group.add_argument(
       '-f', '--test-filter', dest='test_filter',
       help=('Test filter (if not fully qualified, will run all matches).'))
   argument_group.add_argument(
+      '--repeat', dest='repeat', type=int, default=0,
+      help='Number of times to repeat the specified set of tests.')
+  argument_group.add_argument(
       '-A', '--annotation', dest='annotation_str',
       help=('Comma-separated list of annotations. Run only tests with any of '
             'the given annotations. An annotation can be either a key or a '
             'key-values pair. A test that has no annotation is considered '
             '"SmallTest".'))
   argument_group.add_argument(
       '-E', '--exclude-annotation', dest='exclude_annotation_str',
       help=('Comma-separated list of annotations. Exclude tests with these '
             'annotations.'))
   argument_group.add_argument(
@@ skipping 388 matching lines @@
   results, exit_code = test_dispatcher.RunTests(
       tests, runner_factory, devices, shard=True, test_timeout=60,
       num_retries=args.num_retries)
 
   report_results.LogFull(
       results=results,
       test_type='Linker test',
       test_package='ChromiumLinkerTest')
 
   if args.json_results_file:
-    json_results.GenerateJsonResultsFile(results, args.json_results_file)
+    json_results.GenerateJsonResultsFile([results], args.json_results_file)
 
   return exit_code
 
 
 def _RunInstrumentationTests(args, devices):
   """Subcommand of RunTestsCommands which runs instrumentation tests."""
   logging.info('_RunInstrumentationTests(%s, %s)', str(args), str(devices))
 
   instrumentation_options = ProcessInstrumentationOptions(args)
 
   if len(devices) > 1 and args.wait_for_debugger:
     logging.warning('Debugger can not be sharded, using first available device')
     devices = devices[:1]
 
   results = base_test_result.TestRunResults()
   exit_code = 0
 
   if args.run_java_tests:
-    runner_factory, tests = instrumentation_setup.Setup(
+    java_runner_factory, java_tests = instrumentation_setup.Setup(
         instrumentation_options, devices)
-
-    test_results, exit_code = test_dispatcher.RunTests(
-        tests, runner_factory, devices, shard=True, test_timeout=None,
-        num_retries=args.num_retries)
-
-    results.AddTestRunResults(test_results)
+  else:
+    java_runner_factory = None
+    java_tests = None
 
   if args.run_python_tests:
-    runner_factory, tests = host_driven_setup.InstrumentationSetup(
+    py_runner_factory, py_tests = host_driven_setup.InstrumentationSetup(
         args.host_driven_root, args.official_build,
         instrumentation_options)
+  else:
+    py_runner_factory = None
+    py_tests = None
 
-    if tests:
+  results = []
+  repetitions = (xrange(args.repeat + 1) if args.repeat >= 0
+                 else itertools.count())
+  for _ in repetitions:
+    iteration_results = base_test_result.TestRunResults()
+    if java_tests:
       test_results, test_exit_code = test_dispatcher.RunTests(
-          tests, runner_factory, devices, shard=True, test_timeout=None,
-          num_retries=args.num_retries)
-
-      results.AddTestRunResults(test_results)
+          java_tests, java_runner_factory, devices, shard=True,
+          test_timeout=None, num_retries=args.num_retries)
+      iteration_results.AddTestRunResults(test_results)
 
       # Only allow exit code escalation
       if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
         exit_code = test_exit_code
 
-  if args.device_flags:
-    args.device_flags = os.path.join(constants.DIR_SOURCE_ROOT,
-                                     args.device_flags)
+    if py_tests:
+      test_results, test_exit_code = test_dispatcher.RunTests(
+          py_tests, py_runner_factory, devices, shard=True, test_timeout=None,
+          num_retries=args.num_retries)
+      iteration_results.AddTestRunResults(test_results)
 
-  report_results.LogFull(
-      results=results,
-      test_type='Instrumentation',
-      test_package=os.path.basename(args.test_apk),
-      annotation=args.annotations,
-      flakiness_server=args.flakiness_dashboard_server)
+      # Only allow exit code escalation
+      if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
+        exit_code = test_exit_code
+
+    results.append(iteration_results)
+    report_results.LogFull(
+        results=iteration_results,
+        test_type='Instrumentation',
+        test_package=os.path.basename(args.test_apk),
+        annotation=args.annotations,
+        flakiness_server=args.flakiness_dashboard_server)
 
   if args.json_results_file:
     json_results.GenerateJsonResultsFile(results, args.json_results_file)
 
   return exit_code
 
 
 def _RunUIAutomatorTests(args, devices):
   """Subcommand of RunTestsCommands which runs uiautomator tests."""
   uiautomator_options = ProcessUIAutomatorOptions(args)
 
   runner_factory, tests = uiautomator_setup.Setup(uiautomator_options, devices)
 
   results, exit_code = test_dispatcher.RunTests(
       tests, runner_factory, devices, shard=True, test_timeout=None,
       num_retries=args.num_retries)
 
   report_results.LogFull(
       results=results,
       test_type='UIAutomator',
       test_package=os.path.basename(args.test_jar),
       annotation=args.annotations,
       flakiness_server=args.flakiness_dashboard_server)
 
   if args.json_results_file:
-    json_results.GenerateJsonResultsFile(results, args.json_results_file)
+    json_results.GenerateJsonResultsFile([results], args.json_results_file)
 
   return exit_code
 
 
 def _RunJUnitTests(args):
   """Subcommand of RunTestsCommand which runs junit tests."""
   runner_factory, tests = junit_setup.Setup(args)
   results, exit_code = junit_dispatcher.RunTests(tests, runner_factory)
 
   report_results.LogFull(
       results=results,
       test_type='JUnit',
       test_package=args.test_suite)
 
   if args.json_results_file:
-    json_results.GenerateJsonResultsFile(results, args.json_results_file)
+    json_results.GenerateJsonResultsFile([results], args.json_results_file)
 
   return exit_code
 
 
 def _RunMonkeyTests(args, devices):
   """Subcommand of RunTestsCommands which runs monkey tests."""
   monkey_options = ProcessMonkeyTestOptions(args)
 
   runner_factory, tests = monkey_setup.Setup(monkey_options)
 
   results, exit_code = test_dispatcher.RunTests(
       tests, runner_factory, devices, shard=False, test_timeout=None,
       num_retries=args.num_retries)
 
   report_results.LogFull(
       results=results,
       test_type='Monkey',
       test_package='Monkey')
 
   if args.json_results_file:
-    json_results.GenerateJsonResultsFile(results, args.json_results_file)
+    json_results.GenerateJsonResultsFile([results], args.json_results_file)
 
   return exit_code
 
 
 def _RunPerfTests(args, active_devices):
   """Subcommand of RunTestsCommands which runs perf tests."""
   perf_options = ProcessPerfTestOptions(args)
 
   # Just save a simple json with a list of test names.
   if perf_options.output_json_list:
@@ skipping 15 matching lines @@
   results, _ = test_dispatcher.RunTests(
       tests, runner_factory, devices, shard=False, test_timeout=None,
       num_retries=args.num_retries)
 
   report_results.LogFull(
       results=results,
       test_type='Perf',
       test_package='Perf')
 
   if args.json_results_file:
-    json_results.GenerateJsonResultsFile(results, args.json_results_file)
+    json_results.GenerateJsonResultsFile([results], args.json_results_file)
 
   if perf_options.single_step:
     return perf_test_runner.PrintTestOutput('single_step')
 
   perf_test_runner.PrintSummary(tests)
 
   # Always return 0 on the sharding stage. Individual tests exit_code
   # will be returned on the print_step stage.
   return 0
 
@@ skipping 106 matching lines @@
   def infra_error(message):
     parser.exit(status=constants.INFRA_EXIT_CODE, message=message)
 
   if args.command not in _SUPPORTED_IN_PLATFORM_MODE:
     infra_error('%s is not yet supported in platform mode' % args.command)
 
   with environment_factory.CreateEnvironment(args, infra_error) as env:
     with test_instance_factory.CreateTestInstance(args, infra_error) as test:
       with test_run_factory.CreateTestRun(
           args, env, test, infra_error) as test_run:
-        results = test_run.RunTests()
+        results = []
+        repetitions = (xrange(args.repeat + 1) if args.repeat >= 0
+                       else itertools.count())
+        for _ in repetitions:
+          iteration_results = test_run.RunTests()
+          results.append(iteration_results)
 
-        if args.environment == 'remote_device' and args.trigger:
-          return 0 # Not returning results, only triggering.
-
-        report_results.LogFull(
-            results=results,
-            test_type=test.TestType(),
-            test_package=test_run.TestPackage(),
-            annotation=getattr(args, 'annotations', None),
-            flakiness_server=getattr(args, 'flakiness_dashboard_server', None))
+          if iteration_results is not None:
+            report_results.LogFull(
+                results=iteration_results,
+                test_type=test.TestType(),
+                test_package=test_run.TestPackage(),
+                annotation=getattr(args, 'annotations', None),
+                flakiness_server=getattr(args, 'flakiness_dashboard_server',
+                                         None))
 
         if args.json_results_file:
           json_results.GenerateJsonResultsFile(
               results, args.json_results_file)
 
-  return 0 if results.DidRunPass() else constants.ERROR_EXIT_CODE
+  return (0 if all(r.DidRunPass() for r in results)
+          else constants.ERROR_EXIT_CODE)
 
 
 CommandConfigTuple = collections.namedtuple(
     'CommandConfigTuple',
     ['add_options_func', 'help_txt'])
 VALID_COMMANDS = {
     'gtest': CommandConfigTuple(
         AddGTestOptions,
         'googletest-based C++ tests'),
     'instrumentation': CommandConfigTuple(
@@ skipping 50 matching lines @@
     if e.is_infra_error:
       return constants.INFRA_EXIT_CODE
     return constants.ERROR_EXIT_CODE
   except: # pylint: disable=W0702
     logging.exception('Unrecognized error occurred.')
     return constants.ERROR_EXIT_CODE
 
 
 if __name__ == '__main__':
   sys.exit(main())