[Android] Redesigns the sharder to allow replicated vs distributed tests

build/android/test_runner.py

 #!/usr/bin/env python
 #
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Runs all types of tests from one unified interface.
 
 TODO(gkanwar):
 * Add options to run Monkey tests.
 """
 
 import collections
 import optparse
 import os
 import sys
 
 from pylib import cmd_helper
 from pylib import constants
 from pylib import ports
 from pylib.base import base_test_result
-from pylib.browsertests import dispatch as browsertests_dispatch
-from pylib.gtest import dispatch as gtest_dispatch
+from pylib.base import test_dispatcher
+from pylib.browsertests import setup as browsertests_setup
+from pylib.gtest import setup as gtest_setup
+from pylib.gtest import gtest_config
 from pylib.host_driven import run_python_tests as python_dispatch
-from pylib.instrumentation import dispatch as instrumentation_dispatch
-from pylib.uiautomator import dispatch as uiautomator_dispatch
-from pylib.utils import emulator, report_results, run_tests_helper
+from pylib.instrumentation import setup as instrumentation_setup
+from pylib.uiautomator import setup as uiautomator_setup
+from pylib.utils import report_results
+from pylib.utils import run_tests_helper
 
 
 _SDK_OUT_DIR = os.path.join(constants.DIR_SOURCE_ROOT, 'out')
 
 
 def AddBuildTypeOption(option_parser):
   """Adds the build type option to |option_parser|."""
   default_build_type = 'Debug'
   if 'BUILDTYPE' in os.environ:
     default_build_type = os.environ['BUILDTYPE']
   option_parser.add_option('--debug', action='store_const', const='Debug',
                            dest='build_type', default=default_build_type,
                            help=('If set, run test suites under out/Debug. '
                                  'Default is env var BUILDTYPE or Debug.'))
   option_parser.add_option('--release', action='store_const',
                            const='Release', dest='build_type',
                            help=('If set, run test suites under out/Release.'
                                  ' Default is env var BUILDTYPE or Debug.'))
 
 
-def AddEmulatorOptions(option_parser):
-  """Adds all emulator-related options to |option_parser|."""
-
-  # TODO(gkanwar): Figure out what we're doing with the emulator setup
-  # and determine whether these options should be deprecated/removed.
-  option_parser.add_option('-e', '--emulator', dest='use_emulator',
-                           action='store_true',
-                           help='Run tests in a new instance of emulator.')
-  option_parser.add_option('-n', '--emulator-count',
-                           type='int', default=1,
-                           help=('Number of emulators to launch for '
-                                 'running the tests.'))
-  option_parser.add_option('--abi', default='armeabi-v7a',
-                           help='Platform of emulators to launch.')
-
-
-def ProcessEmulatorOptions(options):
-  """Processes emulator options."""
-  if options.use_emulator:
-    emulator.DeleteAllTempAVDs()
-
-
-def AddCommonOptions(option_parser):
+def AddCommonOptions(option_parser, default_timeout=60):
   """Adds all common options to |option_parser|."""
 
   AddBuildTypeOption(option_parser)
 
   option_parser.add_option('--out-directory', dest='out_directory',
                            help=('Path to the out/ directory, irrespective of '
                                  'the build type. Only for non-Chromium uses.'))
   option_parser.add_option('-c', dest='cleanup_test_files',
                            help='Cleanup test files on the device after run',
                            action='store_true')
@@ skipping 23 matching lines @@
                            help=('Address of the server that is hosting the '
                                  'Chrome for Android flakiness dashboard.'))
   option_parser.add_option('--skip-deps-push', dest='push_deps',
                            action='store_false', default=True,
                            help=('Do not push dependencies to the device. '
                                  'Use this at own risk for speeding up test '
                                  'execution on local machine.'))
   option_parser.add_option('-d', '--device', dest='test_device',
                            help=('Target device for the test suite '
                                  'to run on.'))
+  option_parser.add_option('-t', dest='timeout',
+                           help='Timeout to wait for each test',
+                           type='int',
+                           default=default_timeout)
 
 
 def ProcessCommonOptions(options):
   """Processes and handles all common options."""
   if options.out_directory:
     cmd_helper.OutDirectory.set(options.out_directory)
   run_tests_helper.SetLogLevel(options.verbose_count)
 
 
-def AddCoreGTestOptions(option_parser, default_timeout=60):
+def AddCoreGTestOptions(option_parser):
   """Add options specific to the gtest framework to |option_parser|."""
 
   # TODO(gkanwar): Consolidate and clean up test filtering for gtests and
   # content_browsertests.
-  option_parser.add_option('--gtest_filter', dest='test_filter',
+  option_parser.add_option('--gtest_filter', dest='gtest_filter',
                            help='Filter GTests by name.')
   option_parser.add_option('-a', '--test_arguments', dest='test_arguments',
                            help='Additional arguments to pass to the test.')
-  # TODO(gkanwar): Most likely deprecate/remove this option once we've pinned
-  # down what we're doing with the emulator setup.
-  option_parser.add_option('-x', '--xvfb', dest='use_xvfb',
-                           action='store_true',
-                           help='Use Xvfb around tests (ignored if not Linux).')
   # TODO(gkanwar): Possible deprecate this flag. Waiting on word from Peter
   # Beverloo.
   option_parser.add_option('--webkit', action='store_true',
                            help='Run the tests from a WebKit checkout.')
   option_parser.add_option('--exe', action='store_true',
                            help='If set, use the exe test runner instead of '
                            'the APK.')
-  option_parser.add_option('-t', dest='timeout',
-                           help='Timeout to wait for each test',
-                           type='int',
-                           default=default_timeout)
 
 
 def AddContentBrowserTestOptions(option_parser):
   """Adds Content Browser test options to |option_parser|."""
 
   option_parser.usage = '%prog content_browsertests [options]'
   option_parser.command_list = []
   option_parser.example = '%prog content_browsertests'
 
   AddCoreGTestOptions(option_parser)
   AddCommonOptions(option_parser)
 
 
 def AddGTestOptions(option_parser):
   """Adds gtest options to |option_parser|."""
 
   option_parser.usage = '%prog gtest [options]'
   option_parser.command_list = []
   option_parser.example = '%prog gtest -s base_unittests'
 
+  # TODO(gkanwar): Make this option required
   option_parser.add_option('-s', '--suite', dest='test_suite',
                            help=('Executable name of the test suite to run '
                                  '(use -s help to list them).'))
   AddCoreGTestOptions(option_parser)
   # TODO(gkanwar): Move these to Common Options once we have the plumbing
   # in our other test types to handle these commands
-  AddEmulatorOptions(option_parser)
   AddCommonOptions(option_parser)
 
 
+def ProcessGTestOptions(options):
+  """Intercept test suite help to list test suites.
+
+  Args:
+    options: command line options.
+
+  Returns:
+    True if the command should continue.
+  """
+  if options.test_suite == 'help':
+    print 'Available test suites are:'
+    for test_suite in gtest_config.STABLE_TEST_SUITES:

[frankf, 2013/07/16 18:51:34]

print the name not the tuple

[gkanwar, 2013/07/16 20:48:32]

Done.

+      print test_suite
+    return False
+
+  # Convert to a list, assuming all test suites if nothing was specified.
+  # TODO(gkanwar): Require having a test suite
+  if options.test_suite:
+    options.test_suite = [options.test_suite]
+  else:
+    options.test_suite = [suite.name
+                          for suite in gtest_config.STABLE_TEST_SUITES]
+  return True
+
+
 def AddJavaTestOptions(option_parser):
   """Adds the Java test options to |option_parser|."""
 
   option_parser.add_option('-f', '--test_filter', dest='test_filter',
                            help=('Test filter (if not fully qualified, '
                                  'will run all matches).'))
   option_parser.add_option(
       '-A', '--annotation', dest='annotation_str',
       help=('Comma-separated list of annotations. Run only tests with any of '
             'the given annotations. An annotation can be either a key or a '
             'key-values pair. A test that has no annotation is considered '
             '"SmallTest".'))
   option_parser.add_option(
       '-E', '--exclude-annotation', dest='exclude_annotation_str',
       help=('Comma-separated list of annotations. Exclude tests with these '
             'annotations.'))
   option_parser.add_option('-j', '--java_only', action='store_true',
                            default=False, help='Run only the Java tests.')
   option_parser.add_option('-p', '--python_only', action='store_true',
                            default=False,
                            help='Run only the host-driven tests.')
   option_parser.add_option('--screenshot', dest='screenshot_failures',
                            action='store_true',
                            help='Capture screenshots of test failures')
   option_parser.add_option('--save-perf-json', action='store_true',
                            help='Saves the JSON file for each UI Perf test.')
-  # TODO(gkanwar): Remove this option. It is not used anywhere.
-  option_parser.add_option('--shard_retries', type=int, default=1,
-                           help=('Number of times to retry each failure when '
-                                 'sharding.'))
   option_parser.add_option('--official-build', help='Run official build tests.')
   option_parser.add_option('--python_test_root',
                            help='Root of the host-driven tests.')
   option_parser.add_option('--keep_test_server_ports',
                            action='store_true',
                            help=('Indicates the test server ports must be '
                                  'kept. When this is run via a sharder '
                                  'the test server ports should be kept and '
                                  'should not be reset.'))
   # TODO(gkanwar): This option is deprecated. Remove it in the future.
@@ skipping 126 matching lines @@
     options.uiautomator_jar = options.test_jar
   else:
     options.uiautomator_jar = os.path.join(
         _SDK_OUT_DIR, options.build_type, constants.SDK_BUILD_JAVALIB_DIR,
         '%s.dex.jar' % options.test_jar)
   options.uiautomator_info_jar = (
       options.uiautomator_jar[:options.uiautomator_jar.find('.dex.jar')] +
       '_java.jar')
 
 
+def _RunGTests(options, error_func):
+  """Subcommand of RunTestsCommands which runs gtests."""
+  if not ProcessGTestOptions(options):
+    return 0
+
+  exit_code = 0
+  results = base_test_result.TestRunResults()
+  for test_suite in options.test_suite:
+    runner_factory, tests = gtest_setup.Setup(
+        options.exe, test_suite, options.test_arguments,
+        options.timeout, options.cleanup_test_files, options.tool,
+        options.build_type, options.webkit, options.push_deps,
+        options.gtest_filter)
+
+    test_results, test_exit_code = test_dispatcher.RunTests(
+        tests, runner_factory, False, options.test_device,
+        test_allocation='shard',
+        build_type=options.build_type,
+        test_timeout=options.timeout,
+        num_retries=options.num_retries)
+    results.AddTestRunResults(test_results)

[frankf, 2013/07/16 18:51:34]

be consistant with whitespace

[gkanwar, 2013/07/16 20:48:32]

Done.

+    if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
+      exit_code = test_exit_code
+
+  report_results.LogFull(

[frankf, 2013/07/16 18:51:34]

Was the old behaviour to tally the results?

[gkanwar, 2013/07/16 20:48:32]

Ah, looks like the old behavior was to log each suite separately... fixed.

+      results=results,
+      test_type='Unit test',
+      test_package=options.test_suite,
+      build_type=options.build_type,
+      flakiness_server=options.flakiness_dashboard_server)
+
+  return exit_code
+
+
+def _RunContentBrowserTests(options, error_func):
+  """Subcommand of RunTestsCommands which runs content_browsertests."""
+  runner_factory, tests = browsertests_setup.Setup(
+      options.test_arguments, options.timeout, options.cleanup_test_files,
+      options.tool, options.build_type, options.webkit, options.push_deps,
+      options.gtest_filter)
+
+  # TODO(nileshagrawal): remove this abnormally long setup timeout once fewer
+  # files are pushed to the devices for content_browsertests: crbug.com/138275
+  setup_timeout = 20 * 60  # 20 minutes
+  results, exit_code = test_dispatcher.RunTests(
+      tests, runner_factory, False, options.test_device,
+      test_allocation='shard',
+      build_type=options.build_type,
+      test_timeout=options.timeout,
+      setup_timeout=setup_timeout,
+      num_retries=options.num_retries)
+
+  report_results.LogFull(
+      results=results,
+      test_type='Unit test',
+      test_package=constants.BROWSERTEST_SUITE_NAME,
+      build_type=options.build_type,
+      flakiness_server=options.flakiness_dashboard_server)
+
+  return exit_code
+
+
+def _RunInstrumentationTests(options, error_func):
+  """Subcommand of RunTestsCommands which runs instrumentation tests."""
+  ProcessInstrumentationOptions(options, error_func)
+
+  results = base_test_result.TestRunResults()
+  exit_code = 0
+
+  if options.run_java_tests:
+    runner_factory, tests = instrumentation_setup.Setup(
+        options.test_apk_path, options.test_apk_jar_path, options.annotations,
+        options.exclude_annotations, options.test_filter, options.build_type,
+        options.test_data, options.install_apk, options.save_perf_json,
+        options.screenshot_failures, options.tool, options.wait_for_debugger,
+        options.disable_assertions, options.push_deps)
+
+    test_results, exit_code = test_dispatcher.ShardAndRunTests(
+        tests, runner_factory, options.wait_for_debugger, options.test_device,
+        test_allocation='shard',
+        build_type=options.build_type,
+        test_timeout=options.timeout,
+        num_retries=options.num_retries)
+
+    results.AddTestRunResults(test_results)
+
+  if options.run_python_tests:
+    test_results, test_exit_code = (python_dispatch.
+                                    DispatchPythonTests(options))
+    results.AddTestRunResults(test_results)
+    # Only allow exit code escalation
+    if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
+      exit_code = test_exit_code
+
+  report_results.LogFull(
+      results=results,
+      test_type='Instrumentation',
+      test_package=os.path.basename(options.test_apk),
+      annotation=options.annotations,
+      build_type=options.build_type,
+      flakiness_server=options.flakiness_dashboard_server)
+
+  return exit_code
+
+
+def _RunUIAutomatorTests(options, error_func):
+  """Subcommand of RunTestsCommands which runs uiautomator tests."""
+  ProcessUIAutomatorOptions(options, error_func)
+
+  results = base_test_result.TestRunResults()
+  exit_code = 0
+
+  if options.run_java_tests:
+    runner_factory, tests = uiautomator_setup.Setup(
+        options.uiautomator_jar, options.uiautomator_info_jar,
+        options.annotations, options.exclude_annotations, options.test_filter,
+        options.package_name, options.build_type, options.test_data,
+        options.save_perf_json, options.screenshot_failures, options.tool,
+        options.disable_assertions, options.push_deps)
+
+    test_results, exit_code = test_dispatcher.ShardAndRunTests(
+        tests, runner_factory, False, options.test_device,
+        test_allocation='shard',
+        build_type=options.build_type,
+        test_timeout=options.timeout,
+        num_retries=options.num_retries)
+
+    results.AddTestRunResults(test_results)
+
+  if options.run_python_tests:
+    test_results, test_exit_code = (python_dispatch.
+                                    DispatchPythonTests(options))
+    results.AddTestRunResults(test_results)
+    # Only allow exit code escalation
+    if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
+      exit_code = test_exit_code
+
+  report_results.LogFull(
+      results=results,
+      test_type='UIAutomator',
+      test_package=os.path.basename(options.test_jar),
+      annotation=options.annotations,
+      build_type=options.build_type,
+      flakiness_server=options.flakiness_dashboard_server)
+
+  return exit_code
+
+
 def RunTestsCommand(command, options, args, option_parser):
   """Checks test type and dispatches to the appropriate function.
 
   Args:
     command: String indicating the command that was received to trigger
         this function.
     options: optparse options dictionary.
     args: List of extra args from optparse.
     option_parser: optparse.OptionParser object.
 
   Returns:
     Integer indicated exit code.
 
   Raises:
     Exception: Unknown command name passed in, or an exception from an
         individual test runner.
   """
 
   ProcessCommonOptions(options)
 
   if command == 'gtest':
-    # TODO(gkanwar): See the emulator TODO above -- this call should either go
-    # away or become generalized.
-    ProcessEmulatorOptions(options)
-    results, exit_code = gtest_dispatch.Dispatch(options)
+    return _RunGTests(options, option_parser.error)
   elif command == 'content_browsertests':
-    results, exit_code = browsertests_dispatch.Dispatch(options)
+    return _RunContentBrowserTests(options, option_parser.error)
   elif command == 'instrumentation':
-    ProcessInstrumentationOptions(options, option_parser.error)
-    results = base_test_result.TestRunResults()
-    exit_code = 0
-    if options.run_java_tests:
-      test_results, exit_code = instrumentation_dispatch.Dispatch(options)
-      results.AddTestRunResults(test_results)
-    if options.run_python_tests:
-      test_results, test_exit_code = (python_dispatch.
-                                      DispatchPythonTests(options))
-      results.AddTestRunResults(test_results)
-      # Only allow exit code escalation
-      if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
-        exit_code = test_exit_code
-    report_results.LogFull(
-        results=results,
-        test_type='Instrumentation',
-        test_package=os.path.basename(options.test_apk),
-        annotation=options.annotations,
-        build_type=options.build_type,
-        flakiness_server=options.flakiness_dashboard_server)
+    return _RunInstrumentationTests(options, option_parser.error)
   elif command == 'uiautomator':
-    ProcessUIAutomatorOptions(options, option_parser.error)
-    results = base_test_result.TestRunResults()
-    exit_code = 0
-    if options.run_java_tests:
-      test_results, exit_code = uiautomator_dispatch.Dispatch(options)
-      results.AddTestRunResults(test_results)
-    if options.run_python_tests:
-      test_results, test_exit_code = (python_dispatch.
-                                      DispatchPythonTests(options))
-      results.AddTestRunResults(test_results)
-      # Only allow exit code escalation
-      if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
-        exit_code = test_exit_code
-    report_results.LogFull(
-        results=results,
-        test_type='UIAutomator',
-        test_package=os.path.basename(options.test_jar),
-        annotation=options.annotations,
-        build_type=options.build_type,
-        flakiness_server=options.flakiness_dashboard_server)
+    return _RunUIAutomatorTests(options, option_parser.error)
   else:
     raise Exception('Unknown test type state')
 
   return exit_code
 
 
 def HelpCommand(command, options, args, option_parser):
   """Display help for a certain command, or overall help.
 
   Args:
@@ skipping 84 matching lines @@
     return 0
   command = argv[1]
   VALID_COMMANDS[command].add_options_func(option_parser)
   options, args = option_parser.parse_args(argv)
   return VALID_COMMANDS[command].run_command_func(
       command, options, args, option_parser)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))