[Android] Re-write the gtest TestRunner and introduce a new generic sharder.

build/android/pylib/gtest/dispatch.py

 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import copy
+import fnmatch
 import logging
 import os
 
 from pylib import android_commands
 from pylib import cmd_helper
 from pylib import ports
+from pylib.base import shard
+from pylib.base import test_result
 from pylib.utils import emulator
 from pylib.utils import xvfb
 
 import gtest_config
-import test_sharder
+import test_runner
 
 
 def _FullyQualifiedTestSuites(exe, option_test_suite, build_type):
   """Get a list of absolute paths to test suite targets.
 
   Args:
     exe: if True, use the executable-based test runner.
     option_test_suite: the test_suite specified as an option.
     build_type: 'Release' or 'Debug'.
 
@@ skipping 20 matching lines @@
                              for t in all_test_suites]
   for t, q in zip(all_test_suites, qualified_test_suites):
     if not os.path.exists(q):
       raise Exception('Test suite %s not found in %s.\n'
                       'Supported test suites:\n %s\n'
                       'Ensure it has been built.\n' %
                       (t, q, gtest_config.STABLE_TEST_SUITES))
   return zip(all_test_suites, qualified_test_suites)
 
 
+def _GetTestsFromDevice(runner):
+  """Get a list of test from a device, excluding disabled tests.

[nilesh, 2013/02/19 23:00:39]

list of tests

[craigdh, 2013/02/19 23:38:31]

Done.

+
+  Args:
+    runner: a TestRunner.
+  """
+  # The executable/apk needs to be copied before we can call GetAllTests.
+  runner.test_package.StripAndCopyExecutable()
+  all_tests = runner.test_package.GetAllTests()
+  # Only includes tests that do not have any match in the disabled list.
+  disabled_list = runner.GetDisabledTests()
+  return filter(lambda t: not any([fnmatch.fnmatch(t, disabled_pattern)
+                                   for disabled_pattern in disabled_list]),
+                all_tests)
+
+
+def _GetAllEnabledTests(runner_factory, devices, is_on_emulator):
+  """Get all enabled tests and available devices.

[nilesh, 2013/02/19 23:00:39]

Why does it say "all enabled tests and available devices"?

[craigdh, 2013/02/19 23:38:31]

Done.

+
+  Obtains a list of enabled tests from the test package on the device,
+  then filters it again using the diabled list on the host.

[nilesh, 2013/02/19 23:00:39]

disabled.

[craigdh, 2013/02/19 23:38:31]

Done.

+
+  Args:
+    runner_factory: callable that takes a devices and returns a TestRunner.
+    devices: list of devices.
+    is_on_emulator: whether to exclude tests disabled on the emulator.
+
+  Returns:
+    List of all enabled tests.
+
+  Raises Exception if all devices failed.
+  """
+  for device in devices:
+    print 'trying device: %s' % device
+    try:
+      logging.info('Obtaining tests from %s', device)
+      runner = runner_factory(device)
+      return _GetTestsFromDevice(runner)
+    except Exception as e:
+      logging.warning('Failed obtaining tests from %s with exception: %s',
+                      device, e)
+  raise Exception('No device available to get the list of tests.')
+
+
 def _RunATestSuite(options, suite_name):
   """Run a single test suite.
 
   Helper for Dispatch() to allow stop/restart of the emulator across
   test bundles.  If using the emulator, we start it on entry and stop
   it on exit.
 
   Args:
     options: options for running the tests.
     suite_name: name of the test suite being run.
@@ skipping 16 matching lines @@
 
   if not attached_devices:
     logging.critical('A device must be attached and online.')
     return 1
 
   # Reset the test port allocation. It's important to do it before starting
   # to dispatch any tests.
   if not ports.ResetTestServerPortAllocation():
     raise Exception('Failed to reset test server port.')
 
+  # Constructs a new TestRunner with the current options.
+  def RunnerFactory(device):
+    return test_runner.TestRunner(
+        device,
+        options.test_suite,
+        options.test_arguments,
+        options.timeout,
+        options.cleanup_test_files,
+        options.tool,
+        options.build_type,
+        options.webkit)
+
+  # Get tests and split them up based on the number of devices.
   if options.gtest_filter:
-    logging.warning('Sharding is not possible with these configurations.')
-    attached_devices = [attached_devices[0]]
+    all_tests = [t for t in options.gtest_filter.split(':') if t]

[nilesh, 2013/02/19 23:00:39]

This is changing the current behavior right? We didn't use to shard if a test
filter is specified and now we will try to.

I dont think it matters much, but filter is only used when debugging or running
tests by developers, and they can always attach only one device to disable
sharding. Just wanted to check if this was intended.

[craigdh, 2013/02/19 23:38:31]

Yeah, it simplified the code and seemed like a good thing to support. People can
always pass the list of devices they want to use on the command line if they
don't want to use all attached devices.

+  else:
+    all_tests = _GetAllEnabledTests(RunnerFactory, attached_devices,
+                                    options.use_emulator)
+  num_devices = len(attached_devices)
+  tests = [t for t in
+              [':'.join(all_tests[i::num_devices]) for i in xrange(num_devices)]
+           if t]
 
-  sharder = test_sharder.TestSharder(
-      attached_devices,
-      options.test_suite,
-      options.gtest_filter,
-      options.test_arguments,
-      options.timeout,
-      options.cleanup_test_files,
-      options.tool,
-      options.build_type,
-      options.webkit)
+  # Run tests.
+  test_results = shard.Shard(RunnerFactory, attached_devices, tests,
+                             options.build_type)
 
-  test_results = sharder.RunShardedTests()
   test_results.LogFull(
       test_type='Unit test',
       test_package=suite_name,
       build_type=options.build_type,
       flakiness_server=options.flakiness_dashboard_server)
   test_results.PrintAnnotation()
 
   for buildbot_emulator in buildbot_emulators:
     buildbot_emulator.Shutdown()
 
@@ skipping 32 matching lines @@
   failures = 0
   for suite_name, suite_path in all_test_suites:
     # Give each test suite its own copy of options.
     test_options = copy.deepcopy(options)
     test_options.test_suite = suite_path
     failures += _RunATestSuite(test_options, suite_name)
 
   if options.use_xvfb:
     framebuffer.Stop()
   return failures