[Android] Clean up gtest filtering logic.

build/android/pylib/gtest/dispatch.py

 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Dispatches GTests."""
 
 import copy
 import fnmatch
 import glob
 import logging
 import os
 import shutil
+import sys
 
 from pylib import android_commands
 from pylib import cmd_helper
 from pylib import constants
 from pylib import ports
 from pylib.base import base_test_result
 from pylib.base import shard
 from pylib.utils import emulator
 from pylib.utils import report_results
 from pylib.utils import xvfb
 
 import gtest_config
 import test_runner
 
+sys.path.insert(0,
+                os.path.join(constants.DIR_SOURCE_ROOT, 'build', 'util', 'lib'))
+from common import unittest_util
+
 
 # TODO(frankf): Add more test targets here after making sure we don't
 # blow up the dependency size (and the world).
 _ISOLATE_FILE_PATHS = {
     'base_unittests': 'base/base_unittests.isolate',
     'breakpad_unittests': 'breakpad/breakpad_unittests.isolate',
     'cc_perftests': 'cc/cc_perftests.isolate',
     'components_unittests': 'components/components_unittests.isolate',
     'content_browsertests': 'content/content_browsertests.isolate',
     'content_unittests': 'content/content_unittests.isolate',
@@ skipping 27 matching lines @@
 ]
 
 _ISOLATE_SCRIPT = os.path.join(
     constants.DIR_SOURCE_ROOT, 'tools', 'swarm_client', 'isolate.py')
 
 
 def _GenerateDepsDirUsingIsolate(test_suite, build_type):
   """Generate the dependency dir for the test suite using isolate.
 
   Args:
-    test_suite: The test suite basename (e.g. base_unittests).
+    test_suite: Name of the test suite (e.g. base_unittests).
     build_type: Release/Debug
   """
   product_dir = os.path.join(cmd_helper.OutDirectory.get(), build_type)
   assert os.path.isabs(product_dir)
 
   if os.path.isdir(constants.ISOLATE_DEPS_DIR):
     shutil.rmtree(constants.ISOLATE_DEPS_DIR)
 
   isolate_rel_path = _ISOLATE_FILE_PATHS.get(test_suite)
   if not isolate_rel_path:
@@ skipping 99 matching lines @@
 
   for t, q in qualified_test_suites:
     if not os.path.exists(q):
       raise Exception('Test suite %s not found in %s.\n'
                       'Supported test suites:\n %s\n'
                       'Ensure it has been built.\n' %
                       (t, q, [s.name for s in gtest_config.STABLE_TEST_SUITES]))
   return qualified_test_suites
 
 
-def GetTestsFromDevice(runner):
-  """Get a list of tests from a device, excluding disabled tests.
+def _GetDisabledTestsFilterFromFile(test_suite):
+  """Returns a gtest filter based on the *_disabled file.
 
   Args:
-    runner: a TestRunner.
+    test_suite: Name of the test suite (e.g. base_unittests).
+
   Returns:
-    All non-disabled tests on the device.
+    A gtest filter which excludes disabled tests.
+    Example: '*-StackTrace.*:StringPrintfTest.StringPrintfMisc'
   """
-  # The executable/apk needs to be copied before we can call GetAllTests.
-  runner.test_package.Install()
-  all_tests = runner.test_package.GetAllTests()
-  # Only includes tests that do not have any match in the disabled list.
-  disabled_list = runner.GetDisabledTests()
-  return filter(lambda t: not any([fnmatch.fnmatch(t, disabled_pattern)
-                                   for disabled_pattern in disabled_list]),
-                all_tests)
+  filter_file_path = os.path.join(
+      os.path.abspath(os.path.dirname(__file__)),
+      'filter', '%s_disabled' % test_suite)
+
+  if not filter_file_path or not os.path.exists(filter_file_path):

[craigdh, 2013/07/17 21:27:03]

logging.info that no filter file was found?

[frankf, 2013/07/17 22:18:54]

Done.

+    return '*'
+
+  filters = [x for x in [x.strip() for x in file(filter_file_path).readlines()]
+             if x and x[0] != '#']
+  disabled_filter = '*-%s' % ':'.join(filters)
+  logging.info('Applying filter "%s" obtained from %s',
+               disabled_filter, filter_file_path)
+  return disabled_filter
 
 
-def GetAllEnabledTests(runner_factory, devices):
-  """Get all enabled tests.
-
-  Obtains a list of enabled tests from the test package on the device,
-  then filters it again using the disabled list on the host.
+def _GetTestsFromDevice(runner_factory, devices):
+  """Get a list of tests from a device.
 
   Args:
-    runner_factory: callable that takes a devices and returns a TestRunner.
-    devices: list of devices.
+    runner_factory: Callable that takes a devices and returns a TestRunner.

[craigdh, 2013/07/17 21:27:03]

devices -> device and shard index

[frankf, 2013/07/17 22:18:54]

Done.

+    devices: List of devices.
 
   Returns:
-    List of all enabled tests.
-
-  Raises:
-    Exception: If no devices available.
+    All the tests in the test suite.
   """
   for device in devices:
     try:
       logging.info('Obtaining tests from %s', device)
       runner = runner_factory(device, 0)
-      return GetTestsFromDevice(runner)
+      runner.test_package.Install()
+      return runner.test_package.GetAllTests()
     except Exception as e:
       logging.warning('Failed obtaining tests from %s with exception: %s',
                       device, e)
   raise Exception('No device available to get the list of tests.')

[craigdh, 2013/07/17 21:27:03]

This is a pain right now, because if there is an issue in the test scripts it
often shows up when we first install (which is here), but we lose the stack
trace because we only print the exception message. If we fail on the last device
we should just raise that exception instead of catching it and raising our own.

[frankf, 2013/07/17 22:18:54]

The issue is that we're catching too general exception. We should only catch
android_commands specific ones.

On 2013/07/17 21:27:03, craigdh wrote:

 
 
+def _FilterTestsUsingPrefixes(all_tests, pre=False, manual=False):
+  """Removes tests with disabled prefixes.
+
+  Args:
+    all_tests: List of tests to filter.
+    pre: If True, include tests with _PRE prefix.
+    manual: If True, include tests with _MANUAL prefix.
+
+  Returns:
+    List of tests remaining.
+  """
+  filtered_tests = []
+  filter_prefixes = ['DISABLED_', 'FLAKY_', 'FAILS_']
+
+  if not pre:
+    filter_prefixes.append('PRE_')
+
+  if not manual:
+    filter_prefixes.append('MANUAL_')
+
+  for t in all_tests:
+    test_case, test = t.split('.', 1)
+    if any([test_case.startswith(prefix) or test.startswith(prefix) for prefix

[craigdh, 2013/07/17 21:27:03]

lose the continue, just do:

if not any(...):
  filtered_tests.append(t)

[frankf, 2013/07/17 22:18:54]

Done.

+            in filter_prefixes]):
+      continue
+    filtered_tests.append(t)
+  return filtered_tests
+
+
+def GetTestsFiltered(test_suite, gtest_filter, runner_factory, devices):
+  """Get all tests in the suite and filter them.
+
+  Obtains a list of tests from the test package on the device, and
+  applies the following filters in order:
+    1. Remove tests with disabled prefixes.
+    2. Remove tests specified in the *_disabled files in the 'filter' dir
+    3. Applies |gtest_filter|.
+
+  Args:
+    test_suite: Name of the test suite (e.g. base_unittests).
+    gtest_filter: A filter including negative or positive patterns.
+    runner_factory: Callable that takes a devices and returns a TestRunner.
+    devices: List of devices.
+
+  Returns:
+    List of tests remaining.
+  """
+  tests = _GetTestsFromDevice(runner_factory, devices)
+  tests = _FilterTestsUsingPrefixes(
+      tests, bool(gtest_filter), bool(gtest_filter))
+  tests = unittest_util.FilterTestNames(
+      tests, _GetDisabledTestsFilterFromFile(test_suite))
+
+  if gtest_filter:
+    tests = unittest_util.FilterTestNames(tests, gtest_filter)
+
+  return tests
+
+
 def _RunATestSuite(options, suite_name):
   """Run a single test suite.
 
   Helper for Dispatch() to allow stop/restart of the emulator across
   test bundles.  If using the emulator, we start it on entry and stop
   it on exit.
 
   Args:
     options: options for running the tests.
     suite_name: name of the test suite being run.
@@ skipping 38 matching lines @@
         options.cleanup_test_files,
         options.tool,
         options.build_type,
         options.webkit,
         options.push_deps,
         constants.GTEST_TEST_PACKAGE_NAME,
         constants.GTEST_TEST_ACTIVITY_NAME,
         constants.GTEST_COMMAND_LINE_FILE)
 
   # Get tests and split them up based on the number of devices.
-  if options.test_filter:
-    all_tests = [t for t in options.test_filter.split(':') if t]
-  else:
-    all_tests = GetAllEnabledTests(RunnerFactory, attached_devices)
+  tests = GetTestsFiltered(suite_name, options.test_filter,
+                           RunnerFactory, attached_devices)
   num_devices = len(attached_devices)
-  tests = [':'.join(all_tests[i::num_devices]) for i in xrange(num_devices)]
+  tests = [':'.join(tests[i::num_devices]) for i in xrange(num_devices)]
   tests = [t for t in tests if t]
 
   # Run tests.
   test_results, exit_code = shard.ShardAndRunTests(
       RunnerFactory, attached_devices, tests, options.build_type,
       test_timeout=None, num_retries=options.num_retries)
 
   report_results.LogFull(
       results=test_results,
       test_type='Unit test',
@@ skipping 48 matching lines @@
     test_options.test_suite = suite_path
     test_results, test_exit_code = _RunATestSuite(test_options, suite_name)
     results.AddTestRunResults(test_results)
     if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
       exit_code = test_exit_code
 
   if options.use_xvfb:
     framebuffer.Stop()
 
   return (results, exit_code)