Clean up Android gtest runners.

build/android/run_tests.py

 #!/usr/bin/env python
 #
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Runs all the native unit tests.
 
 1. Copy over test binary to /data/local on device.
 2. Resources: chrome/unit_tests requires resources (chrome.pak and en-US.pak)
    to be deployed to the device. We use the device's $EXTERNAL_STORAGE as the
    base dir (which maps to Context.getExternalFilesDir()).
 3. Environment:
 3.1. chrome/unit_tests requires (via chrome_paths.cc) a directory named:
      $EXTERNAL_STORAGE + /chrome/test/data
 3.2. page_cycler_tests have following requirements,
 3.2.1  the following data on host:
        <chrome_src_dir>/tools/page_cycler
        <chrome_src_dir>/data/page_cycler
 3.2.2. two data directories to store above test data on device named:
        $EXTERNAL_STORAGE + /tools/ (for database perf test)
        $EXTERNAL_STORAGE + /data/ (for other perf tests)
 3.2.3. a http server to serve http perf tests.
        The http root is host's <chrome_src_dir>/data/page_cycler/, port 8000.
 3.2.4  a tool named forwarder is also required to run on device to
        forward the http request/response between host and device.
 3.2.5  Chrome is installed on device.
 4. Run the binary in the device and stream the log to the host.
 4.1. Optionally, filter specific tests.
-4.2. Optionally, rebaseline: run the available tests and update the
-     suppressions file for failures.
-4.3. If we're running a single test suite and we have multiple devices
+4.2. If we're running a single test suite and we have multiple devices
      connected, we'll shard the tests.
 5. Clean up the device.
 
 Suppressions:
 
 Individual tests in a test binary can be suppressed by listing it in
 the gtest_filter directory in a file of the same name as the test binary,
 one test per line. Here is an example:
 
   $ cat gtest_filter/base_unittests_disabled
   DataPackTest.Load
   ReadOnlyFileUtilTest.ContentsEqual
 
 This file is generated by the tests running on devices. If running on emulator,
 additonal filter file which lists the tests only failed in emulator will be
 loaded. We don't care about the rare testcases which succeeded on emuatlor, but
 failed on device.
 """
 
 import copy
 import fnmatch
 import logging
 import optparse
 import os
 import signal
 import subprocess
 import sys
 import time
 
+import emulator
 from pylib import android_commands
-from pylib.base_test_sharder import BaseTestSharder
 from pylib import buildbot_report
 from pylib import cmd_helper
-from pylib import constants
 from pylib import debug_info
-import emulator
 from pylib import ports
 from pylib import run_tests_helper
 from pylib import test_options_parser
+from pylib.base_test_sharder import BaseTestSharder
 from pylib.single_test_runner import SingleTestRunner
-from pylib.test_result import BaseTestResult, TestResults
 
 
 _TEST_SUITES = ['base_unittests',
                 'cc_unittests',
                 'content_unittests',
                 'gpu_unittests',
                 'ipc_tests',
                 'media_unittests',
                 'net_unittests',
                 'sql_unittests',
                 'sync_unit_tests',
                 'ui_unittests',
                 'unit_tests',
                 'webkit_compositor_bindings_unittests',
                ]
 
 
 def FullyQualifiedTestSuites(exe, option_test_suite, build_type):
-  """Return a fully qualified list
+  """Get a list of absolute paths to test suite targets.
 
   Args:
     exe: if True, use the executable-based test runner.
     option_test_suite: the test_suite specified as an option.
     build_type: 'Release' or 'Debug'.
   """
   test_suite_dir = os.path.join(cmd_helper.OutDirectory.get(), build_type)
   if option_test_suite:
     all_test_suites = [option_test_suite]
   else:
@@ skipping 79 matching lines @@
       except:
         pass
       del os.environ['DISPLAY']
       self._pid = 0
 
 
 class TestSharder(BaseTestSharder):
   """Responsible for sharding the tests on the connected devices."""
 
   def __init__(self, attached_devices, test_suite, gtest_filter,
-               test_arguments, timeout, rebaseline, performance_test,
+               test_arguments, timeout, performance_test,
                cleanup_test_files, tool, log_dump_name, fast_and_loose,
                build_type, in_webkit_checkout):
     BaseTestSharder.__init__(self, attached_devices, build_type)
     self.test_suite = test_suite
     self.test_suite_basename = os.path.basename(test_suite)
     self.gtest_filter = gtest_filter or ''
     self.test_arguments = test_arguments
     self.timeout = timeout
-    self.rebaseline = rebaseline
     self.performance_test = performance_test
     self.cleanup_test_files = cleanup_test_files
     self.tool = tool
     self.log_dump_name = log_dump_name
     self.fast_and_loose = fast_and_loose
-    self.build_type = build_type
     self.in_webkit_checkout = in_webkit_checkout
-    self.tests = []
+    self.all_tests = []
     if not self.gtest_filter:
       # No filter has been specified, let's add all tests then.
-      self.tests, self.attached_devices = self._GetTests()
+      self.all_tests = self._GetAllEnabledTests()
+    self.tests = self.all_tests
 
-  def _GetTests(self):
-    """Returns a tuple of (all_tests, available_devices).
+  def _GetAllEnabledTests(self):
+    """Returns a list of all enabled tests.
 
-    Tries to obtain the list of available tests.
+    Obtains a list of enabled tests from the test package on the device,
+    then filters it again using the diabled list on the host.

[bulach, 2012/12/13 14:55:21]

I dislike this combination :)
the while + try is tricky enough, and there's no need to make it even more
complex by combining with the list from the host here...
keep this function as only getting things from the device itself, then create
another wrapper to filter the list..

also, in general, such changes could be made in separate patches, that is, one
for removing the baseline option, another one for improving these other bits..

[frankf, 2012/12/13 19:54:04]

Done.

On 2012/12/13 14:55:21, bulach wrote:

+
     Raises Exception if all devices failed.
     """
     available_devices = list(self.attached_devices)
     while available_devices:
       try:
-        logging.info('Obtaining tests from %s', available_devices[-1])
-        all_tests = self._GetTestsFromDevice(available_devices[-1])
-        return all_tests, available_devices
+        current_device = available_devices[-1]
+        logging.info('Obtaining tests from %s', current_device)
+        test_runner = SingleTestRunner(
+            current_device,
+            self.test_suite,
+            self.gtest_filter,
+            self.test_arguments,
+            self.timeout,
+            self.performance_test,
+            self.cleanup_test_files,
+            self.tool,
+            0,
+            not not self.log_dump_name,

[Yaron, 2012/12/13 00:04:34]

cute

+            self.fast_and_loose,
+            self.build_type,
+            self.in_webkit_checkout)
+        # The executable/apk needs to be copied before we can call GetAllTests.
+        test_runner.test_package.StripAndCopyExecutable()
+        all_tests = test_runner.test_package.GetAllTests()
+        disabled_list = test_runner.GetDisabledTests()
+        # Only includes tests that do not have any match in the disabled list.
+        all_tests = filter(lambda t:
+                           not any([fnmatch.fnmatch(t, disabled_pattern)
+                                    for disabled_pattern in disabled_list]),
+                           all_tests)
+        return all_tests
       except Exception as e:
-        logging.info('Failed obtaining tests from %s %s',
-                     available_devices[-1], e)
+        logging.warning('Failed obtaining tests from %s %s',
+                        current_device, e)
         available_devices.pop()
+
     raise Exception('No device available to get the list of tests.')
 
-  def _GetTestsFromDevice(self, device):
-    test = SingleTestRunner(device, self.test_suite, self.gtest_filter,
-                            self.test_arguments, self.timeout, self.rebaseline,
-                            self.performance_test, self.cleanup_test_files,
-                            self.tool, 0,
-                            not not self.log_dump_name, self.fast_and_loose,
-                            self.build_type, self.in_webkit_checkout)
-    # The executable/apk needs to be copied before we can call GetAllTests.
-    test.test_package.StripAndCopyExecutable()
-    all_tests = test.test_package.GetAllTests()
-    if not self.rebaseline:
-      disabled_list = test.GetDisabledTests()
-      # Only includes tests that do not have any match in the disabled list.
-      all_tests = filter(lambda t:
-                         not any([fnmatch.fnmatch(t, disabled_pattern)
-                                  for disabled_pattern in disabled_list]),
-                         all_tests)
-    return all_tests
-
   def CreateShardedTestRunner(self, device, index):
     """Creates a suite-specific test runner.
 
     Args:
       device: Device serial where this shard will run.
       index: Index of this device in the pool.
 
     Returns:
       A SingleTestRunner object.
     """
     device_num = len(self.attached_devices)
     shard_size = (len(self.tests) + device_num - 1) / device_num
     shard_test_list = self.tests[index * shard_size : (index + 1) * shard_size]
     test_filter = ':'.join(shard_test_list) + self.gtest_filter
-    return SingleTestRunner(device, self.test_suite,
-                            test_filter, self.test_arguments, self.timeout,
-                            self.rebaseline, self.performance_test,
-                            self.cleanup_test_files, self.tool, index,
-                            not not self.log_dump_name, self.fast_and_loose,
-                            self.build_type, self.in_webkit_checkout)
+    return SingleTestRunner(
+        device,
+        self.test_suite,
+        test_filter,
+        self.test_arguments,
+        self.timeout,
+        self.performance_test,
+        self.cleanup_test_files, self.tool, index,
+        not not self.log_dump_name,
+        self.fast_and_loose,
+        self.build_type,
+        self.in_webkit_checkout)
 
   def OnTestsCompleted(self, test_runners, test_results):
     """Notifies that we completed the tests."""
     test_results.LogFull('Unit test', os.path.basename(self.test_suite),
-                         self.build_type, self.tests)
+                         self.build_type, self.all_tests)
     test_results.PrintAnnotation()
-    if test_results.failed and self.rebaseline:
-      test_runners[0].UpdateFilter(test_results.failed)
     if self.log_dump_name:
       # Zip all debug info outputs into a file named by log_dump_name.
       debug_info.GTestDebugInfo.ZipAndCleanResults(
-          os.path.join(cmd_helper.OutDirectory.get(), self.build_type,
+          os.path.join(
+              cmd_helper.OutDirectory.get(), self.build_type,
               'debug_info_dumps'),
           self.log_dump_name)
 
 
 def _RunATestSuite(options):
   """Run a single test suite.
 
   Helper for Dispatch() to allow stop/restart of the emulator across
   test bundles.  If using the emulator, we start it on entry and stop
   it on exit.
 
   Args:
     options: options for running the tests.
 
   Returns:
     0 if successful, number of failing tests otherwise.
   """
   step_name = os.path.basename(options.test_suite).replace('-debug.apk', '')
   buildbot_report.PrintNamedStep(step_name)
   attached_devices = []
   buildbot_emulators = []
 
   if options.use_emulator:
     for n in range(options.emulator_count):
       t = TimeProfile('Emulator launch %d' % n)
-      avd_name =  None
+      avd_name = None
       if n > 0:
         # Creates a temporary AVD for the extra emulators.
         avd_name = 'run_tests_avd_%d' % n
       buildbot_emulator = emulator.Emulator(avd_name, options.fast_and_loose)
       buildbot_emulator.Launch(kill_all_emulators=n == 0)
       t.Stop()
       buildbot_emulators.append(buildbot_emulator)
       attached_devices.append(buildbot_emulator.device)
     # Wait for all emulators to boot completed.
     map(lambda buildbot_emulator: buildbot_emulator.ConfirmLaunch(True),
         buildbot_emulators)
   elif options.test_device:
     attached_devices = [options.test_device]
   else:
     attached_devices = android_commands.GetAttachedDevices()
 
   if not attached_devices:
     logging.critical('A device must be attached and online.')
     buildbot_report.PrintError()
     return 1
 
   # Reset the test port allocation. It's important to do it before starting
   # to dispatch any tests.
   if not ports.ResetTestServerPortAllocation():
     raise Exception('Failed to reset test server port.')
 
   if options.performance_test or options.gtest_filter:
-    # These configuration can't be split in multiple devices.
+    logging.warning('Sharding is not possible with these configurations.')
     attached_devices = [attached_devices[0]]
-  sharder = TestSharder(attached_devices, options.test_suite,
-                        options.gtest_filter, options.test_arguments,
-                        options.timeout, options.rebaseline,
-                        options.performance_test,
-                        options.cleanup_test_files, options.tool,
-                        options.log_dump, options.fast_and_loose,
-                        options.build_type, options.webkit)
+
+  sharder = TestSharder(
+      attached_devices,
+      options.test_suite,
+      options.gtest_filter,
+      options.test_arguments,
+      options.timeout,
+      options.performance_test,
+      options.cleanup_test_files,
+      options.tool,
+      options.log_dump,
+      options.fast_and_loose,
+      options.build_type,
+      options.webkit)
   test_results = sharder.RunShardedTests()
 
   for buildbot_emulator in buildbot_emulators:
     buildbot_emulator.Shutdown()
 
   return len(test_results.failed)
 
 
 def Dispatch(options):
   """Dispatches the tests, sharding if possible.
@@ skipping 40 matching lines @@
   option_parser = optparse.OptionParser()
   test_options_parser.AddTestRunnerOptions(option_parser, default_timeout=0)
   option_parser.add_option('-s', '--suite', dest='test_suite',
                            help='Executable name of the test suite to run '
                            '(use -s help to list them)')
   option_parser.add_option('--out-directory', dest='out_directory',
                            help='Path to the out/ directory, irrespective of '
                            'the build type. Only for non-Chromium uses.')
   option_parser.add_option('-d', '--device', dest='test_device',
                            help='Target device the test suite to run ')
-  option_parser.add_option('-r', dest='rebaseline',
-                           help='Rebaseline and update *testsuite_disabled',
-                           action='store_true')
   option_parser.add_option('-f', '--gtest_filter', dest='gtest_filter',
                            help='gtest filter')
   option_parser.add_option('-a', '--test_arguments', dest='test_arguments',
                            help='Additional arguments to pass to the test')
   option_parser.add_option('-p', dest='performance_test',

[frankf, 2012/12/12 22:46:06]

@bulach: What's the history behind this option. It's not used on any bots. Do we
want to keep it?

[bulach, 2012/12/13 14:55:21]

I tried some archeology in the other trees, but no much luck... :( it could've
been back from when we ran page_cyclers as c++ unit tests...
I can't find any reference now, so feel free to dump it.

                            help='Indicator of performance test',
                            action='store_true')
   option_parser.add_option('-L', dest='log_dump',
                            help='file name of log dump, which will be put in '
                            'subfolder debug_info_dumps under the same '
                            'directory in where the test_suite exists.')
   option_parser.add_option('-e', '--emulator', dest='use_emulator',
                            action='store_true',
                            help='Run tests in a new instance of emulator')
   option_parser.add_option('-n', '--emulator_count',
@@ skipping 41 matching lines @@
   # the batch (this happens because the exit status is a sum of all failures
   # from all suites, but the buildbot associates the exit status only with the
   # most recent step).
   if options.exit_code:
     return failed_tests_count
   return 0
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))