Clean up Android gtest runners.

build/android/run_tests.py

 #!/usr/bin/env python
 #
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Runs all the native unit tests.
 
 1. Copy over test binary to /data/local on device.
 2. Resources: chrome/unit_tests requires resources (chrome.pak and en-US.pak)
    to be deployed to the device. We use the device's $EXTERNAL_STORAGE as the
    base dir (which maps to Context.getExternalFilesDir()).
 3. Environment:
 3.1. chrome/unit_tests requires (via chrome_paths.cc) a directory named:
      $EXTERNAL_STORAGE + /chrome/test/data
 3.2. page_cycler_tests have following requirements,
 3.2.1  the following data on host:
        <chrome_src_dir>/tools/page_cycler
        <chrome_src_dir>/data/page_cycler
 3.2.2. two data directories to store above test data on device named:
        $EXTERNAL_STORAGE + /tools/ (for database perf test)
        $EXTERNAL_STORAGE + /data/ (for other perf tests)
 3.2.3. a http server to serve http perf tests.
        The http root is host's <chrome_src_dir>/data/page_cycler/, port 8000.
 3.2.4  a tool named forwarder is also required to run on device to
        forward the http request/response between host and device.
 3.2.5  Chrome is installed on device.
 4. Run the binary in the device and stream the log to the host.
 4.1. Optionally, filter specific tests.
-4.2. Optionally, rebaseline: run the available tests and update the
-     suppressions file for failures.
-4.3. If we're running a single test suite and we have multiple devices
+4.2. If we're running a single test suite and we have multiple devices
      connected, we'll shard the tests.
 5. Clean up the device.
 
 Suppressions:
 
 Individual tests in a test binary can be suppressed by listing it in
 the gtest_filter directory in a file of the same name as the test binary,
 one test per line. Here is an example:
 
   $ cat gtest_filter/base_unittests_disabled
   DataPackTest.Load
   ReadOnlyFileUtilTest.ContentsEqual
 
 This file is generated by the tests running on devices. If running on emulator,
 additonal filter file which lists the tests only failed in emulator will be
 loaded. We don't care about the rare testcases which succeeded on emuatlor, but
 failed on device.
 """
 
 import copy
 import fnmatch
 import logging
 import optparse
 import os
 import signal
 import subprocess
 import sys
 import time
 
+import emulator
 from pylib import android_commands
-from pylib.base_test_sharder import BaseTestSharder
 from pylib import buildbot_report
 from pylib import cmd_helper
-from pylib import constants
 from pylib import debug_info
-import emulator
 from pylib import ports
 from pylib import run_tests_helper
 from pylib import test_options_parser
+from pylib.base_test_sharder import BaseTestSharder
 from pylib.single_test_runner import SingleTestRunner
-from pylib.test_result import BaseTestResult, TestResults
 
 
 _TEST_SUITES = ['base_unittests',
                 'cc_unittests',
                 'content_unittests',
                 'gpu_unittests',
                 'ipc_tests',
                 'media_unittests',
                 'net_unittests',
                 'sql_unittests',
                 'sync_unit_tests',
                 'ui_unittests',
                 'unit_tests',
                 'webkit_compositor_bindings_unittests',
                ]
 
 
 def FullyQualifiedTestSuites(exe, option_test_suite, build_type):
-  """Return a fully qualified list
+  """Get a list of absolute paths to test suite targets.
 
   Args:
     exe: if True, use the executable-based test runner.
     option_test_suite: the test_suite specified as an option.
     build_type: 'Release' or 'Debug'.
   """
   test_suite_dir = os.path.join(cmd_helper.OutDirectory.get(), build_type)
   if option_test_suite:
     all_test_suites = [option_test_suite]
   else:
@@ skipping 79 matching lines @@
       except:
         pass
       del os.environ['DISPLAY']
       self._pid = 0
 
 
 class TestSharder(BaseTestSharder):
   """Responsible for sharding the tests on the connected devices."""
 
   def __init__(self, attached_devices, test_suite, gtest_filter,
-               test_arguments, timeout, rebaseline, performance_test,
+               test_arguments, timeout, performance_test,
                cleanup_test_files, tool, log_dump_name, fast_and_loose,
                build_type, in_webkit_checkout):
     BaseTestSharder.__init__(self, attached_devices, build_type)
     self.test_suite = test_suite
     self.test_suite_basename = os.path.basename(test_suite)
     self.gtest_filter = gtest_filter or ''
     self.test_arguments = test_arguments
     self.timeout = timeout
-    self.rebaseline = rebaseline
     self.performance_test = performance_test
     self.cleanup_test_files = cleanup_test_files
     self.tool = tool
     self.log_dump_name = log_dump_name
     self.fast_and_loose = fast_and_loose
-    self.build_type = build_type
     self.in_webkit_checkout = in_webkit_checkout
-    self.tests = []
+    self.all_tests = []
     if not self.gtest_filter:
       # No filter has been specified, let's add all tests then.
-      self.tests, self.attached_devices = self._GetTests()
+      self.all_tests = self._GetAllEnabledTests()
+    self.tests = self.all_tests

[bulach, 2012/12/14 10:36:11]

what is the difference between tests and all_tests?
at the sharder level, it seems better to have "all_tests" rather than "tests",
I'm just not sure it needs both..

also, and this is really important: when obtaining the tests, we'd drop out sick
devices... ilevy says it greatly reduced the number of flukes in the bots.
this patch changes that behavior... :(

[frankf, 2012/12/14 19:12:41]

self.tests get reassigned on retries (in base_test_sharder.py). We want the
original list to pass to LogFull. Take a look at the bogus TESTS_TO_RUN here:

http://master.chrome.corp.google.com:8000/builders/android/builds/594/steps/i...

On 2012/12/14 10:36:11, bulach wrote:

 
-  def _GetTests(self):
-    """Returns a tuple of (all_tests, available_devices).
+  def _GetAllEnabledTests(self):
+    """Returns a list of all enabled tests.
 
-    Tries to obtain the list of available tests.
+    Obtains a list of enabled tests from the test package on the device,
+    then filters it again using the diabled list on the host.
+
     Raises Exception if all devices failed.
     """
     available_devices = list(self.attached_devices)
     while available_devices:
       try:
-        logging.info('Obtaining tests from %s', available_devices[-1])
-        all_tests = self._GetTestsFromDevice(available_devices[-1])
-        return all_tests, available_devices

[bulach, 2012/12/14 10:36:11]

this was the really important bit... :)

[frankf, 2012/12/14 19:12:41]

As discussed offline, we need a better approach to device flakiness. We also
don't do it systematically by testing all devices, rather than breaking on the
first successull run.

Since this CL was doing too much, here's a partial revert here:
https://codereview.chromium.org/11573038/

On 2012/12/14 10:36:11, bulach wrote:

+        return self._GetTestsFromDevice(available_devices[-1])
       except Exception as e:
-        logging.info('Failed obtaining tests from %s %s',
-                     available_devices[-1], e)
+        logging.warning('Failed obtaining tests from %s %s',
+                        current_device, e)
         available_devices.pop()
+
     raise Exception('No device available to get the list of tests.')
 
   def _GetTestsFromDevice(self, device):
-    test = SingleTestRunner(device, self.test_suite, self.gtest_filter,
-                            self.test_arguments, self.timeout, self.rebaseline,
-                            self.performance_test, self.cleanup_test_files,
-                            self.tool, 0,
-                            not not self.log_dump_name, self.fast_and_loose,
-                            self.build_type, self.in_webkit_checkout)
+    logging.info('Obtaining tests from %s', device)
+    test_runner = SingleTestRunner(
+        device,
+        self.test_suite,
+        self.gtest_filter,
+        self.test_arguments,
+        self.timeout,
+        self.performance_test,
+        self.cleanup_test_files,
+        self.tool,
+        0,
+        not not self.log_dump_name,
+        self.fast_and_loose,
+        self.build_type,
+        self.in_webkit_checkout)
     # The executable/apk needs to be copied before we can call GetAllTests.
-    test.test_package.StripAndCopyExecutable()
-    all_tests = test.test_package.GetAllTests()
-    if not self.rebaseline:
-      disabled_list = test.GetDisabledTests()
-      # Only includes tests that do not have any match in the disabled list.
-      all_tests = filter(lambda t:
-                         not any([fnmatch.fnmatch(t, disabled_pattern)
-                                  for disabled_pattern in disabled_list]),
-                         all_tests)
+    test_runner.test_package.StripAndCopyExecutable()
+    all_tests = test_runner.test_package.GetAllTests()
+    disabled_list = test_runner.GetDisabledTests()
+    # Only includes tests that do not have any match in the disabled list.
+    all_tests = filter(lambda t:
+                       not any([fnmatch.fnmatch(t, disabled_pattern)
+                                for disabled_pattern in disabled_list]),
+                       all_tests)
     return all_tests
 
   def CreateShardedTestRunner(self, device, index):
     """Creates a suite-specific test runner.
 
     Args:
       device: Device serial where this shard will run.
       index: Index of this device in the pool.
 
     Returns:
       A SingleTestRunner object.
     """
     device_num = len(self.attached_devices)
     shard_size = (len(self.tests) + device_num - 1) / device_num
     shard_test_list = self.tests[index * shard_size : (index + 1) * shard_size]
     test_filter = ':'.join(shard_test_list) + self.gtest_filter
-    return SingleTestRunner(device, self.test_suite,
-                            test_filter, self.test_arguments, self.timeout,
-                            self.rebaseline, self.performance_test,
-                            self.cleanup_test_files, self.tool, index,
-                            not not self.log_dump_name, self.fast_and_loose,
-                            self.build_type, self.in_webkit_checkout)
+    return SingleTestRunner(
+        device,
+        self.test_suite,
+        test_filter,
+        self.test_arguments,
+        self.timeout,
+        self.performance_test,
+        self.cleanup_test_files, self.tool, index,
+        not not self.log_dump_name,
+        self.fast_and_loose,
+        self.build_type,
+        self.in_webkit_checkout)
 
   def OnTestsCompleted(self, test_runners, test_results):
     """Notifies that we completed the tests."""
     test_results.LogFull('Unit test', os.path.basename(self.test_suite),
-                         self.build_type, self.tests)
+                         self.build_type, self.all_tests)
     test_results.PrintAnnotation()
-    if test_results.failed and self.rebaseline:
-      test_runners[0].UpdateFilter(test_results.failed)
     if self.log_dump_name:
       # Zip all debug info outputs into a file named by log_dump_name.
       debug_info.GTestDebugInfo.ZipAndCleanResults(
-          os.path.join(cmd_helper.OutDirectory.get(), self.build_type,
+          os.path.join(
+              cmd_helper.OutDirectory.get(), self.build_type,
               'debug_info_dumps'),
           self.log_dump_name)
 
 
 def _RunATestSuite(options):
   """Run a single test suite.
 
   Helper for Dispatch() to allow stop/restart of the emulator across
   test bundles.  If using the emulator, we start it on entry and stop
   it on exit.
 
   Args:
     options: options for running the tests.
 
   Returns:
     0 if successful, number of failing tests otherwise.
   """
   step_name = os.path.basename(options.test_suite).replace('-debug.apk', '')
   buildbot_report.PrintNamedStep(step_name)
   attached_devices = []
   buildbot_emulators = []
 
   if options.use_emulator:
     for n in range(options.emulator_count):
       t = TimeProfile('Emulator launch %d' % n)
-      avd_name =  None
+      avd_name = None
       if n > 0:
         # Creates a temporary AVD for the extra emulators.
         avd_name = 'run_tests_avd_%d' % n
       buildbot_emulator = emulator.Emulator(avd_name, options.fast_and_loose)
       buildbot_emulator.Launch(kill_all_emulators=n == 0)
       t.Stop()
       buildbot_emulators.append(buildbot_emulator)
       attached_devices.append(buildbot_emulator.device)
     # Wait for all emulators to boot completed.
     map(lambda buildbot_emulator: buildbot_emulator.ConfirmLaunch(True),
         buildbot_emulators)
   elif options.test_device:
     attached_devices = [options.test_device]
   else:
     attached_devices = android_commands.GetAttachedDevices()
 
   if not attached_devices:
     logging.critical('A device must be attached and online.')
     buildbot_report.PrintError()
     return 1
 
   # Reset the test port allocation. It's important to do it before starting
   # to dispatch any tests.
   if not ports.ResetTestServerPortAllocation():
     raise Exception('Failed to reset test server port.')
 
   if options.performance_test or options.gtest_filter:
-    # These configuration can't be split in multiple devices.
+    logging.warning('Sharding is not possible with these configurations.')
     attached_devices = [attached_devices[0]]
-  sharder = TestSharder(attached_devices, options.test_suite,
-                        options.gtest_filter, options.test_arguments,
-                        options.timeout, options.rebaseline,
-                        options.performance_test,
-                        options.cleanup_test_files, options.tool,
-                        options.log_dump, options.fast_and_loose,
-                        options.build_type, options.webkit)
+
+  sharder = TestSharder(
+      attached_devices,
+      options.test_suite,
+      options.gtest_filter,
+      options.test_arguments,
+      options.timeout,
+      options.performance_test,
+      options.cleanup_test_files,
+      options.tool,
+      options.log_dump,
+      options.fast_and_loose,
+      options.build_type,
+      options.webkit)
   test_results = sharder.RunShardedTests()
 
   for buildbot_emulator in buildbot_emulators:
     buildbot_emulator.Shutdown()
 
   return len(test_results.failed)
 
 
 def Dispatch(options):
   """Dispatches the tests, sharding if possible.
@@ skipping 40 matching lines @@
   option_parser = optparse.OptionParser()
   test_options_parser.AddTestRunnerOptions(option_parser, default_timeout=0)
   option_parser.add_option('-s', '--suite', dest='test_suite',
                            help='Executable name of the test suite to run '
                            '(use -s help to list them)')
   option_parser.add_option('--out-directory', dest='out_directory',
                            help='Path to the out/ directory, irrespective of '
                            'the build type. Only for non-Chromium uses.')
   option_parser.add_option('-d', '--device', dest='test_device',
                            help='Target device the test suite to run ')
-  option_parser.add_option('-r', dest='rebaseline',
-                           help='Rebaseline and update *testsuite_disabled',
-                           action='store_true')
   option_parser.add_option('-f', '--gtest_filter', dest='gtest_filter',
                            help='gtest filter')
   option_parser.add_option('-a', '--test_arguments', dest='test_arguments',
                            help='Additional arguments to pass to the test')
   option_parser.add_option('-p', dest='performance_test',
                            help='Indicator of performance test',
                            action='store_true')
   option_parser.add_option('-L', dest='log_dump',
                            help='file name of log dump, which will be put in '
                            'subfolder debug_info_dumps under the same '
@@ skipping 22 matching lines @@
                            default=2,
                            help='Repeat count on test timeout')
   option_parser.add_option('--exit_code', action='store_true',
                            help='If set, the exit code will be total number '
                            'of failures.')
   option_parser.add_option('--exe', action='store_true',
                            help='If set, use the exe test runner instead of '
                            'the APK.')
 
   options, args = option_parser.parse_args(argv)
+
   if len(args) > 1:
     print 'Unknown argument:', args[1:]
     option_parser.print_usage()
     sys.exit(1)
+
   run_tests_helper.SetLogLevel(options.verbose_count)
+
   if options.out_directory:
     cmd_helper.OutDirectory.set(options.out_directory)
-  emulator.DeleteAllTempAVDs()
+
+  if options.use_emulator:
+    emulator.DeleteAllTempAVDs()
+
   failed_tests_count = Dispatch(options)
 
   # Failures of individual test suites are communicated by printing a
   # STEP_FAILURE message.
   # Returning a success exit status also prevents the buildbot from incorrectly
   # marking the last suite as failed if there were failures in other suites in
   # the batch (this happens because the exit status is a sum of all failures
   # from all suites, but the buildbot associates the exit status only with the
   # most recent step).
   if options.exit_code:
     return failed_tests_count
   return 0
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))