Upstream: Test scripts for Android (phase 2)

build/android/run_tests.py

+#!/usr/bin/python
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Runs all the native unit tests.
+
+1. Copy over test binary to /data/local on device.
+2. Resources: chrome/unit_tests requires resources (chrome.pak and en-US.pak)
+   to be deployed to the device (in /data/local/tmp).
+3. Environment:
+3.1. chrome/unit_tests requires (via chrome_paths.cc) a directory named:
+     /data/local/tmp/chrome/test/data
+3.2. page_cycler_tests have following requirements,
+3.2.1  the following data on host:
+       <chrome_src_dir>/tools/page_cycler
+       <chrome_src_dir>/data/page_cycler
+3.2.2. two data directories to store above test data on device named:
+       /data/local/tmp/tools/ (for database perf test)
+       /data/local/tmp/data/ (for other perf tests)
+3.2.3. a http server to serve http perf tests.
+       The http root is host's <chrome_src_dir>/data/page_cycler/, port 8000.
+3.2.4  a tool named forwarder is also required to run on device to
+       forward the http request/response between host and device.
+3.2.5  Chrome is installed on device.
+4. Run the binary in the device and stream the log to the host.
+4.1. Optionally, filter specific tests.
+4.2. Optionally, rebaseline: run the available tests and update the
+     suppressions file for failures.
+4.3. If we're running a single test suite and we have multiple devices
+     connected, we'll shard the tests.
+5. Clean up the device.
+
+Suppressions:
+
+Individual tests in a test binary can be suppressed by listing it in
+the gtest_filter directory in a file of the same name as the test binary,
+one test per line. Here is an example:
+
+  $ cat gtest_filter/base_unittests_disabled
+  DataPackTest.Load
+  ReadOnlyFileUtilTest.ContentsEqual
+
+This file is generated by the tests running on devices. If running on emulator,
+additonal filter file which lists the tests only failed in emulator will be
+loaded. We don't care about the rare testcases which succeeded on emuatlor, but
+failed on device.
+"""
+
+import logging
+import multiprocessing
+import os
+import re
+import subprocess
+import sys
+
+import android_commands
+from base_test_sharder import *

[Nirnimesh, 2011/10/21 08:16:15]

do not import *

[michaelbai, 2011/10/21 21:08:41]

Done.

+import cmd_helper
+import debug_info
+import emulator
+from run_tests_helper import *
+from single_test_runner import *
+from test_package_executable import TestPackageExecutable
+from test_result import *
+
+_TEST_SUITES = ['base_unittests',
+                ]

[Nirnimesh, 2011/10/21 08:16:15]

align under _

[michaelbai, 2011/10/21 21:08:41]

Done.

+
+def RunTests(device, test_suite, gtest_filter, test_arguments, rebaseline,
+             timeout, performance_test, cleanup_test_files, tool,
+             log_dump_name):
+  """Runs the tests.
+
+  Args:
+    device: Device to run the tests.
+    test_suite: A specific test suite to run, empty to run all.
+    gtest_filter: A gtest_filter flag.
+    test_arguments: Additional arguments to pass to the test binary.
+    rebaseline: Whether or not to run tests in isolation and update the filter.
+    timeout: Timeout for each test.
+    performance_test: Whether or not performance test(s).
+    cleanup_test_files: Whether or not to cleanup test files on device.
+    tool: Name of the Valgrind tool.
+    log_dump_name: Name of log dump file.
+
+  Returns:
+    A TestResults object.
+  """
+  results = []
+
+  if test_suite:
+    global _TEST_SUITES
+    if not os.path.exists(test_suite):
+      logging.critical('Unrecognized test suite, supported: %s' %
+                       _TEST_SUITES)
+      if test_suite in _TEST_SUITES:
+        logging.critical('(Remember to include the path: out/Release/%s)',
+                         test_suite)
+      return TestResults.FromOkAndFailed([], [BaseTestResult(test_suite, '')])
+    _TEST_SUITES = [test_suite]
+  else:
+    # If not specified, assume the test suites are in out/Release
+    test_suite_dir = os.path.abspath(os.path.join(CHROME_DIR, 'out', 'Release'))
+    _TEST_SUITES = [os.path.join(test_suite_dir, t) for t in _TEST_SUITES]
+  debug_info_list = []
+  for t in _TEST_SUITES:
+    test = SingleTestRunner(device, t, gtest_filter, test_arguments,
+                            timeout, rebaseline, performance_test,
+                            cleanup_test_files, tool, not not log_dump_name)
+    test.RunTests()
+    results += [test.test_results]
+    # Collect debug info.
+    debug_info_list += [test.dump_debug_info]
+    if rebaseline:
+      test.UpdateFilter(test.test_results.failed)
+    elif test.test_results.failed:
+      # Stop running test if encountering failed test.
+      test.test_results.LogFull()
+      break
+  # Zip all debug info outputs into a file named by log_dump_name.
+  debug_info.GTestDebugInfo.ZipAndCleanResults(
+      os.path.join(CHROME_DIR, 'out', 'Release', 'debug_info_dumps'),
+      log_dump_name, [d for d in debug_info_list if d])
+  return TestResults.FromTestResults(results)
+
+
+class TestSharder(BaseTestSharder):
+  """Responsible for sharding the tests on the connected devices."""
+
+  def __init__(self, attached_devices, test_suite, gtest_filter,
+               test_arguments, timeout, rebaseline, performance_test,
+               cleanup_test_files, tool):
+    BaseTestSharder.__init__(self, attached_devices)
+    self.test_suite = test_suite
+    self.test_suite_basename = os.path.basename(test_suite)
+    self.gtest_filter = gtest_filter
+    self.test_arguments = test_arguments
+    self.timeout = timeout
+    self.rebaseline = rebaseline
+    self.performance_test = performance_test
+    self.cleanup_test_files = cleanup_test_files
+    self.tool = tool
+    test = SingleTestRunner(self.attached_devices[0], test_suite, gtest_filter,
+                            test_arguments, timeout, rebaseline,
+                            performance_test, cleanup_test_files, tool)
+    all_tests = set(test.test_package.GetAllTests())
+    if not rebaseline:
+      all_tests -= set(test.GetDisabledTests())
+    self.tests = list(all_tests)
+
+  def CreateShardedTestRunner(self, device, index):
+    """Creates a suite-specific test runner.
+
+    Args:
+      device: Device serial where this shard will run.
+      index: Index of this device in the pool.
+
+    Returns:
+      A SingleTestRunner object.
+    """
+    shard_size = len(self.tests) / len(self.attached_devices)
+    shard_test_list = self.tests[index * shard_size : (index + 1) * shard_size]
+    test_filter = ':'.join(shard_test_list)
+    return SingleTestRunner(device, self.test_suite,
+                            test_filter, self.test_arguments, self.timeout,
+                            self.rebaseline, self.performance_test,
+                            self.cleanup_test_files, self.tool)
+
+  def OnTestsCompleted(self, test_runners, test_results):
+    """Notifies that we completed the tests."""
+    test_results.LogFull()
+    if test_results.failed and self.rebaseline:
+      test_runners[0].UpdateFilter(test_results.failed)
+
+
+def Dispatch(options):
+  """Dispatches the tests, sharding if possible.
+
+  If options.use_emulator is True, all tests will be run in a new emulator
+  instance.
+
+  Args:
+    options: options for running the tests.
+
+  Returns:
+    0 if successful, number of failing tests otherwise.
+  """
+  if options.test_suite == 'help':
+    ListTestSuites()
+    return 0
+  buildbot_emulator = None
+  attached_devices = []
+
+  if options.use_emulator:
+    buildbot_emulator = emulator.Emulator()
+    buildbot_emulator.Launch()
+    attached_devices.append(buildbot_emulator.device)
+  else:
+    attached_devices = android_commands.GetAttachedDevices()
+
+  if not attached_devices:
+    logging.critical('A device must be attached and online.')
+    return 1
+
+  if (len(attached_devices) > 1 and options.test_suite and
+      not options.gtest_filter and not options.performance_test):
+    sharder = TestSharder(attached_devices, options.test_suite,
+                          options.gtest_filter, options.test_arguments,
+                          options.timeout, options.rebaseline,
+                          options.performance_test,
+                          options.cleanup_test_files, options.tool)
+    test_results = sharder.RunShardedTests()
+  else:
+    test_results = RunTests(attached_devices[0], options.test_suite,
+                            options.gtest_filter, options.test_arguments,
+                            options.rebaseline, options.timeout,
+                            options.performance_test,
+                            options.cleanup_test_files, options.tool,
+                            options.log_dump)
+  if buildbot_emulator:
+    buildbot_emulator.Shutdown()
+  return len(test_results.failed)
+
+def ListTestSuites():
+  """Display a list of available test suites
+  """
+  print 'Available test suites are:'
+  for test_suite in _TEST_SUITES:
+    print test_suite
+
+
+def main(argv):
+  option_parser = CreateTestRunnerOptionParser(None, default_timeout=0)
+  option_parser.add_option('-s', dest='test_suite',
+                           help='Executable name of the test suite to run '
+                           '(use -s help to list them)')
+  option_parser.add_option('-r', dest='rebaseline',
+                           help='Rebaseline and update *testsuite_disabled',
+                           action='store_true',
+                           default=False)
+  option_parser.add_option('-f', dest='gtest_filter',
+                           help='gtest filter')
+  option_parser.add_option('-a', '--test_arguments', dest='test_arguments',
+                           help='Additional arguments to pass to the test')
+  option_parser.add_option('-p', dest='performance_test',
+                           help='Indicator of performance test',
+                           action='store_true',
+                           default=False)
+  option_parser.add_option('-L', dest='log_dump',
+                           help='file name of log dump, which will be put in'
+                           'subfolder debug_info_dumps under the same directory'
+                           'in where the test_suite exists.')
+  option_parser.add_option('-e', '--emulator', dest='use_emulator',
+                           help='Run tests in a new instance of emulator',
+                           action='store_true',
+                           default=False)
+  options, args = option_parser.parse_args(argv)
+  if len(args) > 1:
+    print 'Unknown argument:', args[1:]
+    option_parser.print_usage()
+    sys.exit(1)
+  SetLogLevel(options.verbose_count)
+  return Dispatch(options)
+

[Nirnimesh, 2011/10/21 08:16:15]

nit: need another blank line here

[michaelbai, 2011/10/21 21:08:41]

Done.

+if __name__ == '__main__':
+  sys.exit(main(sys.argv))