Support xctests on simulators

ios/build/bots/scripts/test_runner.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Test runners for iOS."""
 
 import argparse
 import collections
 import errno
 import os
 import shutil
 import subprocess
 import sys
 import tempfile
 import time
 
 import find_xcode
 import gtest_utils
+import xctest_utils
 
 
 class Error(Exception):
   """Base class for errors."""
   pass
 
 
 class TestRunnerError(Error):
   """Base class for TestRunner-related errors."""
   pass
@@ skipping 11 matching lines @@
       'App does not exist: %s' % app_path)
 
 
 class DeviceDetectionError(TestRunnerError):
   """Unexpected number of devices detected."""
   def __init__(self, udids):
     super(DeviceDetectionError, self).__init__(
       'Expected one device, found %s:\n%s' % (len(udids), '\n'.join(udids)))
 
 
+class PlugInsNotFoundError(TestRunnerError):
+  """The PlugIns directory was not found."""
+  def __init__(self, plugins_dir):
+    super(PlugInsNotFoundError, self).__init__(
+      'PlugIns directory does not exist: %s' % plugins_dir)
+
+
 class SimulatorNotFoundError(TestRunnerError):
   """The given simulator binary was not found."""
   def __init__(self, iossim_path):
     super(SimulatorNotFoundError, self).__init__(
         'Simulator does not exist: %s' % iossim_path)
 
 
-class XcodeVersionNotFound(TestRunnerError):
+class XcodeVersionNotFoundError(TestRunnerError):
   """The requested version of Xcode was not found."""
   def __init__(self, xcode_version):
     super(XcodeVersionNotFoundError, self).__init__(
         'Xcode version not found: %s', xcode_version)
 
 
+class XCTestPlugInNotFoundError(TestRunnerError):
+  """The .xctest PlugIn was not found."""
+  def __init__(self, xctest_path):
+    super(XCTestPlugInNotFoundError, self).__init__(
+        'XCTest not found: %s', xctest_path)
+
+
 def get_kif_test_filter(tests, invert=False):
   """Returns the KIF test filter to filter the given test cases.
 
   Args:
     tests: List of test cases to filter.
     invert: Whether to invert the filter or not. Inverted, the filter will match
       everything except the given test cases.
 
   Returns:
     A string which can be supplied to GKIF_SCENARIO_FILTER.
@@ skipping 24 matching lines @@
   test_filter = ':'.join(test for test in tests)
   if invert:
     return '-%s' % test_filter
   return test_filter
 
 
 class TestRunner(object):
   """Base class containing common functionality."""
 
   def __init__(
-    self, app_path, xcode_version, out_dir, env_vars=None, test_args=None):
+    self,
+    app_path,
+    xcode_version,
+    out_dir,
+    env_vars=None,
+    test_args=None,
+    xctest=False,
+  ):
     """Initializes a new instance of this class.
 
     Args:
       app_path: Path to the compiled .app to run.
       xcode_version: Version of Xcode to use when running the test.
       out_dir: Directory to emit test data into.
       env_vars: List of environment variables to pass to the test itself.
       test_args: List of strings to pass as arguments to the test when
         launching.
+      xctest: Whether or not this is an XCTest.
 
     Raises:
       AppNotFoundError: If the given app does not exist.
+      PlugInsNotFoundError: If the PlugIns directory does not exist for XCTests.
       XcodeVersionNotFoundError: If the given Xcode version does not exist.
+      XCTestPlugInNotFoundError: If the .xctest PlugIn does not exist.
     """
     if not os.path.exists(app_path):
       raise AppNotFoundError(app_path)
 
     if not find_xcode.find_xcode(xcode_version)['found']:
       raise XcodeVersionNotFoundError(xcode_version)
 
     if not os.path.exists(out_dir):
       os.makedirs(out_dir)
 
     self.app_name = os.path.splitext(os.path.split(app_path)[-1])[0]
     self.app_path = app_path
     self.cfbundleid = subprocess.check_output([
         '/usr/libexec/PlistBuddy',
         '-c', 'Print:CFBundleIdentifier',
         os.path.join(app_path, 'Info.plist'),
     ]).rstrip()
     self.env_vars = env_vars or []
     self.logs = collections.OrderedDict()
     self.out_dir = out_dir
     self.test_args = test_args or []
     self.xcode_version = xcode_version
+    self.xctest_path = ''
+
+    if xctest:
+      plugins_dir = os.path.join(self.app_path, 'PlugIns')
+      if not os.path.exists(plugins_dir):
+        raise PlugInsNotFoundError(plugins_dir)
+      for plugin in os.listdir(plugins_dir):
+        if plugin.endswith('.xctest'):
+          self.xctest_path = os.path.join(plugins_dir, plugin)
+      if not os.path.exists(self.xctest_path):
+        raise XCTestPlugInNotFoundError(self.xctest_path)
 
   def get_launch_command(self, test_filter=None, invert=False):
     """Returns the command that can be used to launch the test app.
 
     Args:
       test_filter: List of test cases to filter.
       invert: Whether to invert the filter or not. Inverted, the filter will
         match everything except the given test cases.
 
     Returns:
       A list of strings forming the command to launch the test.
     """
     raise NotImplementedError
 
   def set_up(self):
     """Performs setup actions which must occur prior to every test launch."""
     raise NotImplementedError
 
   def tear_down(self):
     """Performs cleanup actions which must occur after every test launch."""
     raise NotImplementedError
 
   def screenshot_desktop(self):
     """Saves a screenshot of the desktop in the output directory."""
     subprocess.check_call([
         'screencapture',
         os.path.join(self.out_dir, 'desktop_%s.png' % time.time()),
     ])
 
-  @staticmethod
-  def _run(cmd):
+  def _run(self, cmd):
     """Runs the specified command, parsing GTest output.
 
     Args:
       cmd: List of strings forming the command to run.
 
     Returns:
       GTestResult instance.
     """
     print ' '.join(cmd)
     print
 
-    parser = gtest_utils.GTestLogParser()
     result = gtest_utils.GTestResult(cmd)
+    if self.xctest_path:
+      parser = xctest_utils.XCTestLogParser()
+    else:
+      parser = gtest_utils.GTestLogParser()
 
     proc = subprocess.Popen(
         cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 
     while True:
       line = proc.stdout.readline()
       if not line:
         break
       line = line.rstrip()
       parser.ProcessLine(line)
@@ skipping 33 matching lines @@
         result = self._run(cmd)
 
       if result.crashed and not result.crashed_test:
         raise AppLaunchError
 
       passed = result.passed_tests
       failed = result.failed_tests
       flaked = result.flaked_tests
 
       try:
-        while result.crashed and result.crashed_test:
+        # XCTests cannot currently be resumed at the next test case.
+        while not self.xctest_path and result.crashed and result.crashed_test:
           # If the app crashes during a specific test case, then resume at the
           # next test case. This is achieved by filtering out every test case
           # which has already run.
           print 'Crashed during %s, resuming...' % result.crashed_test
           print
           result = self._run(self.get_launch_command(
               test_filter=passed + failed.keys() + flaked.keys(), invert=True,
           ))
           passed.extend(result.passed_tests)
           failed.update(result.failed_tests)
@@ skipping 22 matching lines @@
   def __init__(
       self,
       app_path,
       iossim_path,
       platform,
       version,
       xcode_version,
       out_dir,
       env_vars=None,
       test_args=None,
+      xctest=False,
   ):
     """Initializes a new instance of this class.
 
     Args:
       app_path: Path to the compiled .app or .ipa to run.
       iossim_path: Path to the compiled iossim binary to use.
       platform: Name of the platform to simulate. Supported values can be found
         by running "iossim -l". e.g. "iPhone 5s", "iPad Retina".
       version: Version of iOS the platform should be running. Supported values
         can be found by running "iossim -l". e.g. "9.3", "8.2", "7.1".
       xcode_version: Version of Xcode to use when running the test.
       out_dir: Directory to emit test data into.
       env_vars: List of environment variables to pass to the test itself.
       test_args: List of strings to pass as arguments to the test when
         launching.
+      xctest: Whether or not this is an XCTest.
 
     Raises:
       AppNotFoundError: If the given app does not exist.
+      PlugInsNotFoundError: If the PlugIns directory does not exist for XCTests.
       XcodeVersionNotFoundError: If the given Xcode version does not exist.
+      XCTestPlugInNotFoundError: If the .xctest PlugIn does not exist.
     """
     super(SimulatorTestRunner, self).__init__(
         app_path,
         xcode_version,
         out_dir,
         env_vars=env_vars,
         test_args=test_args,
+        xctest=xctest,
     )
 
     if not os.path.exists(iossim_path):
       raise SimulatorNotFoundError(iossim_path)
 
     self.homedir = ''
     self.iossim_path = iossim_path
     self.platform = platform
     self.start_time = None
     self.version = version
@@ skipping 119 matching lines @@
 
       if self.xcode_version == '8.0':
         args.extend(['-c', '--gtest_filter=%s' % gtest_filter])
       else:
         args.append('--gtest_filter=%s' % gtest_filter)
 
     for env_var in self.env_vars:
       cmd.extend(['-e', env_var])
 
     cmd.append(self.app_path)
+    if self.xctest_path:
+      cmd.append(self.xctest_path)
     cmd.extend(self.test_args)
     cmd.extend(args)
     return cmd
 
 
 class DeviceTestRunner(TestRunner):
   """Class for running tests on devices."""
 
   def __init__(
     self, app_path, xcode_version, out_dir, env_vars=None, test_args=None):
@@ skipping 77 matching lines @@
 
     for env_var in self.env_vars:
       cmd.extend(['-D', env_var])
 
     if args or self.test_args:
       cmd.append('--args')
       cmd.extend(self.test_args)
       cmd.extend(args)
 
     return cmd