Use iossim/test-without-building for Earlgrey tests.

scripts/slave/ios/test_runner.py

 #!/usr/bin/python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Contains test runners for launching tests on simulators and devices."""
 
 # pylint: disable=relative-import
 import environment_setup
 
@@ skipping 73 matching lines @@
       'Unexpected app path: %s. %s' % (app_path, valid_extensions))
 
 
 class SimulatorNotFoundError(TestRunnerError):
   """The iossim path was not found."""
   def __init__(self, iossim_path):
     super(SimulatorNotFoundError, self).__init__(
       'Simulator does not exist: %s.' % iossim_path)
 
 
+class XctestNotFoundError(TestRunnerError):
+  """The xctest intended to be run was not found."""
+  def __init__(self, xctest_path):
+    super(XctestNotFoundError, self).__init__(
+      'Xctest does not exist: %s.' % xctest_path)
+
+
 class AppLaunchError(TestRunnerError):
   """There was an error launching the app."""
   pass
 
 
 class TestRunner(object):
   """Base class containing common TestRunner functionality."""
   def __init__(
     self,
     app_path,
@@ skipping 675 matching lines @@
       result = self._Run(self.GetLaunchCommand(), *args, **kwargs)
 
     return self.RunAllTests(result, *args, **kwargs)
 
 
 class XCTestRunner(TestRunner):
   """Base class containing common functionalities to run xctests."""
   def __init__(
     self,
     app_path,
-    test_host,
-    test_project_dir,
     xcode_version=None,
     gs_bucket=None,
     perf_bot_name=None,
     perf_build_number=None,
     perf_builder_name=None,
     perf_master_name=None,
     perf_revision=None,
     perf_x_value=None,
     test_args=None,
     env_vars=None,
   ):
     """Initializes an instance of the SimulatorXCTestRunner class.
 
     Args:
       app_path: Full path to the compiled app to run.
-      test_host: Name of the compiled test host app to run tests.
-      test_project_dir: Directory of the dummy test project.
       xcode_version: Version of Xcode to use.
       gs_bucket: Google Storage bucket to upload test data to, or None if the
       test data should not be uploaded.
       perf_bot_name: Name of this bot as indicated to the perf dashboard.
       perf_build_number: Build number to indicate to the perf dashboard.
       perf_builder_name: Name of this builder as indicated to the perf
       dashboard.
       perf_master_name: Name of the master as indicated to the perf dashboard.
       perf_revision: Revision to indicate to the perf dashboard.
       perf_x_value: Value to use on the x axis for all data uploaded to the
@@ skipping 12 matching lines @@
       perf_bot_name=perf_bot_name,
       perf_build_number=perf_build_number,
       perf_builder_name=perf_builder_name,
       perf_master_name=perf_master_name,
       perf_revision=perf_revision,
       perf_x_value=perf_x_value,
       test_args=test_args,
       xcode_version=xcode_version,
     )
     self.app_path = os.path.abspath(app_path)
-    self.test_host_name = test_host
-    # Test target name is its host name without '_host' suffix.
-    self.test_target_name = test_host.rsplit('_', 1)[0]
-    self.test_project_dir = test_project_dir
     self.timeout = '120'
     self.homedir = ''
     self.start_time = None
 
   def TearDown(self):
     """Performs post-test tear down."""
     raise NotImplementedError
 
   def HandleJsonFileWithPath(self, summary):
     """Parse data in summarydir and send to perf dashboard."""
     with open(summary) as jsonFile:
       return json.load(jsonFile)
 
   def GetLaunchEnvironment(self):
     """Returns the environment which is used to run the xctest.
     """
-    env = dict(os.environ, APP_TARGET_NAME=self.test_host_name,
-               TEST_TARGET_NAME=self.test_target_name,
+    env = dict(os.environ,
                NSUnbufferedIO='YES')
     return env
 
   def GetLaunchCommand(self, test_filter=None, blacklist=False):
     """Returns the command which is used to launch the test.
 
     Args:
       test_filter: A list of tests to filter by, or None to mean all.
       blacklist: Whether to blacklist the elements of test_filter or not. Only
       works when test_filter is not None.
@@ skipping 141 matching lines @@
     # At this point, all the tests have run, so used failed_tests to determine
     # the success/failure.
     return not failed_tests
 
 
 class SimulatorXCTestRunner(XCTestRunner):
   """Class for running xctests on an iOS simulator."""
   def __init__(
     self,
     app_path,
-    test_host,
-    test_project_dir,
+    xctest_path,
+    iossim_path,
     platform,
     version,
     xcode_version=None,
     gs_bucket=None,
     perf_bot_name=None,
     perf_build_number=None,
     perf_builder_name=None,
     perf_master_name=None,
     perf_revision=None,
     perf_x_value=None,
     test_args=None,
     env_vars=None,
   ):
     """Initializes an instance of the SimulatorXCTestRunner class.
 
     Args:
       app_path: Full path to the compiled app to run.
-      test_host: Name of the compiled test host app to run tests.
-      test_project_dir: Directory of the dummy test project.
+      iossim_path: Full path to the iossim executable to launch.
+      xctest_path: Full path to the compiled test bundle.
       platform: The platform to simulate. Supported values can be found by
       running 'xcodebuild -list'. e.g. 'iPhone 5', 'iPhone 5s'.
       version: The iOS version the simulator should be running. Supported values
       can be found by running 'xcodebuild -list'. e.g. '8.0', '7.1'.
       xcode_version: Version of Xcode to use.
       gs_bucket: Google Storage bucket to upload test data to, or None if the
       test data should not be uploaded.
       perf_bot_name: Name of this bot as indicated to the perf dashboard.
       perf_build_number: Build number to indicate to the perf dashboard.
       perf_builder_name: Name of this builder as indicated to the perf
       dashboard.
       perf_master_name: Name of the master as indicated to the perf dashboard.
       perf_revision: Revision to indicate to the perf dashboard.
       perf_x_value: Value to use on the x axis for all data uploaded to the
       perf dashboard.
       test_args: Arguments to pass when launching the test.
       env_vars: Environment variables to set when launching the test.
 
     Raises:
       SimulatorNotFoundError: If the given iossim path cannot be found.
     """
     super(SimulatorXCTestRunner, self).__init__(
       app_path,
-      test_host,
-      test_project_dir,
       env_vars=env_vars,
       gs_bucket=gs_bucket,
       perf_bot_name=perf_bot_name,
       perf_build_number=perf_build_number,
       perf_builder_name=perf_builder_name,
       perf_master_name=perf_master_name,
       perf_revision=perf_revision,
       perf_x_value=perf_x_value,
       test_args=test_args,
       xcode_version=xcode_version,
     )
+
+    if not os.path.exists(iossim_path):
+      raise SimulatorNotFoundError(iossim_path)
+
+    if not os.path.exists(xctest_path):
+      raise XctestNotFoundError(xctest_path)
+
+    self.iossim_path = iossim_path
+    self.xctest_path = xctest_path
     self.platform = platform
     self.version = version
 
   def UploadTestData(self):
     """Uploads the contents of the test's Documents directory.
 
     Returns:
       True if test data was uploaded, False otherwise.
     """
     if not self.gs_bucket:
@@ skipping 168 matching lines @@
     """Returns the invocation command which is used to run the test.
 
     Args:
       test_filter: A list of tests to filter by, or None to mean all.
       blacklist: Whether to blacklist the elements of test_filter or not. Only
       works when test_filter is not None.
 
     Returns:
       A list whose elements are the args representing the command.
     """
-    built_dir = os.path.split(self.app_path)[0]
-
     cmd = [
-      'xcodebuild', 'test-without-building',
-      'BUILT_PRODUCTS_DIR=%s' % built_dir,
-      '-project', self.test_project_dir,
-      '-scheme','TestProject',
-      '-destination','platform=iOS Simulator,name=%s,OS=%s'
-      % (self.platform, self.version),
-      '-archivePath', self.homedir,
-      'APP_TARGET_NAME=%s' % self.test_host_name,
-      'TEST_TARGET_NAME=%s' % self.test_target_name,
-      'NSUnbufferedIO=YES'
+      self.iossim_path,
+      '-d', self.platform,
+      '-s', self.version,
+      self.app_path,
+      self.xctest_path
     ]
     return cmd
 
   @TestRunner.RequireTearDown
   def Launch(self, *args, **kwargs):
     """Launches the test."""
     self.SetUp()
 
     result = self._Run(
       self.GetLaunchCommand(), self.GetLaunchEnvironment(), *args, **kwargs)
@@ skipping 15 matching lines @@
         self.GetLaunchCommand(), self.GetLaunchEnvironment(), *args, **kwargs)
 
     return self.RunAllTests(result, *args, **kwargs)
 
 
 class DeviceXCTestRunner(XCTestRunner):
   """Class for running xctests on an iOS device."""
   def __init__(
     self,
     app_path,
-    test_host,
-    test_project_dir,
+    xctest_path,
+    xcodeproj_path,
     xcode_version=None,
     gs_bucket=None,
     perf_bot_name=None,
     perf_build_number=None,
     perf_builder_name=None,
     perf_master_name=None,
     perf_revision=None,
     perf_x_value=None,
     test_args=None,
     env_vars=None,
@@ skipping 19 matching lines @@
       env_vars: Environment variables to set when launching the test.
 
     Raises:
       DeviceDetectionError: If this machine does not have exactly one device
         connected. Having more than one device connected causes problems when
         trying to issue commands to any one device, which interfere with
         installing and running the test app.
     """
     super(DeviceXCTestRunner, self).__init__(
       app_path,
-      test_host,
-      test_project_dir,
       env_vars=env_vars,
       gs_bucket=gs_bucket,
       perf_bot_name=perf_bot_name,
       perf_build_number=perf_build_number,
       perf_builder_name=perf_builder_name,
       perf_master_name=perf_master_name,
       perf_revision=perf_revision,
       perf_x_value=perf_x_value,
       test_args=test_args,
       xcode_version=xcode_version,
     )
+    # The scheme name is the app name without the '.app' suffix.
+    app_name = os.path.split(self.app_path)[1]
+    self.scheme_name = app_name.split('.', 1)[0]
+    self.xcodeproj_path = xcodeproj_path
     self.cfbundleid = utils.call(
       utils.PLIST_BUDDY,
       '-c', 'Print:CFBundleIdentifier',
       os.path.join(self.app_path, 'Info.plist'),
     ).stdout[0]
 
     call_result = utils.call('idevice_id', '--list')
     self.device_id = call_result.stdout[0]
 
     if len(call_result.stdout) != 1:
@@ skipping 40 matching lines @@
     """Returns the invocation command which is used to run the test.
 
     Args:
       test_filter: A list of tests to filter by, or None to mean all.
       blacklist: Whether to blacklist the elements of test_filter or not. Only
       works when test_filter is not None.
 
     Returns:
       A list whose elements are the args representing the command.
     """
-    built_dir = os.path.split(self.app_path)[0]
-
     cmd = [
       'xcodebuild', 'test-without-building',
-      'BUILT_PRODUCTS_DIR=%s' % built_dir,
-      'CONFIGURATION_BUILD_DIR=%s' % built_dir,
-      '-project', self.test_project_dir,
-      '-configuration', 'iphoneos',
-      '-scheme', 'TestProject',
-      '-destination','id=%s' % self.device_id,
-      'APP_TARGET_NAME=%s' % self.test_host_name,
-      'TEST_TARGET_NAME=%s' % self.test_target_name,
-      'NSUnbufferedIO=YES'
+      '-project', self.xcodeproj_path,
+      '-scheme', self.scheme_name,
+      '-destination', 'id=%s' % self.device_id
     ]
     return cmd
 
   @TestRunner.RequireTearDown
   def Launch(self, *args, **kwargs):
     """Launches the test."""
     self.InstallApp()
 
     result = self._Run(
       self.GetLaunchCommand(), self.GetLaunchEnvironment(), *args, **kwargs)
 
     if result.crashed and not result.crashed_test:
       # If the app crashed, but there is no specific test which crashed,
       # then the app must have failed to even start. Try one more time.
       self.Print(
         '%s appears to have crashed on startup. Retrying...' % self.app_name,
         blank_lines=2,
         time_to_sleep=5,
       )
 
       # Uninstall and re-install the app.
       self.UninstallApp()
       self.InstallApp()
 
       result = self._Run(
         self.GetLaunchCommand(), self.GetLaunchEnvironment(), *args, **kwargs)
 
     return self.RunAllTests(result, *args, **kwargs)