Add iOS device test runner

ios/build/bots/scripts/test_runner.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Test runners for iOS."""
 
 import argparse
 import collections
+import errno
 import os
 import shutil
 import subprocess
 import sys
 import tempfile
 import time
 
 import find_xcode
 import gtest_utils
 
@@ skipping 10 matching lines @@
 
 class AppLaunchError(TestRunnerError):
   """The app failed to launch."""
   pass
 
 
 class AppNotFoundError(TestRunnerError):
   """The requested app was not found."""
   def __init__(self, app_path):
     super(AppNotFoundError, self).__init__(
-        'App does not exist: %s' % app_path)
+      'App does not exist: %s' % app_path)
+
+
+class DeviceDetectionError(TestRunnerError):
+  """Unexpected number of devices detected."""
+  def __init__(self, udids):
+    super(DeviceDetectionError, self).__init__(
+      'Expected one device, found %s:\n%s' % (len(udids), '\n'.join(udids)))
 
 
 class SimulatorNotFoundError(TestRunnerError):
   """The given simulator binary was not found."""
   def __init__(self, iossim_path):
     super(SimulatorNotFoundError, self).__init__(
         'Simulator does not exist: %s' % iossim_path)
 
 
 class XcodeVersionNotFound(TestRunnerError):
@@ skipping 39 matching lines @@
   # e.g. -a:b:c matches everything except a, b, c.
   test_filter = ':'.join(test for test in tests)
   if invert:
     return '-%s' % test_filter
   return test_filter
 
 
 class TestRunner(object):
   """Base class containing common functionality."""
 
-  def __init__(self, app_path, xcode_version, out_dir, test_args=None):
+  def __init__(
+    self, app_path, xcode_version, out_dir, env_vars=None, test_args=None):
     """Initializes a new instance of this class.
 
     Args:
       app_path: Path to the compiled .app to run.
       xcode_version: Version of Xcode to use when running the test.
       out_dir: Directory to emit test data into.
+      env_vars: List of environment variables to pass to the test itself.
       test_args: List of strings to pass as arguments to the test when
         launching.
 
     Raises:
       AppNotFoundError: If the given app does not exist.
       XcodeVersionNotFoundError: If the given Xcode version does not exist.
     """
     if not os.path.exists(app_path):
       raise AppNotFoundError(app_path)
 
     if not find_xcode.find_xcode(xcode_version)['found']:
       raise XcodeVersionNotFoundError(xcode_version)
 
     if not os.path.exists(out_dir):
       os.makedirs(out_dir)
 
     self.app_name = os.path.splitext(os.path.split(app_path)[-1])[0]
     self.app_path = app_path
     self.cfbundleid = subprocess.check_output([
         '/usr/libexec/PlistBuddy',
         '-c', 'Print:CFBundleIdentifier',
         os.path.join(app_path, 'Info.plist'),
     ]).rstrip()
+    self.env_vars = env_vars or []
     self.logs = collections.OrderedDict()
     self.out_dir = out_dir
     self.test_args = test_args or []
+    self.xcode_version = xcode_version
 
   def get_launch_command(self, test_filter=None, invert=False):
     """Returns the command that can be used to launch the test app.
 
     Args:
       test_filter: List of test cases to filter.
       invert: Whether to invert the filter or not. Inverted, the filter will
         match everything except the given test cases.
 
     Returns:
@@ skipping 118 matching lines @@
   """Class for running tests on iossim."""
 
   def __init__(
       self,
       app_path,
       iossim_path,
       platform,
       version,
       xcode_version,
       out_dir,
+      env_vars=None,
       test_args=None,
   ):
     """Initializes a new instance of this class.
 
     Args:
       app_path: Path to the compiled .app or .ipa to run.
       iossim_path: Path to the compiled iossim binary to use.
       platform: Name of the platform to simulate. Supported values can be found
         by running "iossim -l". e.g. "iPhone 5s", "iPad Retina".
       version: Version of iOS the platform should be running. Supported values
         can be found by running "iossim -l". e.g. "9.3", "8.2", "7.1".
       xcode_version: Version of Xcode to use when running the test.
       out_dir: Directory to emit test data into.
+      env_vars: List of environment variables to pass to the test itself.
       test_args: List of strings to pass as arguments to the test when
         launching.
 
     Raises:
       AppNotFoundError: If the given app does not exist.
       XcodeVersionNotFoundError: If the given Xcode version does not exist.
     """
     super(SimulatorTestRunner, self).__init__(
-        app_path, xcode_version, out_dir, test_args=test_args)
+        app_path,
+        xcode_version,
+        out_dir,
+        env_vars=env_vars,
+        test_args=test_args,
+    )
 
     if not os.path.exists(iossim_path):
       raise SimulatorNotFoundError(iossim_path)
 
     self.homedir = ''
     self.iossim_path = iossim_path
     self.platform = platform
     self.start_time = None
     self.version = version
 
@@ skipping 108 matching lines @@
         '-s', self.version,
         '-t', '120',
         '-u', self.homedir,
     ]
     args = []
 
     if test_filter:
       kif_filter = get_kif_test_filter(test_filter, invert=invert)
       gtest_filter = get_gtest_filter(test_filter, invert=invert)
       cmd.extend(['-e', 'GKIF_SCENARIO_FILTER=%s' % kif_filter])
-      args.append('--gtest_filter=%s' % gtest_filter)
+
+      if self.xcode_version == '8.0':
+        args.extend(['-c', '--gtest_filter=%s' % gtest_filter])
+      else:
+        args.append('--gtest_filter=%s' % gtest_filter)
+
+    for env_var in self.env_vars:
+      cmd.extend(['-e', env_var])
 
     cmd.append(self.app_path)
     cmd.extend(self.test_args)
     cmd.extend(args)
     return cmd
+
+
+class DeviceTestRunner(TestRunner):
+  """Class for running tests on devices."""
+
+  def __init__(
+    self, app_path, xcode_version, out_dir, env_vars=None, test_args=None):
+    """Initializes a new instance of this class.
+
+    Args:
+      app_path: Path to the compiled .app to run.
+      xcode_version: Version of Xcode to use when running the test.
+      out_dir: Directory to emit test data into.
+      env_vars: List of environment variables to pass to the test itself.
+      test_args: List of strings to pass as arguments to the test when
+        launching.
+
+    Raises:
+      AppNotFoundError: If the given app does not exist.
+      XcodeVersionNotFoundError: If the given Xcode version does not exist.
+    """
+    super(DeviceTestRunner, self).__init__(
+      app_path, xcode_version, out_dir, env_vars=env_vars, test_args=test_args)
+
+    self.udid = subprocess.check_output(['idevice_id', '--list']).rstrip()
+    if len(self.udid.splitlines()) != 1:
+      raise DeviceDetectionError(self.udid)
+
+  def uninstall_apps(self):
+    """Uninstalls all apps found on the device."""
+    for app in subprocess.check_output(
+      ['idevicefs', '--udid', self.udid, 'ls', '@']).splitlines():
+      subprocess.check_call(
+        ['ideviceinstaller', '--udid', self.udid, '--uninstall', app])
+
+  def install_app(self):
+    """Installs the app."""
+    subprocess.check_call(
+      ['ideviceinstaller', '--udid', self.udid, '--install', self.app_path])
+
+  def set_up(self):
+    """Performs setup actions which must occur prior to every test launch."""
+    self.uninstall_apps()
+    self.install_app()
+
+  def extract_test_data(self):
+    """Extracts data emitted by the test."""
+    subprocess.check_call([
+      'idevicefs',
+      '--udid', self.udid,
+      'pull',
+      '@%s/Documents' % self.cfbundleid,
+      os.path.join(self.out_dir, 'Documents'),
+    ])
+
+  def tear_down(self):
+    """Performs cleanup actions which must occur after every test launch."""
+    self.extract_test_data()
+    self.screenshot_desktop()
+    self.uninstall_apps()
+
+  def get_launch_command(self, test_filter=None, invert=False):
+    """Returns the command that can be used to launch the test app.
+
+    Args:
+      test_filter: List of test cases to filter.
+      invert: Whether to invert the filter or not. Inverted, the filter will
+        match everything except the given test cases.
+
+    Returns:
+      A list of strings forming the command to launch the test.
+    """
+    cmd = [
+      'idevice-app-runner',
+      '--udid', self.udid,
+      '--start', self.cfbundleid,
+    ]
+    args = []
+
+    if test_filter:
+      kif_filter = get_kif_test_filter(test_filter, invert=invert)
+      gtest_filter = get_gtest_filter(test_filter, invert=invert)
+      cmd.extend(['-D', 'GKIF_SCENARIO_FILTER=%s' % kif_filter])
+      args.append('--gtest-filter=%s' % gtest_filter)
+
+    for env_var in self.env_vars:
+      cmd.extend(['-D', env_var])
+
+    if args or self.test_args:
+      cmd.append('--args')
+      cmd.extend(self.test_args)
+      cmd.extend(args)
+
+    return cmd