Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(84)

Side by Side Diff: scripts/slave/ios/test_runner.py

Issue 2229183002: Use iossim/test-without-building for Earlgrey tests. (Closed) Base URL: https://chromium.googlesource.com/chromium/tools/build.git@master
Patch Set: Use iossim/test-without-building for Earlgrey tests. Created 4 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 #!/usr/bin/python 1 #!/usr/bin/python
2 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Copyright 2014 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Contains test runners for launching tests on simulators and devices.""" 6 """Contains test runners for launching tests on simulators and devices."""
7 7
8 # pylint: disable=relative-import 8 # pylint: disable=relative-import
9 import environment_setup 9 import environment_setup
10 10
(...skipping 73 matching lines...) Expand 10 before | Expand all | Expand 10 after
84 'Unexpected app path: %s. %s' % (app_path, valid_extensions)) 84 'Unexpected app path: %s. %s' % (app_path, valid_extensions))
85 85
86 86
87 class SimulatorNotFoundError(TestRunnerError): 87 class SimulatorNotFoundError(TestRunnerError):
88 """The iossim path was not found.""" 88 """The iossim path was not found."""
89 def __init__(self, iossim_path): 89 def __init__(self, iossim_path):
90 super(SimulatorNotFoundError, self).__init__( 90 super(SimulatorNotFoundError, self).__init__(
91 'Simulator does not exist: %s.' % iossim_path) 91 'Simulator does not exist: %s.' % iossim_path)
92 92
93 93
94 class XctestNotFoundError(TestRunnerError):
95 """The xctest intended to be run was not found."""
96 def __init__(self, xctest_path):
97 super(XctestNotFoundError, self).__init__(
98 'Xctest does not exist: %s.' % xctest_path)
99
100
94 class AppLaunchError(TestRunnerError): 101 class AppLaunchError(TestRunnerError):
95 """There was an error launching the app.""" 102 """There was an error launching the app."""
96 pass 103 pass
97 104
98 105
99 class TestRunner(object): 106 class TestRunner(object):
100 """Base class containing common TestRunner functionality.""" 107 """Base class containing common TestRunner functionality."""
101 def __init__( 108 def __init__(
102 self, 109 self,
103 app_path, 110 app_path,
(...skipping 675 matching lines...) Expand 10 before | Expand all | Expand 10 after
779 result = self._Run(self.GetLaunchCommand(), *args, **kwargs) 786 result = self._Run(self.GetLaunchCommand(), *args, **kwargs)
780 787
781 return self.RunAllTests(result, *args, **kwargs) 788 return self.RunAllTests(result, *args, **kwargs)
782 789
783 790
784 class XCTestRunner(TestRunner): 791 class XCTestRunner(TestRunner):
785 """Base class containing common functionalities to run xctests.""" 792 """Base class containing common functionalities to run xctests."""
786 def __init__( 793 def __init__(
787 self, 794 self,
788 app_path, 795 app_path,
789 test_host,
790 test_project_dir,
791 xcode_version=None, 796 xcode_version=None,
792 gs_bucket=None, 797 gs_bucket=None,
793 perf_bot_name=None, 798 perf_bot_name=None,
794 perf_build_number=None, 799 perf_build_number=None,
795 perf_builder_name=None, 800 perf_builder_name=None,
796 perf_master_name=None, 801 perf_master_name=None,
797 perf_revision=None, 802 perf_revision=None,
798 perf_x_value=None, 803 perf_x_value=None,
799 test_args=None, 804 test_args=None,
800 env_vars=None, 805 env_vars=None,
801 ): 806 ):
802 """Initializes an instance of the SimulatorXCTestRunner class. 807 """Initializes an instance of the SimulatorXCTestRunner class.
803 808
804 Args: 809 Args:
805 app_path: Full path to the compiled app to run. 810 app_path: Full path to the compiled app to run.
806 test_host: Name of the compiled test host app to run tests.
807 test_project_dir: Directory of the dummy test project.
808 xcode_version: Version of Xcode to use. 811 xcode_version: Version of Xcode to use.
809 gs_bucket: Google Storage bucket to upload test data to, or None if the 812 gs_bucket: Google Storage bucket to upload test data to, or None if the
810 test data should not be uploaded. 813 test data should not be uploaded.
811 perf_bot_name: Name of this bot as indicated to the perf dashboard. 814 perf_bot_name: Name of this bot as indicated to the perf dashboard.
812 perf_build_number: Build number to indicate to the perf dashboard. 815 perf_build_number: Build number to indicate to the perf dashboard.
813 perf_builder_name: Name of this builder as indicated to the perf 816 perf_builder_name: Name of this builder as indicated to the perf
814 dashboard. 817 dashboard.
815 perf_master_name: Name of the master as indicated to the perf dashboard. 818 perf_master_name: Name of the master as indicated to the perf dashboard.
816 perf_revision: Revision to indicate to the perf dashboard. 819 perf_revision: Revision to indicate to the perf dashboard.
817 perf_x_value: Value to use on the x axis for all data uploaded to the 820 perf_x_value: Value to use on the x axis for all data uploaded to the
(...skipping 12 matching lines...) Expand all
830 perf_bot_name=perf_bot_name, 833 perf_bot_name=perf_bot_name,
831 perf_build_number=perf_build_number, 834 perf_build_number=perf_build_number,
832 perf_builder_name=perf_builder_name, 835 perf_builder_name=perf_builder_name,
833 perf_master_name=perf_master_name, 836 perf_master_name=perf_master_name,
834 perf_revision=perf_revision, 837 perf_revision=perf_revision,
835 perf_x_value=perf_x_value, 838 perf_x_value=perf_x_value,
836 test_args=test_args, 839 test_args=test_args,
837 xcode_version=xcode_version, 840 xcode_version=xcode_version,
838 ) 841 )
839 self.app_path = os.path.abspath(app_path) 842 self.app_path = os.path.abspath(app_path)
840 self.test_host_name = test_host
841 # Test target name is its host name without '_host' suffix.
842 self.test_target_name = test_host.rsplit('_', 1)[0]
843 self.test_project_dir = test_project_dir
844 self.timeout = '120' 843 self.timeout = '120'
845 self.homedir = '' 844 self.homedir = ''
846 self.start_time = None 845 self.start_time = None
847 846
848 def TearDown(self): 847 def TearDown(self):
849 """Performs post-test tear down.""" 848 """Performs post-test tear down."""
850 raise NotImplementedError 849 raise NotImplementedError
851 850
852 def HandleJsonFileWithPath(self, summary): 851 def HandleJsonFileWithPath(self, summary):
853 """Parse data in summarydir and send to perf dashboard.""" 852 """Parse data in summarydir and send to perf dashboard."""
854 with open(summary) as jsonFile: 853 with open(summary) as jsonFile:
855 return json.load(jsonFile) 854 return json.load(jsonFile)
856 855
857 def GetLaunchEnvironment(self): 856 def GetLaunchEnvironment(self):
858 """Returns the environment which is used to run the xctest. 857 """Returns the environment which is used to run the xctest.
859 """ 858 """
860 env = dict(os.environ, APP_TARGET_NAME=self.test_host_name, 859 env = dict(os.environ,
861 TEST_TARGET_NAME=self.test_target_name,
862 NSUnbufferedIO='YES') 860 NSUnbufferedIO='YES')
863 return env 861 return env
864 862
865 def GetLaunchCommand(self, test_filter=None, blacklist=False): 863 def GetLaunchCommand(self, test_filter=None, blacklist=False):
866 """Returns the command which is used to launch the test. 864 """Returns the command which is used to launch the test.
867 865
868 Args: 866 Args:
869 test_filter: A list of tests to filter by, or None to mean all. 867 test_filter: A list of tests to filter by, or None to mean all.
870 blacklist: Whether to blacklist the elements of test_filter or not. Only 868 blacklist: Whether to blacklist the elements of test_filter or not. Only
871 works when test_filter is not None. 869 works when test_filter is not None.
(...skipping 141 matching lines...) Expand 10 before | Expand all | Expand 10 after
1013 # At this point, all the tests have run, so used failed_tests to determine 1011 # At this point, all the tests have run, so used failed_tests to determine
1014 # the success/failure. 1012 # the success/failure.
1015 return not failed_tests 1013 return not failed_tests
1016 1014
1017 1015
1018 class SimulatorXCTestRunner(XCTestRunner): 1016 class SimulatorXCTestRunner(XCTestRunner):
1019 """Class for running xctests on an iOS simulator.""" 1017 """Class for running xctests on an iOS simulator."""
1020 def __init__( 1018 def __init__(
1021 self, 1019 self,
1022 app_path, 1020 app_path,
1023 test_host, 1021 xctest_path,
1024 test_project_dir, 1022 iossim_path,
1025 platform, 1023 platform,
1026 version, 1024 version,
1027 xcode_version=None, 1025 xcode_version=None,
1028 gs_bucket=None, 1026 gs_bucket=None,
1029 perf_bot_name=None, 1027 perf_bot_name=None,
1030 perf_build_number=None, 1028 perf_build_number=None,
1031 perf_builder_name=None, 1029 perf_builder_name=None,
1032 perf_master_name=None, 1030 perf_master_name=None,
1033 perf_revision=None, 1031 perf_revision=None,
1034 perf_x_value=None, 1032 perf_x_value=None,
1035 test_args=None, 1033 test_args=None,
1036 env_vars=None, 1034 env_vars=None,
1037 ): 1035 ):
1038 """Initializes an instance of the SimulatorXCTestRunner class. 1036 """Initializes an instance of the SimulatorXCTestRunner class.
1039 1037
1040 Args: 1038 Args:
1041 app_path: Full path to the compiled app to run. 1039 app_path: Full path to the compiled app to run.
1042 test_host: Name of the compiled test host app to run tests. 1040 iossim_path: Full path to the iossim executable to launch.
1043 test_project_dir: Directory of the dummy test project. 1041 xctest_path: Full path to the compiled test bundle.
1044 platform: The platform to simulate. Supported values can be found by 1042 platform: The platform to simulate. Supported values can be found by
1045 running 'xcodebuild -list'. e.g. 'iPhone 5', 'iPhone 5s'. 1043 running 'xcodebuild -list'. e.g. 'iPhone 5', 'iPhone 5s'.
1046 version: The iOS version the simulator should be running. Supported values 1044 version: The iOS version the simulator should be running. Supported values
1047 can be found by running 'xcodebuild -list'. e.g. '8.0', '7.1'. 1045 can be found by running 'xcodebuild -list'. e.g. '8.0', '7.1'.
1048 xcode_version: Version of Xcode to use. 1046 xcode_version: Version of Xcode to use.
1049 gs_bucket: Google Storage bucket to upload test data to, or None if the 1047 gs_bucket: Google Storage bucket to upload test data to, or None if the
1050 test data should not be uploaded. 1048 test data should not be uploaded.
1051 perf_bot_name: Name of this bot as indicated to the perf dashboard. 1049 perf_bot_name: Name of this bot as indicated to the perf dashboard.
1052 perf_build_number: Build number to indicate to the perf dashboard. 1050 perf_build_number: Build number to indicate to the perf dashboard.
1053 perf_builder_name: Name of this builder as indicated to the perf 1051 perf_builder_name: Name of this builder as indicated to the perf
1054 dashboard. 1052 dashboard.
1055 perf_master_name: Name of the master as indicated to the perf dashboard. 1053 perf_master_name: Name of the master as indicated to the perf dashboard.
1056 perf_revision: Revision to indicate to the perf dashboard. 1054 perf_revision: Revision to indicate to the perf dashboard.
1057 perf_x_value: Value to use on the x axis for all data uploaded to the 1055 perf_x_value: Value to use on the x axis for all data uploaded to the
1058 perf dashboard. 1056 perf dashboard.
1059 test_args: Arguments to pass when launching the test. 1057 test_args: Arguments to pass when launching the test.
1060 env_vars: Environment variables to set when launching the test. 1058 env_vars: Environment variables to set when launching the test.
1061 1059
1062 Raises: 1060 Raises:
1063 SimulatorNotFoundError: If the given iossim path cannot be found. 1061 SimulatorNotFoundError: If the given iossim path cannot be found.
1064 """ 1062 """
1065 super(SimulatorXCTestRunner, self).__init__( 1063 super(SimulatorXCTestRunner, self).__init__(
1066 app_path, 1064 app_path,
1067 test_host,
1068 test_project_dir,
1069 env_vars=env_vars, 1065 env_vars=env_vars,
1070 gs_bucket=gs_bucket, 1066 gs_bucket=gs_bucket,
1071 perf_bot_name=perf_bot_name, 1067 perf_bot_name=perf_bot_name,
1072 perf_build_number=perf_build_number, 1068 perf_build_number=perf_build_number,
1073 perf_builder_name=perf_builder_name, 1069 perf_builder_name=perf_builder_name,
1074 perf_master_name=perf_master_name, 1070 perf_master_name=perf_master_name,
1075 perf_revision=perf_revision, 1071 perf_revision=perf_revision,
1076 perf_x_value=perf_x_value, 1072 perf_x_value=perf_x_value,
1077 test_args=test_args, 1073 test_args=test_args,
1078 xcode_version=xcode_version, 1074 xcode_version=xcode_version,
1079 ) 1075 )
1076
1077 if not os.path.exists(iossim_path):
1078 raise SimulatorNotFoundError(iossim_path)
1079
1080 if not os.path.exists(xctest_path):
1081 raise XctestNotFoundError(xctest_path)
1082
1083 self.iossim_path = iossim_path
1084 self.xctest_path = xctest_path
1080 self.platform = platform 1085 self.platform = platform
1081 self.version = version 1086 self.version = version
1082 1087
1083 def UploadTestData(self): 1088 def UploadTestData(self):
1084 """Uploads the contents of the test's Documents directory. 1089 """Uploads the contents of the test's Documents directory.
1085 1090
1086 Returns: 1091 Returns:
1087 True if test data was uploaded, False otherwise. 1092 True if test data was uploaded, False otherwise.
1088 """ 1093 """
1089 if not self.gs_bucket: 1094 if not self.gs_bucket:
(...skipping 171 matching lines...) Expand 10 before | Expand all | Expand 10 after
1261 test_filter: A list of tests to filter by, or None to mean all. 1266 test_filter: A list of tests to filter by, or None to mean all.
1262 blacklist: Whether to blacklist the elements of test_filter or not. Only 1267 blacklist: Whether to blacklist the elements of test_filter or not. Only
1263 works when test_filter is not None. 1268 works when test_filter is not None.
1264 1269
1265 Returns: 1270 Returns:
1266 A list whose elements are the args representing the command. 1271 A list whose elements are the args representing the command.
1267 """ 1272 """
1268 built_dir = os.path.split(self.app_path)[0] 1273 built_dir = os.path.split(self.app_path)[0]
1269 1274
1270 cmd = [ 1275 cmd = [
1271 'xcodebuild', 'test-without-building', 1276 self.iossim_path,
1272 'BUILT_PRODUCTS_DIR=%s' % built_dir, 1277 '-d', self.platform,
1273 '-project', self.test_project_dir, 1278 '-s', self.version,
1274 '-scheme','TestProject', 1279 self.app_path,
1275 '-destination','platform=iOS Simulator,name=%s,OS=%s' 1280 self.xctest_path
1276 % (self.platform, self.version),
1277 '-archivePath', self.homedir,
1278 'APP_TARGET_NAME=%s' % self.test_host_name,
1279 'TEST_TARGET_NAME=%s' % self.test_target_name,
1280 'NSUnbufferedIO=YES'
1281 ] 1281 ]
1282 return cmd 1282 return cmd
1283 1283
1284 @TestRunner.RequireTearDown 1284 @TestRunner.RequireTearDown
1285 def Launch(self, *args, **kwargs): 1285 def Launch(self, *args, **kwargs):
1286 """Launches the test.""" 1286 """Launches the test."""
1287 self.SetUp() 1287 self.SetUp()
1288 1288
1289 result = self._Run( 1289 result = self._Run(
1290 self.GetLaunchCommand(), self.GetLaunchEnvironment(), *args, **kwargs) 1290 self.GetLaunchCommand(), self.GetLaunchEnvironment(), *args, **kwargs)
(...skipping 15 matching lines...) Expand all
1306 self.GetLaunchCommand(), self.GetLaunchEnvironment(), *args, **kwargs) 1306 self.GetLaunchCommand(), self.GetLaunchEnvironment(), *args, **kwargs)
1307 1307
1308 return self.RunAllTests(result, *args, **kwargs) 1308 return self.RunAllTests(result, *args, **kwargs)
1309 1309
1310 1310
1311 class DeviceXCTestRunner(XCTestRunner): 1311 class DeviceXCTestRunner(XCTestRunner):
1312 """Class for running xctests on an iOS device.""" 1312 """Class for running xctests on an iOS device."""
1313 def __init__( 1313 def __init__(
1314 self, 1314 self,
1315 app_path, 1315 app_path,
1316 test_host, 1316 xctest_path,
1317 test_project_dir, 1317 xcodeproj_path,
1318 xcode_version=None, 1318 xcode_version=None,
1319 gs_bucket=None, 1319 gs_bucket=None,
1320 perf_bot_name=None, 1320 perf_bot_name=None,
1321 perf_build_number=None, 1321 perf_build_number=None,
1322 perf_builder_name=None, 1322 perf_builder_name=None,
1323 perf_master_name=None, 1323 perf_master_name=None,
1324 perf_revision=None, 1324 perf_revision=None,
1325 perf_x_value=None, 1325 perf_x_value=None,
1326 test_args=None, 1326 test_args=None,
1327 env_vars=None, 1327 env_vars=None,
(...skipping 19 matching lines...) Expand all
1347 env_vars: Environment variables to set when launching the test. 1347 env_vars: Environment variables to set when launching the test.
1348 1348
1349 Raises: 1349 Raises:
1350 DeviceDetectionError: If this machine does not have exactly one device 1350 DeviceDetectionError: If this machine does not have exactly one device
1351 connected. Having more than one device connected causes problems when 1351 connected. Having more than one device connected causes problems when
1352 trying to issue commands to any one device, which interfere with 1352 trying to issue commands to any one device, which interfere with
1353 installing and running the test app. 1353 installing and running the test app.
1354 """ 1354 """
1355 super(DeviceXCTestRunner, self).__init__( 1355 super(DeviceXCTestRunner, self).__init__(
1356 app_path, 1356 app_path,
1357 test_host,
1358 test_project_dir,
1359 env_vars=env_vars, 1357 env_vars=env_vars,
1360 gs_bucket=gs_bucket, 1358 gs_bucket=gs_bucket,
1361 perf_bot_name=perf_bot_name, 1359 perf_bot_name=perf_bot_name,
1362 perf_build_number=perf_build_number, 1360 perf_build_number=perf_build_number,
1363 perf_builder_name=perf_builder_name, 1361 perf_builder_name=perf_builder_name,
1364 perf_master_name=perf_master_name, 1362 perf_master_name=perf_master_name,
1365 perf_revision=perf_revision, 1363 perf_revision=perf_revision,
1366 perf_x_value=perf_x_value, 1364 perf_x_value=perf_x_value,
1367 test_args=test_args, 1365 test_args=test_args,
1368 xcode_version=xcode_version, 1366 xcode_version=xcode_version,
1369 ) 1367 )
1368 # The scheme name is the app name without the '.app' suffix.
1369 app_name = os.path.split(self.app_path)[1]
1370 self.scheme_name = app_name.split('.', 1)[0]
1371 self.xcodeproj_path = xcodeproj_path
1370 self.cfbundleid = utils.call( 1372 self.cfbundleid = utils.call(
1371 utils.PLIST_BUDDY, 1373 utils.PLIST_BUDDY,
1372 '-c', 'Print:CFBundleIdentifier', 1374 '-c', 'Print:CFBundleIdentifier',
1373 os.path.join(self.app_path, 'Info.plist'), 1375 os.path.join(self.app_path, 'Info.plist'),
1374 ).stdout[0] 1376 ).stdout[0]
1375 1377
1376 call_result = utils.call('idevice_id', '--list') 1378 call_result = utils.call('idevice_id', '--list')
1377 self.device_id = call_result.stdout[0] 1379 self.device_id = call_result.stdout[0]
1378 1380
1379 if len(call_result.stdout) != 1: 1381 if len(call_result.stdout) != 1:
(...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after
1420 """Returns the invocation command which is used to run the test. 1422 """Returns the invocation command which is used to run the test.
1421 1423
1422 Args: 1424 Args:
1423 test_filter: A list of tests to filter by, or None to mean all. 1425 test_filter: A list of tests to filter by, or None to mean all.
1424 blacklist: Whether to blacklist the elements of test_filter or not. Only 1426 blacklist: Whether to blacklist the elements of test_filter or not. Only
1425 works when test_filter is not None. 1427 works when test_filter is not None.
1426 1428
1427 Returns: 1429 Returns:
1428 A list whose elements are the args representing the command. 1430 A list whose elements are the args representing the command.
1429 """ 1431 """
1430 built_dir = os.path.split(self.app_path)[0]
1431
1432 cmd = [ 1432 cmd = [
1433 'xcodebuild', 'test-without-building', 1433 'xcodebuild', 'test-without-building',
1434 'BUILT_PRODUCTS_DIR=%s' % built_dir, 1434 '-project', self.xcodeproj_path,
1435 'CONFIGURATION_BUILD_DIR=%s' % built_dir, 1435 '-scheme', self.scheme_name,
1436 '-project', self.test_project_dir, 1436 '-destination', 'id=%s' % self.device_id
1437 '-configuration', 'iphoneos',
1438 '-scheme', 'TestProject',
1439 '-destination','id=%s' % self.device_id,
1440 'APP_TARGET_NAME=%s' % self.test_host_name,
1441 'TEST_TARGET_NAME=%s' % self.test_target_name,
1442 'NSUnbufferedIO=YES'
1443 ] 1437 ]
1444 return cmd 1438 return cmd
1445 1439
1446 @TestRunner.RequireTearDown 1440 @TestRunner.RequireTearDown
1447 def Launch(self, *args, **kwargs): 1441 def Launch(self, *args, **kwargs):
1448 """Launches the test.""" 1442 """Launches the test."""
1449 self.InstallApp() 1443 self.InstallApp()
1450 1444
1451 result = self._Run( 1445 result = self._Run(
1452 self.GetLaunchCommand(), self.GetLaunchEnvironment(), *args, **kwargs) 1446 self.GetLaunchCommand(), self.GetLaunchEnvironment(), *args, **kwargs)
1453 1447
1454 if result.crashed and not result.crashed_test: 1448 if result.crashed and not result.crashed_test:
1455 # If the app crashed, but there is no specific test which crashed, 1449 # If the app crashed, but there is no specific test which crashed,
1456 # then the app must have failed to even start. Try one more time. 1450 # then the app must have failed to even start. Try one more time.
1457 self.Print( 1451 self.Print(
1458 '%s appears to have crashed on startup. Retrying...' % self.app_name, 1452 '%s appears to have crashed on startup. Retrying...' % self.app_name,
1459 blank_lines=2, 1453 blank_lines=2,
1460 time_to_sleep=5, 1454 time_to_sleep=5,
1461 ) 1455 )
1462 1456
1463 # Uninstall and re-install the app. 1457 # Uninstall and re-install the app.
1464 self.UninstallApp() 1458 self.UninstallApp()
1465 self.InstallApp() 1459 self.InstallApp()
1466 1460
1467 result = self._Run( 1461 result = self._Run(
1468 self.GetLaunchCommand(), self.GetLaunchEnvironment(), *args, **kwargs) 1462 self.GetLaunchCommand(), self.GetLaunchEnvironment(), *args, **kwargs)
1469 1463
1470 return self.RunAllTests(result, *args, **kwargs) 1464 return self.RunAllTests(result, *args, **kwargs)
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698