| OLD | NEW |
| 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import logging | 5 import logging |
| 6 import os | 6 import os |
| 7 import re | 7 import re |
| 8 | 8 |
| 9 from pylib import android_commands | 9 from pylib import android_commands |
| 10 from pylib import constants | 10 from pylib import constants |
| 11 from pylib import pexpect | 11 from pylib import pexpect |
| 12 from pylib.base import base_test_result | 12 from pylib.base import base_test_result |
| 13 from pylib.base import base_test_runner | 13 from pylib.base import base_test_runner |
| 14 from pylib.utils import run_tests_helper | |
| 15 | |
| 16 import test_package_apk | |
| 17 import test_package_exe | |
| 18 | 14 |
| 19 | 15 |
| 20 def _TestSuiteRequiresMockTestServer(suite_basename): | 16 def _TestSuiteRequiresMockTestServer(suite_name): |
| 21 """Returns True if the test suite requires mock test server.""" | 17 """Returns True if the test suite requires mock test server.""" |
| 22 tests_require_net_test_server = ['unit_tests', 'net_unittests', | 18 tests_require_net_test_server = ['unit_tests', 'net_unittests', |
| 23 'content_unittests', | 19 'content_unittests', |
| 24 'content_browsertests'] | 20 'content_browsertests'] |
| 25 return (suite_basename in | 21 return (suite_name in |
| 26 tests_require_net_test_server) | 22 tests_require_net_test_server) |
| 27 | 23 |
| 28 | 24 |
| 29 class TestRunner(base_test_runner.BaseTestRunner): | 25 class TestRunner(base_test_runner.BaseTestRunner): |
| 30 def __init__(self, device, suite_name, test_arguments, timeout, | 26 def __init__(self, device, test_package, test_arguments, timeout, |
| 31 cleanup_test_files, tool_name, build_type, | 27 cleanup_test_files, tool_name, build_type, push_deps): |
| 32 push_deps, test_apk_package_name=None, | |
| 33 test_activity_name=None, command_line_file=None): | |
| 34 """Single test suite attached to a single device. | 28 """Single test suite attached to a single device. |
| 35 | 29 |
| 36 Args: | 30 Args: |
| 37 device: Device to run the tests. | 31 device: Device to run the tests. |
| 38 suite_name: A specific test suite to run, empty to run all. | 32 test_package: An instance of TestPackage class. |
| 39 test_arguments: Additional arguments to pass to the test binary. | 33 test_arguments: Additional arguments to pass to the test binary. |
| 40 timeout: Timeout for each test. | 34 timeout: Timeout for each test. |
| 41 cleanup_test_files: Whether or not to cleanup test files on device. | 35 cleanup_test_files: Whether or not to cleanup test files on device. |
| 42 tool_name: Name of the Valgrind tool. | 36 tool_name: Name of the Valgrind tool. |
| 43 build_type: 'Release' or 'Debug'. | 37 build_type: 'Release' or 'Debug'. |
| 44 push_deps: If True, push all dependencies to the device. | 38 push_deps: If True, push all dependencies to the device. |
| 45 test_apk_package_name: Apk package name for tests running in APKs. | |
| 46 test_activity_name: Test activity to invoke for APK tests. | |
| 47 command_line_file: Filename to use to pass arguments to tests. | |
| 48 """ | 39 """ |
| 49 super(TestRunner, self).__init__(device, tool_name, build_type, push_deps, | 40 super(TestRunner, self).__init__(device, tool_name, build_type, push_deps, |
| 50 cleanup_test_files) | 41 cleanup_test_files) |
| 42 self.test_package = test_package |
| 43 self.test_package.tool = self.tool |
| 51 self._test_arguments = test_arguments | 44 self._test_arguments = test_arguments |
| 52 if timeout == 0: | 45 if timeout == 0: |
| 53 timeout = 60 | 46 timeout = 60 |
| 54 # On a VM (e.g. chromium buildbots), this timeout is way too small. | 47 # On a VM (e.g. chromium buildbots), this timeout is way too small. |
| 55 if os.environ.get('BUILDBOT_SLAVENAME'): | 48 if os.environ.get('BUILDBOT_SLAVENAME'): |
| 56 timeout = timeout * 2 | 49 timeout = timeout * 2 |
| 57 self.timeout = timeout * self.tool.GetTimeoutScale() | 50 self._timeout = timeout * self.tool.GetTimeoutScale() |
| 58 | |
| 59 logging.warning('Test suite: ' + str(suite_name)) | |
| 60 if os.path.splitext(suite_name)[1] == '.apk': | |
| 61 self.test_package = test_package_apk.TestPackageApk( | |
| 62 self.adb, | |
| 63 device, | |
| 64 suite_name, | |
| 65 self.tool, | |
| 66 test_apk_package_name, | |
| 67 test_activity_name, | |
| 68 command_line_file) | |
| 69 else: | |
| 70 # Put a copy into the android out/target directory, to allow stack trace | |
| 71 # generation. | |
| 72 symbols_dir = os.path.join(constants.DIR_SOURCE_ROOT, 'out', build_type, | |
| 73 'lib.target') | |
| 74 self.test_package = test_package_exe.TestPackageExecutable( | |
| 75 self.adb, | |
| 76 device, | |
| 77 suite_name, | |
| 78 self.tool, | |
| 79 symbols_dir) | |
| 80 | 51 |
| 81 #override | 52 #override |
| 82 def InstallTestPackage(self): | 53 def InstallTestPackage(self): |
| 83 self.test_package.Install() | 54 self.test_package.Install(self.adb) |
| 55 |
| 56 def GetAllTests(self): |
| 57 """Install test package and get a list of all tests.""" |
| 58 self.test_package.Install(self.adb) |
| 59 return self.test_package.GetAllTests(self.adb) |
| 84 | 60 |
| 85 #override | 61 #override |
| 86 def PushDataDeps(self): | 62 def PushDataDeps(self): |
| 87 self.adb.WaitForSdCardReady(20) | 63 self.adb.WaitForSdCardReady(20) |
| 88 self.tool.CopyFiles() | 64 self.tool.CopyFiles() |
| 89 if os.path.exists(constants.ISOLATE_DEPS_DIR): | 65 if os.path.exists(constants.ISOLATE_DEPS_DIR): |
| 90 device_dir = self.adb.GetExternalStorage() | 66 device_dir = self.adb.GetExternalStorage() |
| 91 # TODO(frankf): linux_dumper_unittest_helper needs to be in the same dir | 67 # TODO(frankf): linux_dumper_unittest_helper needs to be in the same dir |
| 92 # as breakpad_unittests exe. Find a better way to do this. | 68 # as breakpad_unittests exe. Find a better way to do this. |
| 93 if self.test_package.suite_basename == 'breakpad_unittests': | 69 if self.test_package.suite_name == 'breakpad_unittests': |
| 94 device_dir = constants.TEST_EXECUTABLE_DIR | 70 device_dir = constants.TEST_EXECUTABLE_DIR |
| 95 for p in os.listdir(constants.ISOLATE_DEPS_DIR): | 71 for p in os.listdir(constants.ISOLATE_DEPS_DIR): |
| 96 self.adb.PushIfNeeded( | 72 self.adb.PushIfNeeded( |
| 97 os.path.join(constants.ISOLATE_DEPS_DIR, p), | 73 os.path.join(constants.ISOLATE_DEPS_DIR, p), |
| 98 os.path.join(device_dir, p)) | 74 os.path.join(device_dir, p)) |
| 99 | 75 |
| 100 def _ParseTestOutput(self, p): | 76 def _ParseTestOutput(self, p): |
| 101 """Process the test output. | 77 """Process the test output. |
| 102 | 78 |
| 103 Args: | 79 Args: |
| (...skipping 14 matching lines...) Expand all Loading... |
| 118 re_runner_fail = re.compile('\[ RUNNER_FAILED \] ?(.*)\r\n') | 94 re_runner_fail = re.compile('\[ RUNNER_FAILED \] ?(.*)\r\n') |
| 119 # Signal handlers are installed before starting tests | 95 # Signal handlers are installed before starting tests |
| 120 # to output the CRASHED marker when a crash happens. | 96 # to output the CRASHED marker when a crash happens. |
| 121 re_crash = re.compile('\[ CRASHED \](.*)\r\n') | 97 re_crash = re.compile('\[ CRASHED \](.*)\r\n') |
| 122 | 98 |
| 123 log = '' | 99 log = '' |
| 124 try: | 100 try: |
| 125 while True: | 101 while True: |
| 126 full_test_name = None | 102 full_test_name = None |
| 127 found = p.expect([re_run, re_passed, re_runner_fail], | 103 found = p.expect([re_run, re_passed, re_runner_fail], |
| 128 timeout=self.timeout) | 104 timeout=self._timeout) |
| 129 if found == 1: # re_passed | 105 if found == 1: # re_passed |
| 130 break | 106 break |
| 131 elif found == 2: # re_runner_fail | 107 elif found == 2: # re_runner_fail |
| 132 break | 108 break |
| 133 else: # re_run | 109 else: # re_run |
| 134 full_test_name = p.match.group(1).replace('\r', '') | 110 full_test_name = p.match.group(1).replace('\r', '') |
| 135 found = p.expect([re_ok, re_fail, re_crash], timeout=self.timeout) | 111 found = p.expect([re_ok, re_fail, re_crash], timeout=self._timeout) |
| 136 log = p.before.replace('\r', '') | 112 log = p.before.replace('\r', '') |
| 137 if found == 0: # re_ok | 113 if found == 0: # re_ok |
| 138 if full_test_name == p.match.group(1).replace('\r', ''): | 114 if full_test_name == p.match.group(1).replace('\r', ''): |
| 139 results.AddResult(base_test_result.BaseTestResult( | 115 results.AddResult(base_test_result.BaseTestResult( |
| 140 full_test_name, base_test_result.ResultType.PASS, | 116 full_test_name, base_test_result.ResultType.PASS, |
| 141 log=log)) | 117 log=log)) |
| 142 elif found == 2: # re_crash | 118 elif found == 2: # re_crash |
| 143 results.AddResult(base_test_result.BaseTestResult( | 119 results.AddResult(base_test_result.BaseTestResult( |
| 144 full_test_name, base_test_result.ResultType.CRASH, | 120 full_test_name, base_test_result.ResultType.CRASH, |
| 145 log=log)) | 121 log=log)) |
| 146 break | 122 break |
| 147 else: # re_fail | 123 else: # re_fail |
| 148 results.AddResult(base_test_result.BaseTestResult( | 124 results.AddResult(base_test_result.BaseTestResult( |
| 149 full_test_name, base_test_result.ResultType.FAIL, log=log)) | 125 full_test_name, base_test_result.ResultType.FAIL, log=log)) |
| 150 except pexpect.EOF: | 126 except pexpect.EOF: |
| 151 logging.error('Test terminated - EOF') | 127 logging.error('Test terminated - EOF') |
| 152 # We're here because either the device went offline, or the test harness | 128 # We're here because either the device went offline, or the test harness |
| 153 # crashed without outputting the CRASHED marker (crbug.com/175538). | 129 # crashed without outputting the CRASHED marker (crbug.com/175538). |
| 154 if not self.adb.IsOnline(): | 130 if not self.adb.IsOnline(): |
| 155 raise android_commands.errors.DeviceUnresponsiveError( | 131 raise android_commands.errors.DeviceUnresponsiveError( |
| 156 'Device %s went offline.' % self.device) | 132 'Device %s went offline.' % self.device) |
| 157 if full_test_name: | 133 if full_test_name: |
| 158 results.AddResult(base_test_result.BaseTestResult( | 134 results.AddResult(base_test_result.BaseTestResult( |
| 159 full_test_name, base_test_result.ResultType.CRASH, | 135 full_test_name, base_test_result.ResultType.CRASH, |
| 160 log=p.before.replace('\r', ''))) | 136 log=p.before.replace('\r', ''))) |
| 161 except pexpect.TIMEOUT: | 137 except pexpect.TIMEOUT: |
| 162 logging.error('Test terminated after %d second timeout.', | 138 logging.error('Test terminated after %d second timeout.', |
| 163 self.timeout) | 139 self._timeout) |
| 164 if full_test_name: | 140 if full_test_name: |
| 165 results.AddResult(base_test_result.BaseTestResult( | 141 results.AddResult(base_test_result.BaseTestResult( |
| 166 full_test_name, base_test_result.ResultType.TIMEOUT, | 142 full_test_name, base_test_result.ResultType.TIMEOUT, |
| 167 log=p.before.replace('\r', ''))) | 143 log=p.before.replace('\r', ''))) |
| 168 finally: | 144 finally: |
| 169 p.close() | 145 p.close() |
| 170 | 146 |
| 171 ret_code = self.test_package.GetGTestReturnCode() | 147 ret_code = self.test_package.GetGTestReturnCode(self.adb) |
| 172 if ret_code: | 148 if ret_code: |
| 173 logging.critical( | 149 logging.critical( |
| 174 'gtest exit code: %d\npexpect.before: %s\npexpect.after: %s', | 150 'gtest exit code: %d\npexpect.before: %s\npexpect.after: %s', |
| 175 ret_code, p.before, p.after) | 151 ret_code, p.before, p.after) |
| 176 | 152 |
| 177 return results | 153 return results |
| 178 | 154 |
| 179 #override | 155 #override |
| 180 def RunTest(self, test): | 156 def RunTest(self, test): |
| 181 test_results = base_test_result.TestRunResults() | 157 test_results = base_test_result.TestRunResults() |
| 182 if not test: | 158 if not test: |
| 183 return test_results, None | 159 return test_results, None |
| 184 | 160 |
| 185 try: | 161 try: |
| 186 self.test_package.ClearApplicationState() | 162 self.test_package.ClearApplicationState(self.adb) |
| 187 self.test_package.CreateCommandLineFileOnDevice( | 163 self.test_package.CreateCommandLineFileOnDevice( |
| 188 test, self._test_arguments) | 164 self.adb, test, self._test_arguments) |
| 189 test_results = self._ParseTestOutput(self.test_package.SpawnTestProcess()) | 165 test_results = self._ParseTestOutput( |
| 166 self.test_package.SpawnTestProcess(self.adb)) |
| 190 finally: | 167 finally: |
| 191 self.CleanupSpawningServerState() | 168 self.CleanupSpawningServerState() |
| 192 # Calculate unknown test results. | 169 # Calculate unknown test results. |
| 193 all_tests = set(test.split(':')) | 170 all_tests = set(test.split(':')) |
| 194 all_tests_ran = set([t.GetName() for t in test_results.GetAll()]) | 171 all_tests_ran = set([t.GetName() for t in test_results.GetAll()]) |
| 195 unknown_tests = all_tests - all_tests_ran | 172 unknown_tests = all_tests - all_tests_ran |
| 196 test_results.AddResults( | 173 test_results.AddResults( |
| 197 [base_test_result.BaseTestResult(t, base_test_result.ResultType.UNKNOWN) | 174 [base_test_result.BaseTestResult(t, base_test_result.ResultType.UNKNOWN) |
| 198 for t in unknown_tests]) | 175 for t in unknown_tests]) |
| 199 retry = ':'.join([t.GetName() for t in test_results.GetNotPass()]) | 176 retry = ':'.join([t.GetName() for t in test_results.GetNotPass()]) |
| 200 return test_results, retry | 177 return test_results, retry |
| 201 | 178 |
| 202 #override | 179 #override |
| 203 def SetUp(self): | 180 def SetUp(self): |
| 204 """Sets up necessary test enviroment for the test suite.""" | 181 """Sets up necessary test enviroment for the test suite.""" |
| 205 super(TestRunner, self).SetUp() | 182 super(TestRunner, self).SetUp() |
| 206 if _TestSuiteRequiresMockTestServer(self.test_package.suite_basename): | 183 if _TestSuiteRequiresMockTestServer(self.test_package.suite_name): |
| 207 self.LaunchChromeTestServerSpawner() | 184 self.LaunchChromeTestServerSpawner() |
| 208 self.tool.SetupEnvironment() | 185 self.tool.SetupEnvironment() |
| 209 | 186 |
| 210 #override | 187 #override |
| 211 def TearDown(self): | 188 def TearDown(self): |
| 212 """Cleans up the test enviroment for the test suite.""" | 189 """Cleans up the test enviroment for the test suite.""" |
| 213 self.tool.CleanUpEnvironment() | 190 self.tool.CleanUpEnvironment() |
| 214 super(TestRunner, self).TearDown() | 191 super(TestRunner, self).TearDown() |
| OLD | NEW |