OLD | NEW |
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import logging | 5 import logging |
6 import os | 6 import os |
7 import re | 7 import re |
8 | 8 |
9 from pylib import android_commands | 9 from pylib import android_commands |
10 from pylib import constants | 10 from pylib import constants |
11 from pylib import pexpect | 11 from pylib import pexpect |
12 from pylib.base import base_test_result | 12 from pylib.base import base_test_result |
13 from pylib.base import base_test_runner | 13 from pylib.base import base_test_runner |
14 from pylib.utils import run_tests_helper | |
15 | |
16 import test_package_apk | |
17 import test_package_exe | |
18 | 14 |
19 | 15 |
20 def _TestSuiteRequiresMockTestServer(suite_basename): | 16 def _TestSuiteRequiresMockTestServer(suite_name): |
21 """Returns True if the test suite requires mock test server.""" | 17 """Returns True if the test suite requires mock test server.""" |
22 tests_require_net_test_server = ['unit_tests', 'net_unittests', | 18 tests_require_net_test_server = ['unit_tests', 'net_unittests', |
23 'content_unittests', | 19 'content_unittests', |
24 'content_browsertests'] | 20 'content_browsertests'] |
25 return (suite_basename in | 21 return (suite_name in |
26 tests_require_net_test_server) | 22 tests_require_net_test_server) |
27 | 23 |
28 | 24 |
29 class TestRunner(base_test_runner.BaseTestRunner): | 25 class TestRunner(base_test_runner.BaseTestRunner): |
30 def __init__(self, device, suite_name, test_arguments, timeout, | 26 def __init__(self, device, test_package, test_arguments, timeout, |
31 cleanup_test_files, tool_name, build_type, | 27 cleanup_test_files, tool_name, build_type, push_deps): |
32 push_deps, test_apk_package_name=None, | |
33 test_activity_name=None, command_line_file=None): | |
34 """Single test suite attached to a single device. | 28 """Single test suite attached to a single device. |
35 | 29 |
36 Args: | 30 Args: |
37 device: Device to run the tests. | 31 device: Device to run the tests. |
38 suite_name: A specific test suite to run, empty to run all. | 32 test_package: An instance of TestPackage class. |
39 test_arguments: Additional arguments to pass to the test binary. | 33 test_arguments: Additional arguments to pass to the test binary. |
40 timeout: Timeout for each test. | 34 timeout: Timeout for each test. |
41 cleanup_test_files: Whether or not to cleanup test files on device. | 35 cleanup_test_files: Whether or not to cleanup test files on device. |
42 tool_name: Name of the Valgrind tool. | 36 tool_name: Name of the Valgrind tool. |
43 build_type: 'Release' or 'Debug'. | 37 build_type: 'Release' or 'Debug'. |
44 push_deps: If True, push all dependencies to the device. | 38 push_deps: If True, push all dependencies to the device. |
45 test_apk_package_name: Apk package name for tests running in APKs. | |
46 test_activity_name: Test activity to invoke for APK tests. | |
47 command_line_file: Filename to use to pass arguments to tests. | |
48 """ | 39 """ |
49 super(TestRunner, self).__init__(device, tool_name, build_type, push_deps, | 40 super(TestRunner, self).__init__(device, tool_name, build_type, push_deps, |
50 cleanup_test_files) | 41 cleanup_test_files) |
| 42 self.test_package = test_package |
| 43 self.test_package.tool = self.tool |
51 self._test_arguments = test_arguments | 44 self._test_arguments = test_arguments |
52 if timeout == 0: | 45 if timeout == 0: |
53 timeout = 60 | 46 timeout = 60 |
54 # On a VM (e.g. chromium buildbots), this timeout is way too small. | 47 # On a VM (e.g. chromium buildbots), this timeout is way too small. |
55 if os.environ.get('BUILDBOT_SLAVENAME'): | 48 if os.environ.get('BUILDBOT_SLAVENAME'): |
56 timeout = timeout * 2 | 49 timeout = timeout * 2 |
57 self.timeout = timeout * self.tool.GetTimeoutScale() | 50 self._timeout = timeout * self.tool.GetTimeoutScale() |
58 | |
59 logging.warning('Test suite: ' + str(suite_name)) | |
60 if os.path.splitext(suite_name)[1] == '.apk': | |
61 self.test_package = test_package_apk.TestPackageApk( | |
62 self.adb, | |
63 device, | |
64 suite_name, | |
65 self.tool, | |
66 test_apk_package_name, | |
67 test_activity_name, | |
68 command_line_file) | |
69 else: | |
70 # Put a copy into the android out/target directory, to allow stack trace | |
71 # generation. | |
72 symbols_dir = os.path.join(constants.DIR_SOURCE_ROOT, 'out', build_type, | |
73 'lib.target') | |
74 self.test_package = test_package_exe.TestPackageExecutable( | |
75 self.adb, | |
76 device, | |
77 suite_name, | |
78 self.tool, | |
79 symbols_dir) | |
80 | 51 |
81 #override | 52 #override |
82 def InstallTestPackage(self): | 53 def InstallTestPackage(self): |
83 self.test_package.Install() | 54 self.test_package.Install(self.adb) |
| 55 |
| 56 def GetAllTests(self): |
| 57 """Install test package and get a list of all tests.""" |
| 58 self.test_package.Install(self.adb) |
| 59 return self.test_package.GetAllTests(self.adb) |
84 | 60 |
85 #override | 61 #override |
86 def PushDataDeps(self): | 62 def PushDataDeps(self): |
87 self.adb.WaitForSdCardReady(20) | 63 self.adb.WaitForSdCardReady(20) |
88 self.tool.CopyFiles() | 64 self.tool.CopyFiles() |
89 if os.path.exists(constants.ISOLATE_DEPS_DIR): | 65 if os.path.exists(constants.ISOLATE_DEPS_DIR): |
90 device_dir = self.adb.GetExternalStorage() | 66 device_dir = self.adb.GetExternalStorage() |
91 # TODO(frankf): linux_dumper_unittest_helper needs to be in the same dir | 67 # TODO(frankf): linux_dumper_unittest_helper needs to be in the same dir |
92 # as breakpad_unittests exe. Find a better way to do this. | 68 # as breakpad_unittests exe. Find a better way to do this. |
93 if self.test_package.suite_basename == 'breakpad_unittests': | 69 if self.test_package.suite_name == 'breakpad_unittests': |
94 device_dir = constants.TEST_EXECUTABLE_DIR | 70 device_dir = constants.TEST_EXECUTABLE_DIR |
95 for p in os.listdir(constants.ISOLATE_DEPS_DIR): | 71 for p in os.listdir(constants.ISOLATE_DEPS_DIR): |
96 self.adb.PushIfNeeded( | 72 self.adb.PushIfNeeded( |
97 os.path.join(constants.ISOLATE_DEPS_DIR, p), | 73 os.path.join(constants.ISOLATE_DEPS_DIR, p), |
98 os.path.join(device_dir, p)) | 74 os.path.join(device_dir, p)) |
99 | 75 |
100 def _ParseTestOutput(self, p): | 76 def _ParseTestOutput(self, p): |
101 """Process the test output. | 77 """Process the test output. |
102 | 78 |
103 Args: | 79 Args: |
(...skipping 14 matching lines...) Expand all Loading... |
118 re_runner_fail = re.compile('\[ RUNNER_FAILED \] ?(.*)\r\n') | 94 re_runner_fail = re.compile('\[ RUNNER_FAILED \] ?(.*)\r\n') |
119 # Signal handlers are installed before starting tests | 95 # Signal handlers are installed before starting tests |
120 # to output the CRASHED marker when a crash happens. | 96 # to output the CRASHED marker when a crash happens. |
121 re_crash = re.compile('\[ CRASHED \](.*)\r\n') | 97 re_crash = re.compile('\[ CRASHED \](.*)\r\n') |
122 | 98 |
123 log = '' | 99 log = '' |
124 try: | 100 try: |
125 while True: | 101 while True: |
126 full_test_name = None | 102 full_test_name = None |
127 found = p.expect([re_run, re_passed, re_runner_fail], | 103 found = p.expect([re_run, re_passed, re_runner_fail], |
128 timeout=self.timeout) | 104 timeout=self._timeout) |
129 if found == 1: # re_passed | 105 if found == 1: # re_passed |
130 break | 106 break |
131 elif found == 2: # re_runner_fail | 107 elif found == 2: # re_runner_fail |
132 break | 108 break |
133 else: # re_run | 109 else: # re_run |
134 full_test_name = p.match.group(1).replace('\r', '') | 110 full_test_name = p.match.group(1).replace('\r', '') |
135 found = p.expect([re_ok, re_fail, re_crash], timeout=self.timeout) | 111 found = p.expect([re_ok, re_fail, re_crash], timeout=self._timeout) |
136 log = p.before.replace('\r', '') | 112 log = p.before.replace('\r', '') |
137 if found == 0: # re_ok | 113 if found == 0: # re_ok |
138 if full_test_name == p.match.group(1).replace('\r', ''): | 114 if full_test_name == p.match.group(1).replace('\r', ''): |
139 results.AddResult(base_test_result.BaseTestResult( | 115 results.AddResult(base_test_result.BaseTestResult( |
140 full_test_name, base_test_result.ResultType.PASS, | 116 full_test_name, base_test_result.ResultType.PASS, |
141 log=log)) | 117 log=log)) |
142 elif found == 2: # re_crash | 118 elif found == 2: # re_crash |
143 results.AddResult(base_test_result.BaseTestResult( | 119 results.AddResult(base_test_result.BaseTestResult( |
144 full_test_name, base_test_result.ResultType.CRASH, | 120 full_test_name, base_test_result.ResultType.CRASH, |
145 log=log)) | 121 log=log)) |
146 break | 122 break |
147 else: # re_fail | 123 else: # re_fail |
148 results.AddResult(base_test_result.BaseTestResult( | 124 results.AddResult(base_test_result.BaseTestResult( |
149 full_test_name, base_test_result.ResultType.FAIL, log=log)) | 125 full_test_name, base_test_result.ResultType.FAIL, log=log)) |
150 except pexpect.EOF: | 126 except pexpect.EOF: |
151 logging.error('Test terminated - EOF') | 127 logging.error('Test terminated - EOF') |
152 # We're here because either the device went offline, or the test harness | 128 # We're here because either the device went offline, or the test harness |
153 # crashed without outputting the CRASHED marker (crbug.com/175538). | 129 # crashed without outputting the CRASHED marker (crbug.com/175538). |
154 if not self.adb.IsOnline(): | 130 if not self.adb.IsOnline(): |
155 raise android_commands.errors.DeviceUnresponsiveError( | 131 raise android_commands.errors.DeviceUnresponsiveError( |
156 'Device %s went offline.' % self.device) | 132 'Device %s went offline.' % self.device) |
157 if full_test_name: | 133 if full_test_name: |
158 results.AddResult(base_test_result.BaseTestResult( | 134 results.AddResult(base_test_result.BaseTestResult( |
159 full_test_name, base_test_result.ResultType.CRASH, | 135 full_test_name, base_test_result.ResultType.CRASH, |
160 log=p.before.replace('\r', ''))) | 136 log=p.before.replace('\r', ''))) |
161 except pexpect.TIMEOUT: | 137 except pexpect.TIMEOUT: |
162 logging.error('Test terminated after %d second timeout.', | 138 logging.error('Test terminated after %d second timeout.', |
163 self.timeout) | 139 self._timeout) |
164 if full_test_name: | 140 if full_test_name: |
165 results.AddResult(base_test_result.BaseTestResult( | 141 results.AddResult(base_test_result.BaseTestResult( |
166 full_test_name, base_test_result.ResultType.TIMEOUT, | 142 full_test_name, base_test_result.ResultType.TIMEOUT, |
167 log=p.before.replace('\r', ''))) | 143 log=p.before.replace('\r', ''))) |
168 finally: | 144 finally: |
169 p.close() | 145 p.close() |
170 | 146 |
171 ret_code = self.test_package.GetGTestReturnCode() | 147 ret_code = self.test_package.GetGTestReturnCode(self.adb) |
172 if ret_code: | 148 if ret_code: |
173 logging.critical( | 149 logging.critical( |
174 'gtest exit code: %d\npexpect.before: %s\npexpect.after: %s', | 150 'gtest exit code: %d\npexpect.before: %s\npexpect.after: %s', |
175 ret_code, p.before, p.after) | 151 ret_code, p.before, p.after) |
176 | 152 |
177 return results | 153 return results |
178 | 154 |
179 #override | 155 #override |
180 def RunTest(self, test): | 156 def RunTest(self, test): |
181 test_results = base_test_result.TestRunResults() | 157 test_results = base_test_result.TestRunResults() |
182 if not test: | 158 if not test: |
183 return test_results, None | 159 return test_results, None |
184 | 160 |
185 try: | 161 try: |
186 self.test_package.ClearApplicationState() | 162 self.test_package.ClearApplicationState(self.adb) |
187 self.test_package.CreateCommandLineFileOnDevice( | 163 self.test_package.CreateCommandLineFileOnDevice( |
188 test, self._test_arguments) | 164 self.adb, test, self._test_arguments) |
189 test_results = self._ParseTestOutput(self.test_package.SpawnTestProcess()) | 165 test_results = self._ParseTestOutput( |
| 166 self.test_package.SpawnTestProcess(self.adb)) |
190 finally: | 167 finally: |
191 self.CleanupSpawningServerState() | 168 self.CleanupSpawningServerState() |
192 # Calculate unknown test results. | 169 # Calculate unknown test results. |
193 all_tests = set(test.split(':')) | 170 all_tests = set(test.split(':')) |
194 all_tests_ran = set([t.GetName() for t in test_results.GetAll()]) | 171 all_tests_ran = set([t.GetName() for t in test_results.GetAll()]) |
195 unknown_tests = all_tests - all_tests_ran | 172 unknown_tests = all_tests - all_tests_ran |
196 test_results.AddResults( | 173 test_results.AddResults( |
197 [base_test_result.BaseTestResult(t, base_test_result.ResultType.UNKNOWN) | 174 [base_test_result.BaseTestResult(t, base_test_result.ResultType.UNKNOWN) |
198 for t in unknown_tests]) | 175 for t in unknown_tests]) |
199 retry = ':'.join([t.GetName() for t in test_results.GetNotPass()]) | 176 retry = ':'.join([t.GetName() for t in test_results.GetNotPass()]) |
200 return test_results, retry | 177 return test_results, retry |
201 | 178 |
202 #override | 179 #override |
203 def SetUp(self): | 180 def SetUp(self): |
204 """Sets up necessary test enviroment for the test suite.""" | 181 """Sets up necessary test enviroment for the test suite.""" |
205 super(TestRunner, self).SetUp() | 182 super(TestRunner, self).SetUp() |
206 if _TestSuiteRequiresMockTestServer(self.test_package.suite_basename): | 183 if _TestSuiteRequiresMockTestServer(self.test_package.suite_name): |
207 self.LaunchChromeTestServerSpawner() | 184 self.LaunchChromeTestServerSpawner() |
208 self.tool.SetupEnvironment() | 185 self.tool.SetupEnvironment() |
209 | 186 |
210 #override | 187 #override |
211 def TearDown(self): | 188 def TearDown(self): |
212 """Cleans up the test enviroment for the test suite.""" | 189 """Cleans up the test enviroment for the test suite.""" |
213 self.tool.CleanUpEnvironment() | 190 self.tool.CleanUpEnvironment() |
214 super(TestRunner, self).TearDown() | 191 super(TestRunner, self).TearDown() |
OLD | NEW |