OLD | NEW |
---|---|
1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """Dispatches GTests.""" | 5 """Runs GTests.""" |
6 | 6 |
7 import copy | |
8 import fnmatch | 7 import fnmatch |
9 import glob | 8 import glob |
10 import logging | 9 import logging |
11 import os | 10 import os |
12 import shutil | 11 import shutil |
13 | 12 |
14 from pylib import android_commands | 13 from pylib import android_commands |
15 from pylib import cmd_helper | 14 from pylib import cmd_helper |
16 from pylib import constants | 15 from pylib import constants |
17 from pylib import ports | |
18 from pylib.base import base_test_result | 16 from pylib.base import base_test_result |
19 from pylib.base import shard | |
20 from pylib.utils import emulator | |
21 from pylib.utils import report_results | |
22 from pylib.utils import xvfb | |
23 | 17 |
24 import gtest_config | 18 import gtest_config |
25 import test_runner | 19 import test_runner |
26 | 20 |
27 | 21 |
28 # TODO(frankf): Add more test targets here after making sure we don't | 22 # TODO(frankf): Add more test targets here after making sure we don't |
29 # blow up the dependency size (and the world). | 23 # blow up the dependency size (and the world). |
30 _ISOLATE_FILE_PATHS = { | 24 _ISOLATE_FILE_PATHS = { |
31 'base_unittests': 'base/base_unittests.isolate', | 25 'base_unittests': 'base/base_unittests.isolate', |
32 'unit_tests': 'chrome/unit_tests.isolate', | 26 'unit_tests': 'chrome/unit_tests.isolate', |
(...skipping 99 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
132 deps_product_dir = os.path.join(deps_dir, 'out', build_type) | 126 deps_product_dir = os.path.join(deps_dir, 'out', build_type) |
133 if os.path.isdir(deps_product_dir): | 127 if os.path.isdir(deps_product_dir): |
134 for p in os.listdir(deps_product_dir): | 128 for p in os.listdir(deps_product_dir): |
135 shutil.move(os.path.join(deps_product_dir, p), deps_dir) | 129 shutil.move(os.path.join(deps_product_dir, p), deps_dir) |
136 os.rmdir(deps_product_dir) | 130 os.rmdir(deps_product_dir) |
137 os.rmdir(os.path.join(deps_dir, 'out')) | 131 os.rmdir(os.path.join(deps_dir, 'out')) |
138 | 132 |
139 return deps_dir | 133 return deps_dir |
140 | 134 |
141 | 135 |
142 def _FullyQualifiedTestSuites(exe, option_test_suite, build_type): | 136 def _FullyQualifiedTestSuites(use_exe_test_runner, suite_name, |
137 build_type): | |
143 """Get a list of absolute paths to test suite targets. | 138 """Get a list of absolute paths to test suite targets. |
144 | 139 |
145 Args: | 140 Args: |
146 exe: if True, use the executable-based test runner. | 141 use_exe_test_runner: if True, use the executable-based test runner. |
147 option_test_suite: the test_suite specified as an option. | 142 suite_name: the suite name specified on the command line. |
148 build_type: 'Release' or 'Debug'. | 143 build_type: 'Release' or 'Debug'. |
149 | 144 |
150 Returns: | 145 Returns: |
151 A list of tuples containing the suite and absolute path. | 146 A list of tuples containing the suite and absolute path. |
152 Ex. ('content_unittests', | 147 Ex. ('content_unittests', |
153 '/tmp/chrome/src/out/Debug/content_unittests_apk/' | 148 '/tmp/chrome/src/out/Debug/content_unittests_apk/' |
154 'content_unittests-debug.apk') | 149 'content_unittests-debug.apk') |
155 | 150 |
156 Raises: | 151 Raises: |
157 Exception: If test suite not found. | 152 Exception: If test suite not found. |
158 """ | 153 """ |
159 def GetQualifiedSuite(suite): | 154 def GetQualifiedSuite(suite): |
160 if suite.is_suite_exe: | 155 if suite.is_suite_exe: |
161 relpath = suite.name | 156 relpath = suite.name |
162 else: | 157 else: |
163 # out/(Debug|Release)/$SUITE_apk/$SUITE-debug.apk | 158 # out/(Debug|Release)/$SUITE_apk/$SUITE-debug.apk |
164 relpath = os.path.join(suite.name + '_apk', suite.name + '-debug.apk') | 159 relpath = os.path.join(suite.name + '_apk', suite.name + '-debug.apk') |
165 return suite.name, os.path.join(test_suite_dir, relpath) | 160 return suite.name, os.path.join(test_suite_dir, relpath) |
166 | 161 |
167 test_suite_dir = os.path.join(cmd_helper.OutDirectory.get(), build_type) | 162 test_suite_dir = os.path.join(cmd_helper.OutDirectory.get(), build_type) |
168 if option_test_suite: | 163 if suite_name: |
169 all_test_suites = [gtest_config.Suite(exe, option_test_suite)] | 164 all_test_suites = [gtest_config.Suite(use_exe_test_runner, suite_name)] |
170 else: | 165 else: |
171 all_test_suites = gtest_config.STABLE_TEST_SUITES | 166 all_test_suites = gtest_config.STABLE_TEST_SUITES |
172 | 167 |
173 # List of tuples (suite_name, suite_path) | 168 # List of tuples (suite_name, suite_path) |
174 qualified_test_suites = map(GetQualifiedSuite, all_test_suites) | 169 qualified_test_suites = map(GetQualifiedSuite, all_test_suites) |
175 | 170 |
176 for t, q in qualified_test_suites: | 171 for t, q in qualified_test_suites: |
177 if not os.path.exists(q): | 172 if not os.path.exists(q): |
178 raise Exception('Test suite %s not found in %s.\n' | 173 raise Exception('Test suite %s not found in %s.\n' |
179 'Supported test suites:\n %s\n' | 174 'Supported test suites:\n %s\n' |
(...skipping 20 matching lines...) Expand all Loading... | |
200 all_tests) | 195 all_tests) |
201 | 196 |
202 | 197 |
203 def GetAllEnabledTests(runner_factory, devices): | 198 def GetAllEnabledTests(runner_factory, devices): |
204 """Get all enabled tests. | 199 """Get all enabled tests. |
205 | 200 |
206 Obtains a list of enabled tests from the test package on the device, | 201 Obtains a list of enabled tests from the test package on the device, |
207 then filters it again using the disabled list on the host. | 202 then filters it again using the disabled list on the host. |
208 | 203 |
209 Args: | 204 Args: |
210 runner_factory: callable that takes a devices and returns a TestRunner. | 205 runner_factory: callable that takes device and shard_index and returns |
211 devices: list of devices. | 206 a TestRunner. |
207 devices: a list of device ids. | |
212 | 208 |
213 Returns: | 209 Returns: |
214 List of all enabled tests. | 210 List of all enabled tests. |
215 | 211 |
216 Raises: | 212 Raises: |
217 Exception: If no devices available. | 213 Exception: If no devices available. |
218 """ | 214 """ |
219 for device in devices: | 215 for device in devices: |
220 try: | 216 try: |
221 logging.info('Obtaining tests from %s', device) | 217 logging.info('Obtaining tests from %s', device) |
222 runner = runner_factory(device, 0) | 218 runner = runner_factory(device, 0) |
223 return GetTestsFromDevice(runner) | 219 return GetTestsFromDevice(runner) |
224 except Exception as e: | 220 except Exception as e: |
225 logging.warning('Failed obtaining tests from %s with exception: %s', | 221 logging.warning('Failed obtaining tests from %s with exception: %s', |
226 device, e) | 222 device, e) |
227 raise Exception('No device available to get the list of tests.') | 223 raise Exception('No device available to get the list of tests.') |
228 | 224 |
229 | 225 |
230 def _RunATestSuite(options, suite_name): | 226 def Setup(use_exe_test_runner, suite_name, test_arguments, timeout, |
231 """Run a single test suite. | 227 cleanup_test_files, tool, build_type, webkit, push_deps, |
232 | 228 gtest_filter): |
233 Helper for Dispatch() to allow stop/restart of the emulator across | 229 """Create the test runner factory and tests. |
234 test bundles. If using the emulator, we start it on entry and stop | |
235 it on exit. | |
236 | 230 |
237 Args: | 231 Args: |
238 options: options for running the tests. | 232 use_exe_test_runner: if True, use the executable-based test runner. |
239 suite_name: name of the test suite being run. | 233 suite_name: the suite name specified on the command line. |
234 test_arguments: Additional arguments to pass to the test binary. | |
235 timeout: Timeout for each test. | |
236 cleanup_test_files: Whether or not to cleanup test files on device. | |
237 tool: Name of the Valgrind tool. | |
238 build_type: 'Release' or 'Debug'. | |
239 webkit: Whether the suite is being run from a WebKit checkout. | |
240 push_deps: If True, push all dependencies to the device. | |
241 gtest_filter: filter for tests. | |
240 | 242 |
241 Returns: | 243 Returns: |
242 A tuple of (base_test_result.TestRunResult object, exit code). | 244 A dictionary from suite names to tuples of (TestRunnerFactory, tests). |
245 """ | |
246 all_test_suites = _FullyQualifiedTestSuites(use_exe_test_runner, suite_name, | |
247 build_type) | |
248 # TODO(gkanwar): This breaks the abstraction of having dispatch.py deal | |
249 # entirely with the devices. Can we do this another way? | |
250 attached_devices = android_commands.GetAttachedDevices() | |
243 | 251 |
244 Raises: | 252 tests_dict = {} |
245 Exception: For various reasons including device failure or failing to reset | |
246 the test server port. | |
247 """ | |
248 attached_devices = [] | |
249 buildbot_emulators = [] | |
250 | 253 |
251 if options.use_emulator: | 254 for suite_name, suite_path in all_test_suites: |
252 buildbot_emulators = emulator.LaunchEmulators(options.emulator_count, | 255 deps_dir = _GenerateDepsDirUsingIsolate(suite_name, build_type) |
frankf
2013/07/16 00:02:18
this won't work. You're just overwriting the deps
gkanwar
2013/07/16 17:44:36
Moved dealing with no test suite specified into te
| |
253 options.abi, | 256 # Constructs a new TestRunner with the current options. |
254 wait_for_boot=True) | 257 def TestRunnerFactory(device, shard_index): |
255 attached_devices = [e.device for e in buildbot_emulators] | 258 return test_runner.TestRunner( |
256 elif options.test_device: | 259 device, |
257 attached_devices = [options.test_device] | 260 suite_path, |
258 else: | 261 test_arguments, |
259 attached_devices = android_commands.GetAttachedDevices() | 262 timeout, |
263 cleanup_test_files, | |
264 tool, | |
265 build_type, | |
266 webkit, | |
267 push_deps, | |
268 constants.GTEST_TEST_PACKAGE_NAME, | |
269 constants.GTEST_TEST_ACTIVITY_NAME, | |
270 constants.GTEST_COMMAND_LINE_FILE, | |
271 deps_dir=deps_dir) | |
260 | 272 |
261 if not attached_devices: | 273 # Get tests and split them up based on the number of devices. |
262 raise Exception('A device must be attached and online.') | 274 # TODO(gkanwar): Sharding shouldn't happen here. |
275 if gtest_filter: | |
276 all_tests = [t for t in gtest_filter.split(':') if t] | |
277 else: | |
278 all_tests = GetAllEnabledTests(TestRunnerFactory, attached_devices) | |
279 num_devices = len(attached_devices) | |
280 tests = [':'.join(all_tests[i::num_devices]) for i in xrange(num_devices)] | |
281 tests = [t for t in tests if t] | |
263 | 282 |
264 # Reset the test port allocation. It's important to do it before starting | 283 tests_dict[suite_name] = (TestRunnerFactory, tests) |
265 # to dispatch any tests. | |
266 if not ports.ResetTestServerPortAllocation(): | |
267 raise Exception('Failed to reset test server port.') | |
268 | 284 |
269 deps_dir = _GenerateDepsDirUsingIsolate(suite_name, options.build_type) | 285 # TODO(gkanwar): Once the suite option becomes required test_dict should |
270 | 286 # just be a single tuple. |
271 # Constructs a new TestRunner with the current options. | 287 return tests_dict |
272 def RunnerFactory(device, shard_index): | |
273 return test_runner.TestRunner( | |
274 device, | |
275 options.test_suite, | |
276 options.test_arguments, | |
277 options.timeout, | |
278 options.cleanup_test_files, | |
279 options.tool, | |
280 options.build_type, | |
281 options.webkit, | |
282 options.push_deps, | |
283 constants.GTEST_TEST_PACKAGE_NAME, | |
284 constants.GTEST_TEST_ACTIVITY_NAME, | |
285 constants.GTEST_COMMAND_LINE_FILE, | |
286 deps_dir=deps_dir) | |
287 | |
288 # Get tests and split them up based on the number of devices. | |
289 if options.test_filter: | |
290 all_tests = [t for t in options.test_filter.split(':') if t] | |
291 else: | |
292 all_tests = GetAllEnabledTests(RunnerFactory, attached_devices) | |
293 num_devices = len(attached_devices) | |
294 tests = [':'.join(all_tests[i::num_devices]) for i in xrange(num_devices)] | |
295 tests = [t for t in tests if t] | |
296 | |
297 # Run tests. | |
298 test_results, exit_code = shard.ShardAndRunTests( | |
299 RunnerFactory, attached_devices, tests, options.build_type, | |
300 test_timeout=None, num_retries=options.num_retries) | |
301 | |
302 report_results.LogFull( | |
303 results=test_results, | |
304 test_type='Unit test', | |
305 test_package=suite_name, | |
306 build_type=options.build_type, | |
307 flakiness_server=options.flakiness_dashboard_server) | |
308 | |
309 for buildbot_emulator in buildbot_emulators: | |
310 buildbot_emulator.Shutdown() | |
311 | |
312 return (test_results, exit_code) | |
313 | |
314 | |
315 def _ListTestSuites(): | |
316 """Display a list of available test suites.""" | |
317 print 'Available test suites are:' | |
318 for test_suite in gtest_config.STABLE_TEST_SUITES: | |
319 print test_suite | |
320 | |
321 | |
322 def Dispatch(options): | |
323 """Dispatches the tests, sharding if possible. | |
324 | |
325 If options.use_emulator is True, all tests will be run in new emulator | |
326 instance. | |
327 | |
328 Args: | |
329 options: options for running the tests. | |
330 | |
331 Returns: | |
332 base_test_result.TestRunResults object with the results of running the tests | |
333 """ | |
334 results = base_test_result.TestRunResults() | |
335 | |
336 if options.test_suite == 'help': | |
337 _ListTestSuites() | |
338 return (results, 0) | |
339 | |
340 if options.use_xvfb: | |
341 framebuffer = xvfb.Xvfb() | |
342 framebuffer.Start() | |
343 | |
344 all_test_suites = _FullyQualifiedTestSuites(options.exe, options.test_suite, | |
345 options.build_type) | |
346 exit_code = 0 | |
347 for suite_name, suite_path in all_test_suites: | |
348 # Give each test suite its own copy of options. | |
349 test_options = copy.deepcopy(options) | |
350 test_options.test_suite = suite_path | |
351 test_results, test_exit_code = _RunATestSuite(test_options, suite_name) | |
352 results.AddTestRunResults(test_results) | |
353 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE: | |
354 exit_code = test_exit_code | |
355 | |
356 if options.use_xvfb: | |
357 framebuffer.Stop() | |
358 | |
359 return (results, exit_code) | |
OLD | NEW |