Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(174)

Side by Side Diff: build/android/pylib/gtest/setup.py

Issue 18770008: [Android] Redesigns the sharder to allow replicated vs distributed tests (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Adds tagging of tests (for replication) Created 7 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 1 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 """Dispatches GTests.""" 5 """Runs GTests."""
6 6
7 import copy
8 import fnmatch 7 import fnmatch
9 import glob 8 import glob
10 import logging 9 import logging
11 import os 10 import os
12 import shutil 11 import shutil
13 12
14 from pylib import android_commands 13 from pylib import android_commands
15 from pylib import cmd_helper 14 from pylib import cmd_helper
16 from pylib import constants 15 from pylib import constants
17 from pylib import ports
18 from pylib.base import base_test_result 16 from pylib.base import base_test_result
19 from pylib.base import shard
20 from pylib.utils import emulator
21 from pylib.utils import report_results
22 from pylib.utils import xvfb
23 17
24 import gtest_config 18 import gtest_config
25 import test_runner 19 import test_runner
26 20
27 21
28 # TODO(frankf): Add more test targets here after making sure we don't 22 # TODO(frankf): Add more test targets here after making sure we don't
29 # blow up the dependency size (and the world). 23 # blow up the dependency size (and the world).
30 _ISOLATE_FILE_PATHS = { 24 _ISOLATE_FILE_PATHS = {
31 'base_unittests': 'base/base_unittests.isolate', 25 'base_unittests': 'base/base_unittests.isolate',
32 'unit_tests': 'chrome/unit_tests.isolate', 26 'unit_tests': 'chrome/unit_tests.isolate',
(...skipping 99 matching lines...) Expand 10 before | Expand all | Expand 10 after
132 deps_product_dir = os.path.join(deps_dir, 'out', build_type) 126 deps_product_dir = os.path.join(deps_dir, 'out', build_type)
133 if os.path.isdir(deps_product_dir): 127 if os.path.isdir(deps_product_dir):
134 for p in os.listdir(deps_product_dir): 128 for p in os.listdir(deps_product_dir):
135 shutil.move(os.path.join(deps_product_dir, p), deps_dir) 129 shutil.move(os.path.join(deps_product_dir, p), deps_dir)
136 os.rmdir(deps_product_dir) 130 os.rmdir(deps_product_dir)
137 os.rmdir(os.path.join(deps_dir, 'out')) 131 os.rmdir(os.path.join(deps_dir, 'out'))
138 132
139 return deps_dir 133 return deps_dir
140 134
141 135
142 def _FullyQualifiedTestSuites(exe, option_test_suite, build_type): 136 def _FullyQualifiedTestSuites(use_exe_test_runner, suite_name,
137 build_type):
143 """Get a list of absolute paths to test suite targets. 138 """Get a list of absolute paths to test suite targets.
144 139
145 Args: 140 Args:
146 exe: if True, use the executable-based test runner. 141 use_exe_test_runner: if True, use the executable-based test runner.
147 option_test_suite: the test_suite specified as an option. 142 suite_name: the suite name specified on the command line.
148 build_type: 'Release' or 'Debug'. 143 build_type: 'Release' or 'Debug'.
149 144
150 Returns: 145 Returns:
151 A list of tuples containing the suite and absolute path. 146 A list of tuples containing the suite and absolute path.
152 Ex. ('content_unittests', 147 Ex. ('content_unittests',
153 '/tmp/chrome/src/out/Debug/content_unittests_apk/' 148 '/tmp/chrome/src/out/Debug/content_unittests_apk/'
154 'content_unittests-debug.apk') 149 'content_unittests-debug.apk')
155 150
156 Raises: 151 Raises:
157 Exception: If test suite not found. 152 Exception: If test suite not found.
158 """ 153 """
159 def GetQualifiedSuite(suite): 154 def GetQualifiedSuite(suite):
160 if suite.is_suite_exe: 155 if suite.is_suite_exe:
161 relpath = suite.name 156 relpath = suite.name
162 else: 157 else:
163 # out/(Debug|Release)/$SUITE_apk/$SUITE-debug.apk 158 # out/(Debug|Release)/$SUITE_apk/$SUITE-debug.apk
164 relpath = os.path.join(suite.name + '_apk', suite.name + '-debug.apk') 159 relpath = os.path.join(suite.name + '_apk', suite.name + '-debug.apk')
165 return suite.name, os.path.join(test_suite_dir, relpath) 160 return suite.name, os.path.join(test_suite_dir, relpath)
166 161
167 test_suite_dir = os.path.join(cmd_helper.OutDirectory.get(), build_type) 162 test_suite_dir = os.path.join(cmd_helper.OutDirectory.get(), build_type)
168 if option_test_suite: 163 if suite_name:
169 all_test_suites = [gtest_config.Suite(exe, option_test_suite)] 164 all_test_suites = [gtest_config.Suite(use_exe_test_runner, suite_name)]
170 else: 165 else:
171 all_test_suites = gtest_config.STABLE_TEST_SUITES 166 all_test_suites = gtest_config.STABLE_TEST_SUITES
172 167
173 # List of tuples (suite_name, suite_path) 168 # List of tuples (suite_name, suite_path)
174 qualified_test_suites = map(GetQualifiedSuite, all_test_suites) 169 qualified_test_suites = map(GetQualifiedSuite, all_test_suites)
175 170
176 for t, q in qualified_test_suites: 171 for t, q in qualified_test_suites:
177 if not os.path.exists(q): 172 if not os.path.exists(q):
178 raise Exception('Test suite %s not found in %s.\n' 173 raise Exception('Test suite %s not found in %s.\n'
179 'Supported test suites:\n %s\n' 174 'Supported test suites:\n %s\n'
(...skipping 20 matching lines...) Expand all
200 all_tests) 195 all_tests)
201 196
202 197
203 def GetAllEnabledTests(runner_factory, devices): 198 def GetAllEnabledTests(runner_factory, devices):
204 """Get all enabled tests. 199 """Get all enabled tests.
205 200
206 Obtains a list of enabled tests from the test package on the device, 201 Obtains a list of enabled tests from the test package on the device,
207 then filters it again using the disabled list on the host. 202 then filters it again using the disabled list on the host.
208 203
209 Args: 204 Args:
210 runner_factory: callable that takes a devices and returns a TestRunner. 205 runner_factory: callable that takes device and shard_index and returns
211 devices: list of devices. 206 a TestRunner.
207 devices: a list of device ids.
212 208
213 Returns: 209 Returns:
214 List of all enabled tests. 210 List of all enabled tests.
215 211
216 Raises: 212 Raises:
217 Exception: If no devices available. 213 Exception: If no devices available.
218 """ 214 """
219 for device in devices: 215 for device in devices:
220 try: 216 try:
221 logging.info('Obtaining tests from %s', device) 217 logging.info('Obtaining tests from %s', device)
222 runner = runner_factory(device, 0) 218 runner = runner_factory(device, 0)
223 return GetTestsFromDevice(runner) 219 return GetTestsFromDevice(runner)
224 except Exception as e: 220 except Exception as e:
225 logging.warning('Failed obtaining tests from %s with exception: %s', 221 logging.warning('Failed obtaining tests from %s with exception: %s',
226 device, e) 222 device, e)
227 raise Exception('No device available to get the list of tests.') 223 raise Exception('No device available to get the list of tests.')
228 224
229 225
230 def _RunATestSuite(options, suite_name): 226 def Setup(use_exe_test_runner, suite_name, test_arguments, timeout,
231 """Run a single test suite. 227 cleanup_test_files, tool, build_type, webkit, push_deps,
232 228 gtest_filter):
233 Helper for Dispatch() to allow stop/restart of the emulator across 229 """Create the test runner factory and tests.
234 test bundles. If using the emulator, we start it on entry and stop
235 it on exit.
236 230
237 Args: 231 Args:
238 options: options for running the tests. 232 use_exe_test_runner: if True, use the executable-based test runner.
239 suite_name: name of the test suite being run. 233 suite_name: the suite name specified on the command line.
234 test_arguments: Additional arguments to pass to the test binary.
235 timeout: Timeout for each test.
236 cleanup_test_files: Whether or not to cleanup test files on device.
237 tool: Name of the Valgrind tool.
238 build_type: 'Release' or 'Debug'.
239 webkit: Whether the suite is being run from a WebKit checkout.
240 push_deps: If True, push all dependencies to the device.
241 gtest_filter: filter for tests.
240 242
241 Returns: 243 Returns:
242 A tuple of (base_test_result.TestRunResult object, exit code). 244 A dictionary from suite names to tuples of (TestRunnerFactory, tests).
245 """
246 all_test_suites = _FullyQualifiedTestSuites(use_exe_test_runner, suite_name,
247 build_type)
248 # TODO(gkanwar): This breaks the abstraction of having dispatch.py deal
249 # entirely with the devices. Can we do this another way?
250 attached_devices = android_commands.GetAttachedDevices()
243 251
244 Raises: 252 tests_dict = {}
245 Exception: For various reasons including device failure or failing to reset
246 the test server port.
247 """
248 attached_devices = []
249 buildbot_emulators = []
250 253
251 if options.use_emulator: 254 for suite_name, suite_path in all_test_suites:
252 buildbot_emulators = emulator.LaunchEmulators(options.emulator_count, 255 deps_dir = _GenerateDepsDirUsingIsolate(suite_name, build_type)
frankf 2013/07/16 00:02:18 this won't work. You're just overwriting the deps
gkanwar 2013/07/16 17:44:36 Moved dealing with no test suite specified into te
253 options.abi, 256 # Constructs a new TestRunner with the current options.
254 wait_for_boot=True) 257 def TestRunnerFactory(device, shard_index):
255 attached_devices = [e.device for e in buildbot_emulators] 258 return test_runner.TestRunner(
256 elif options.test_device: 259 device,
257 attached_devices = [options.test_device] 260 suite_path,
258 else: 261 test_arguments,
259 attached_devices = android_commands.GetAttachedDevices() 262 timeout,
263 cleanup_test_files,
264 tool,
265 build_type,
266 webkit,
267 push_deps,
268 constants.GTEST_TEST_PACKAGE_NAME,
269 constants.GTEST_TEST_ACTIVITY_NAME,
270 constants.GTEST_COMMAND_LINE_FILE,
271 deps_dir=deps_dir)
260 272
261 if not attached_devices: 273 # Get tests and split them up based on the number of devices.
262 raise Exception('A device must be attached and online.') 274 # TODO(gkanwar): Sharding shouldn't happen here.
275 if gtest_filter:
276 all_tests = [t for t in gtest_filter.split(':') if t]
277 else:
278 all_tests = GetAllEnabledTests(TestRunnerFactory, attached_devices)
279 num_devices = len(attached_devices)
280 tests = [':'.join(all_tests[i::num_devices]) for i in xrange(num_devices)]
281 tests = [t for t in tests if t]
263 282
264 # Reset the test port allocation. It's important to do it before starting 283 tests_dict[suite_name] = (TestRunnerFactory, tests)
265 # to dispatch any tests.
266 if not ports.ResetTestServerPortAllocation():
267 raise Exception('Failed to reset test server port.')
268 284
269 deps_dir = _GenerateDepsDirUsingIsolate(suite_name, options.build_type) 285 # TODO(gkanwar): Once the suite option becomes required test_dict should
270 286 # just be a single tuple.
271 # Constructs a new TestRunner with the current options. 287 return tests_dict
272 def RunnerFactory(device, shard_index):
273 return test_runner.TestRunner(
274 device,
275 options.test_suite,
276 options.test_arguments,
277 options.timeout,
278 options.cleanup_test_files,
279 options.tool,
280 options.build_type,
281 options.webkit,
282 options.push_deps,
283 constants.GTEST_TEST_PACKAGE_NAME,
284 constants.GTEST_TEST_ACTIVITY_NAME,
285 constants.GTEST_COMMAND_LINE_FILE,
286 deps_dir=deps_dir)
287
288 # Get tests and split them up based on the number of devices.
289 if options.test_filter:
290 all_tests = [t for t in options.test_filter.split(':') if t]
291 else:
292 all_tests = GetAllEnabledTests(RunnerFactory, attached_devices)
293 num_devices = len(attached_devices)
294 tests = [':'.join(all_tests[i::num_devices]) for i in xrange(num_devices)]
295 tests = [t for t in tests if t]
296
297 # Run tests.
298 test_results, exit_code = shard.ShardAndRunTests(
299 RunnerFactory, attached_devices, tests, options.build_type,
300 test_timeout=None, num_retries=options.num_retries)
301
302 report_results.LogFull(
303 results=test_results,
304 test_type='Unit test',
305 test_package=suite_name,
306 build_type=options.build_type,
307 flakiness_server=options.flakiness_dashboard_server)
308
309 for buildbot_emulator in buildbot_emulators:
310 buildbot_emulator.Shutdown()
311
312 return (test_results, exit_code)
313
314
315 def _ListTestSuites():
316 """Display a list of available test suites."""
317 print 'Available test suites are:'
318 for test_suite in gtest_config.STABLE_TEST_SUITES:
319 print test_suite
320
321
322 def Dispatch(options):
323 """Dispatches the tests, sharding if possible.
324
325 If options.use_emulator is True, all tests will be run in new emulator
326 instance.
327
328 Args:
329 options: options for running the tests.
330
331 Returns:
332 base_test_result.TestRunResults object with the results of running the tests
333 """
334 results = base_test_result.TestRunResults()
335
336 if options.test_suite == 'help':
337 _ListTestSuites()
338 return (results, 0)
339
340 if options.use_xvfb:
341 framebuffer = xvfb.Xvfb()
342 framebuffer.Start()
343
344 all_test_suites = _FullyQualifiedTestSuites(options.exe, options.test_suite,
345 options.build_type)
346 exit_code = 0
347 for suite_name, suite_path in all_test_suites:
348 # Give each test suite its own copy of options.
349 test_options = copy.deepcopy(options)
350 test_options.test_suite = suite_path
351 test_results, test_exit_code = _RunATestSuite(test_options, suite_name)
352 results.AddTestRunResults(test_results)
353 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
354 exit_code = test_exit_code
355
356 if options.use_xvfb:
357 framebuffer.Stop()
358
359 return (results, exit_code)
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698