Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(812)

Side by Side Diff: build/android/pylib/gtest/dispatch.py

Issue 18233018: [Android] Use isolate remap instead of check. (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/src
Patch Set: Addressed all comments Created 7 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « build/android/pylib/base/base_test_runner.py ('k') | build/android/pylib/gtest/test_package.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 1 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 """Dispatches GTests.""" 5 """Dispatches GTests."""
6 6
7 import copy 7 import copy
8 import fnmatch 8 import fnmatch
9 import glob
9 import logging 10 import logging
10 import os 11 import os
12 import shutil
11 13
12 from pylib import android_commands 14 from pylib import android_commands
13 from pylib import cmd_helper 15 from pylib import cmd_helper
14 from pylib import constants 16 from pylib import constants
15 from pylib import ports 17 from pylib import ports
16 from pylib.base import shard 18 from pylib.base import shard
17 from pylib.utils import emulator 19 from pylib.utils import emulator
18 from pylib.utils import report_results 20 from pylib.utils import report_results
19 from pylib.utils import xvfb 21 from pylib.utils import xvfb
20 22
21 import gtest_config 23 import gtest_config
22 import test_runner 24 import test_runner
23 25
24 26
27 # TODO(frankf): Add more test targets here after making sure we don't
28 # blow up the dependency size (and the world).
29 _ISOLATE_FILE_PATHS = {
30 'base_unittests': 'base/base_unittests.isolate',
31 'unit_tests': 'chrome/unit_tests.isolate',
32 }
33
34 # Used for filtering large data deps at a finer grain than what's allowed in
35 # isolate files since pushing deps to devices is expensive.
36 # Wildcards are allowed.
37 _DEPS_EXCLUSION_LIST = [
38 'chrome/test/data/extensions/api_test',
39 'chrome/test/data/extensions/secure_shell',
40 'chrome/test/data/firefox*',
41 'chrome/test/data/gpu',
42 'chrome/test/data/image_decoding',
43 'chrome/test/data/import',
44 'chrome/test/data/page_cycler',
45 'chrome/test/data/perf',
46 'chrome/test/data/pyauto_private',
47 'chrome/test/data/safari_import',
48 'chrome/test/data/scroll',
49 'chrome/test/data/third_party',
50 'third_party/hunspell_dictionaries/*.dic',
51 ]
52
53 _ISOLATE_SCRIPT = os.path.join(
54 constants.DIR_SOURCE_ROOT, 'tools', 'swarm_client', 'isolate.py')
55
56
57 def _GenerateDepsDirUsingIsolate(test_suite, build_type):
58 """Generate the dependency dir for the test suite using isolate.
59
60 Args:
61 test_suite: The test suite basename (e.g. base_unittests).
62 build_type: Release/Debug
63
64 Returns:
65 If an isolate file exists, returns path to dependency dir on the host.
66 Otherwise, returns False.
67 """
68 product_dir = os.path.join(cmd_helper.OutDirectory.get(), build_type)
69 assert os.path.isabs(product_dir)
70 isolate_rel_path = _ISOLATE_FILE_PATHS.get(test_suite)
71 if not isolate_rel_path:
72 return False
73
74 isolate_abs_path = os.path.join(constants.DIR_SOURCE_ROOT, isolate_rel_path)
75 isolated_abs_path = os.path.join(
76 product_dir, '%s.isolated' % test_suite)
77 assert os.path.exists(isolate_abs_path)
78 deps_dir = os.path.join(product_dir, 'isolate_deps_dir')
79 if os.path.isdir(deps_dir):
80 shutil.rmtree(deps_dir)
81 isolate_cmd = [
82 'python', _ISOLATE_SCRIPT,
83 'remap',
84 '--isolate', isolate_abs_path,
85 '--isolated', isolated_abs_path,
86 '-V', 'PRODUCT_DIR=%s' % product_dir,
87 '-V', 'OS=android',
88 '--outdir', deps_dir,
89 ]
90 assert not cmd_helper.RunCmd(isolate_cmd)
91
92 # We're relying on the fact that timestamps are preserved
93 # by the remap command (hardlinked). Otherwise, all the data
94 # will be pushed to the device once we move to using time diff
95 # instead of md5sum. Perform a sanity check here.
96 for root, _, filenames in os.walk(deps_dir):
97 if filenames:
98 linked_file = os.path.join(root, filenames[0])
99 orig_file = os.path.join(
100 constants.DIR_SOURCE_ROOT,
101 os.path.relpath(linked_file, deps_dir))
102 if os.stat(linked_file).st_ino == os.stat(orig_file).st_ino:
103 break
104 else:
105 raise Exception('isolate remap command did not use hardlinks.')
M-A Ruel 2013/07/11 19:07:10 I'm thinking of changing remap behavior so it does
106
107 # Delete excluded files as defined by _DEPS_EXCLUSION_LIST.
108 old_cwd = os.getcwd()
109 try:
110 os.chdir(deps_dir)
111 excluded_paths = [x for y in _DEPS_EXCLUSION_LIST for x in glob.glob(y)]
112 if excluded_paths:
113 logging.info('Excluding the following from dependency list: %s',
114 excluded_paths)
115 for p in excluded_paths:
116 if os.path.isdir(p):
117 shutil.rmtree(p)
118 else:
119 os.remove(p)
120 finally:
121 os.chdir(old_cwd)
122
123 # On Android, all pak files need to be in the top-level 'paks' directory.
124 paks_dir = os.path.join(deps_dir, 'paks')
125 os.mkdir(paks_dir)
126 for root, _, filenames in os.walk(os.path.join(deps_dir, 'out')):
127 for filename in fnmatch.filter(filenames, '*.pak'):
128 shutil.move(os.path.join(root, filename), paks_dir)
129
130 # Move everything in PRODUCT_DIR to top level.
131 deps_product_dir = os.path.join(deps_dir, 'out', build_type)
132 if os.path.isdir(deps_product_dir):
133 for p in os.listdir(deps_product_dir):
134 shutil.move(os.path.join(deps_product_dir, p), deps_dir)
135 os.rmdir(deps_product_dir)
136 os.rmdir(os.path.join(deps_dir, 'out'))
137
138 return deps_dir
139
140
25 def _FullyQualifiedTestSuites(exe, option_test_suite, build_type): 141 def _FullyQualifiedTestSuites(exe, option_test_suite, build_type):
26 """Get a list of absolute paths to test suite targets. 142 """Get a list of absolute paths to test suite targets.
27 143
28 Args: 144 Args:
29 exe: if True, use the executable-based test runner. 145 exe: if True, use the executable-based test runner.
30 option_test_suite: the test_suite specified as an option. 146 option_test_suite: the test_suite specified as an option.
31 build_type: 'Release' or 'Debug'. 147 build_type: 'Release' or 'Debug'.
32 148
33 Returns: 149 Returns:
34 A list of tuples containing the suite and absolute path. 150 A list of tuples containing the suite and absolute path.
(...skipping 100 matching lines...) Expand 10 before | Expand all | Expand 10 after
135 attached_devices = android_commands.GetAttachedDevices() 251 attached_devices = android_commands.GetAttachedDevices()
136 252
137 if not attached_devices: 253 if not attached_devices:
138 raise Exception('A device must be attached and online.') 254 raise Exception('A device must be attached and online.')
139 255
140 # Reset the test port allocation. It's important to do it before starting 256 # Reset the test port allocation. It's important to do it before starting
141 # to dispatch any tests. 257 # to dispatch any tests.
142 if not ports.ResetTestServerPortAllocation(): 258 if not ports.ResetTestServerPortAllocation():
143 raise Exception('Failed to reset test server port.') 259 raise Exception('Failed to reset test server port.')
144 260
261 deps_dir = _GenerateDepsDirUsingIsolate(suite_name, options.build_type)
262
145 # Constructs a new TestRunner with the current options. 263 # Constructs a new TestRunner with the current options.
146 def RunnerFactory(device, shard_index): 264 def RunnerFactory(device, shard_index):
147 return test_runner.TestRunner( 265 return test_runner.TestRunner(
148 device, 266 device,
149 options.test_suite, 267 options.test_suite,
150 options.test_arguments, 268 options.test_arguments,
151 options.timeout, 269 options.timeout,
152 options.cleanup_test_files, 270 options.cleanup_test_files,
153 options.tool, 271 options.tool,
154 options.build_type, 272 options.build_type,
155 options.webkit, 273 options.webkit,
156 options.push_deps, 274 options.push_deps,
157 constants.GTEST_TEST_PACKAGE_NAME, 275 constants.GTEST_TEST_PACKAGE_NAME,
158 constants.GTEST_TEST_ACTIVITY_NAME, 276 constants.GTEST_TEST_ACTIVITY_NAME,
159 constants.GTEST_COMMAND_LINE_FILE) 277 constants.GTEST_COMMAND_LINE_FILE,
278 deps_dir=deps_dir)
160 279
161 # Get tests and split them up based on the number of devices. 280 # Get tests and split them up based on the number of devices.
162 if options.test_filter: 281 if options.test_filter:
163 all_tests = [t for t in options.test_filter.split(':') if t] 282 all_tests = [t for t in options.test_filter.split(':') if t]
164 else: 283 else:
165 all_tests = GetAllEnabledTests(RunnerFactory, attached_devices) 284 all_tests = GetAllEnabledTests(RunnerFactory, attached_devices)
166 num_devices = len(attached_devices) 285 num_devices = len(attached_devices)
167 tests = [':'.join(all_tests[i::num_devices]) for i in xrange(num_devices)] 286 tests = [':'.join(all_tests[i::num_devices]) for i in xrange(num_devices)]
168 tests = [t for t in tests if t] 287 tests = [t for t in tests if t]
169 288
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after
218 failures = 0 337 failures = 0
219 for suite_name, suite_path in all_test_suites: 338 for suite_name, suite_path in all_test_suites:
220 # Give each test suite its own copy of options. 339 # Give each test suite its own copy of options.
221 test_options = copy.deepcopy(options) 340 test_options = copy.deepcopy(options)
222 test_options.test_suite = suite_path 341 test_options.test_suite = suite_path
223 failures += _RunATestSuite(test_options, suite_name) 342 failures += _RunATestSuite(test_options, suite_name)
224 343
225 if options.use_xvfb: 344 if options.use_xvfb:
226 framebuffer.Stop() 345 framebuffer.Stop()
227 return failures 346 return failures
OLDNEW
« no previous file with comments | « build/android/pylib/base/base_test_runner.py ('k') | build/android/pylib/gtest/test_package.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698