OLD | NEW |
| (Empty) |
1 #!/usr/bin/env python | |
2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | |
3 # Use of this source code is governed by a BSD-style license that can be | |
4 # found in the LICENSE file. | |
5 | |
6 import collections | |
7 import glob | |
8 import json | |
9 import optparse | |
10 import os | |
11 import pipes | |
12 import shutil | |
13 import subprocess | |
14 import sys | |
15 | |
16 sys.path.append(os.path.join(os.path.dirname(__file__), '..')) | |
17 from pylib import buildbot_report | |
18 from pylib import constants | |
19 from pylib.gtest import gtest_config | |
20 | |
21 | |
22 TESTING = 'BUILDBOT_TESTING' in os.environ | |
23 | |
24 CHROME_SRC = constants.CHROME_DIR | |
25 | |
26 # Describes an instrumation test suite: | |
27 # test: Name of test we're running. | |
28 # apk: apk to be installed. | |
29 # apk_package: package for the apk to be installed. | |
30 # test_apk: apk to run tests on. | |
31 # test_data: data folder in format destination:source. | |
32 I_TEST = collections.namedtuple('InstrumentationTest', [ | |
33 'name', 'apk', 'apk_package', 'test_apk', 'test_data']) | |
34 | |
35 INSTRUMENTATION_TESTS = dict((suite.name, suite) for suite in [ | |
36 I_TEST('ContentShell', | |
37 'ContentShell.apk', | |
38 'org.chromium.content_shell', | |
39 'ContentShellTest', | |
40 'content:content/test/data/android/device_files'), | |
41 I_TEST('ChromiumTestShell', | |
42 'ChromiumTestShell.apk', | |
43 'org.chromium.chrome.testshell', | |
44 'ChromiumTestShellTest', | |
45 'chrome:chrome/test/data/android/device_files'), | |
46 I_TEST('AndroidWebView', | |
47 'AndroidWebView.apk', | |
48 'org.chromium.android_webview', | |
49 'AndroidWebViewTest', | |
50 'webview:android_webview/test/data/device_files'), | |
51 ]) | |
52 | |
53 VALID_TESTS = set(['ui', 'unit', 'webkit', 'webkit_layout']) | |
54 | |
55 | |
56 def SpawnCmd(command): | |
57 """Spawn a process without waiting for termination.""" | |
58 # Add adb binary to path. In the future, might use build_internal copy. | |
59 env = dict(os.environ) | |
60 env['PATH'] = os.pathsep.join([ | |
61 env['PATH'], os.path.join(constants.ANDROID_SDK_ROOT, 'platform-tools')]) | |
62 print '>', ' '.join(map(pipes.quote, command)) | |
63 sys.stdout.flush() | |
64 if TESTING: | |
65 class MockPopen(object): | |
66 @staticmethod | |
67 def wait(): | |
68 return 0 | |
69 return MockPopen() | |
70 | |
71 return subprocess.Popen(command, cwd=CHROME_SRC, env=env) | |
72 | |
73 def RunCmd(command, flunk_on_failure=True): | |
74 """Run a command relative to the chrome source root.""" | |
75 code = SpawnCmd(command).wait() | |
76 print '<', ' '.join(map(pipes.quote, command)) | |
77 if code != 0: | |
78 print 'ERROR: non-zero status %d from %s' % (code, command) | |
79 if flunk_on_failure: | |
80 buildbot_report.PrintError() | |
81 else: | |
82 buildbot_report.PrintWarning() | |
83 return code | |
84 | |
85 | |
86 def RunTestSuites(options, suites): | |
87 """Manages an invocation of run_tests.py. | |
88 | |
89 Args: | |
90 options: options object. | |
91 suites: List of suites to run. | |
92 """ | |
93 args = ['--verbose'] | |
94 if options.target == 'Release': | |
95 args.append('--release') | |
96 if options.asan: | |
97 args.append('--tool=asan') | |
98 for suite in suites: | |
99 buildbot_report.PrintNamedStep(suite) | |
100 RunCmd(['build/android/run_tests.py', '-s', suite] + args) | |
101 | |
102 | |
103 def InstallApk(options, test, print_step=False): | |
104 """Install an apk to all phones. | |
105 | |
106 Args: | |
107 options: options object | |
108 test: An I_TEST namedtuple | |
109 print_step: Print a buildbot step | |
110 """ | |
111 if print_step: | |
112 buildbot_report.PrintNamedStep('install_%s' % test.name.lower()) | |
113 args = ['--apk', test.apk, '--apk_package', test.apk_package] | |
114 if options.target == 'Release': | |
115 args.append('--release') | |
116 | |
117 RunCmd(['build/android/adb_install_apk.py'] + args) | |
118 | |
119 | |
120 def RunInstrumentationSuite(options, test): | |
121 """Manages an invocation of run_instrumentaiton_tests.py. | |
122 | |
123 Args: | |
124 options: options object | |
125 test: An I_TEST namedtuple | |
126 """ | |
127 buildbot_report.PrintNamedStep('%s_instrumentation_tests' % test.name.lower()) | |
128 | |
129 InstallApk(options, test) | |
130 args = ['--test-apk', test.test_apk, '--test_data', test.test_data, '-vvv', | |
131 '-I'] | |
132 if options.target == 'Release': | |
133 args.append('--release') | |
134 if options.asan: | |
135 args.append('--tool=asan') | |
136 | |
137 RunCmd(['build/android/run_instrumentation_tests.py'] + args) | |
138 | |
139 | |
140 def RunWebkitLint(target): | |
141 """Lint WebKit's TestExpectation files.""" | |
142 buildbot_report.PrintNamedStep('webkit_lint') | |
143 RunCmd(['webkit/tools/layout_tests/run_webkit_tests.py', | |
144 '--lint-test-files', | |
145 '--chromium', | |
146 '--target', target]) | |
147 | |
148 | |
149 def RunWebkitLayoutTests(options): | |
150 """Run layout tests on an actual device.""" | |
151 buildbot_report.PrintNamedStep('webkit_tests') | |
152 RunCmd(['webkit/tools/layout_tests/run_webkit_tests.py', | |
153 '--no-show-results', | |
154 '--no-new-test-results', | |
155 '--full-results-html', | |
156 '--clobber-old-results', | |
157 '--exit-after-n-failures', '5000', | |
158 '--exit-after-n-crashes-or-timeouts', '100', | |
159 '--debug-rwt-logging', | |
160 '--results-directory', '..layout-test-results', | |
161 '--target', options.target, | |
162 '--builder-name', options.build_properties.get('buildername', ''), | |
163 '--build-number', options.build_properties.get('buildnumber', ''), | |
164 '--master-name', options.build_properties.get('mastername', ''), | |
165 '--build-name', options.build_properties.get('buildername', ''), | |
166 '--platform=chromium-android', | |
167 '--test-results-server', | |
168 options.factory_properties.get('test_results_server', '')]) | |
169 | |
170 | |
171 def MainTestWrapper(options): | |
172 # Device check and alert emails | |
173 buildbot_report.PrintNamedStep('device_status_check') | |
174 RunCmd(['build/android/device_status_check.py'], flunk_on_failure=False) | |
175 | |
176 if options.install: | |
177 test_obj = INSTRUMENTATION_TESTS[options.install] | |
178 InstallApk(options, test_obj, print_step=True) | |
179 | |
180 if not options.test_filter: | |
181 return | |
182 | |
183 # Spawn logcat monitor | |
184 logcat_dir = os.path.join(CHROME_SRC, 'out/logcat') | |
185 shutil.rmtree(logcat_dir, ignore_errors=True) | |
186 SpawnCmd(['build/android/adb_logcat_monitor.py', logcat_dir]) | |
187 | |
188 if 'unit' in options.test_filter: | |
189 RunTestSuites(options, gtest_config.STABLE_TEST_SUITES) | |
190 if 'ui' in options.test_filter: | |
191 for test in INSTRUMENTATION_TESTS.itervalues(): | |
192 RunInstrumentationSuite(options, test) | |
193 if 'webkit' in options.test_filter: | |
194 RunTestSuites(options, ['webkit_unit_tests', 'TestWebKitAPI']) | |
195 RunWebkitLint(options.target) | |
196 if 'webkit_layout' in options.test_filter: | |
197 RunWebkitLayoutTests(options) | |
198 | |
199 if options.experimental: | |
200 RunTestSuites(options, gtest_config.EXPERIMENTAL_TEST_SUITES) | |
201 | |
202 # Print logcat, kill logcat monitor | |
203 buildbot_report.PrintNamedStep('logcat_dump') | |
204 RunCmd(['build/android/adb_logcat_printer.py', logcat_dir]) | |
205 | |
206 buildbot_report.PrintNamedStep('test_report') | |
207 for report in glob.glob( | |
208 os.path.join(CHROME_SRC, 'out', options.target, 'test_logs', '*.log')): | |
209 subprocess.Popen(['cat', report]).wait() | |
210 os.remove(report) | |
211 | |
212 | |
213 def main(argv): | |
214 parser = optparse.OptionParser() | |
215 | |
216 def convert_json(option, _, value, parser): | |
217 setattr(parser.values, option.dest, json.loads(value)) | |
218 | |
219 parser.add_option('--build-properties', action='callback', | |
220 callback=convert_json, type='string', default={}, | |
221 help='build properties in JSON format') | |
222 parser.add_option('--factory-properties', action='callback', | |
223 callback=convert_json, type='string', default={}, | |
224 help='factory properties in JSON format') | |
225 parser.add_option('--slave-properties', action='callback', | |
226 callback=convert_json, type='string', default={}, | |
227 help='Properties set by slave script in JSON format') | |
228 parser.add_option('--experimental', action='store_true', | |
229 help='Run experiemental tests') | |
230 parser.add_option('-f', '--test-filter', metavar='<filter>', default=[], | |
231 action='append', | |
232 help=('Run a test suite. Test suites: "%s"' % | |
233 '", "'.join(VALID_TESTS))) | |
234 parser.add_option('--asan', action='store_true', help='Run tests with asan.') | |
235 parser.add_option('--install', metavar='<apk name>', | |
236 help='Install an apk by name') | |
237 options, args = parser.parse_args(argv[1:]) | |
238 | |
239 def ParserError(msg): | |
240 """We avoid parser.error because it calls sys.exit.""" | |
241 parser.print_help() | |
242 print >> sys.stderr, '\nERROR:', msg | |
243 return 1 | |
244 | |
245 if args: | |
246 return ParserError('Unused args %s' % args) | |
247 | |
248 unknown_tests = set(options.test_filter) - VALID_TESTS | |
249 if unknown_tests: | |
250 return ParserError('Unknown tests %s' % list(unknown_tests)) | |
251 | |
252 setattr(options, 'target', options.factory_properties.get('target', 'Debug')) | |
253 | |
254 MainTestWrapper(options) | |
255 | |
256 | |
257 if __name__ == '__main__': | |
258 sys.exit(main(sys.argv)) | |
OLD | NEW |