OLD | NEW |
---|---|
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 import collections | 6 import collections |
7 import glob | 7 import glob |
8 import multiprocessing | 8 import multiprocessing |
9 import os | 9 import os |
10 import shutil | 10 import shutil |
11 import sys | 11 import sys |
12 | 12 |
13 import bb_utils | 13 import bb_utils |
14 import bb_annotations | 14 import bb_annotations |
15 | 15 |
16 sys.path.append(os.path.join(os.path.dirname(__file__), '..')) | 16 sys.path.append(os.path.join(os.path.dirname(__file__), '..')) |
17 import provision_devices | 17 import provision_devices |
18 from pylib import android_commands | 18 from pylib import android_commands |
19 from pylib import constants | 19 from pylib import constants |
20 from pylib.gtest import gtest_config | 20 from pylib.gtest import gtest_config |
21 | 21 |
22 sys.path.append(os.path.join( | 22 sys.path.append(os.path.join( |
23 constants.DIR_SOURCE_ROOT, 'third_party', 'android_testrunner')) | 23 constants.DIR_SOURCE_ROOT, 'third_party', 'android_testrunner')) |
24 import errors | 24 import errors |
25 | 25 |
26 | 26 |
27 SLAVE_SCRIPTS_DIR = os.path.join(bb_utils.BB_BUILD_DIR, 'scripts', 'slave') | |
27 CHROME_SRC = constants.DIR_SOURCE_ROOT | 28 CHROME_SRC = constants.DIR_SOURCE_ROOT |
28 LOGCAT_DIR = os.path.join(CHROME_SRC, 'out', 'logcat') | 29 LOGCAT_DIR = os.path.join(CHROME_SRC, 'out', 'logcat') |
30 REVISION_FILENAME = 'FULL_BUILD_REVISION' | |
bulach
2013/08/20 17:42:34
what's this?
it'd be nice to reuse the one we have
| |
29 | 31 |
30 # Describes an instrumation test suite: | 32 # Describes an instrumation test suite: |
31 # test: Name of test we're running. | 33 # test: Name of test we're running. |
32 # apk: apk to be installed. | 34 # apk: apk to be installed. |
33 # apk_package: package for the apk to be installed. | 35 # apk_package: package for the apk to be installed. |
34 # test_apk: apk to run tests on. | 36 # test_apk: apk to run tests on. |
35 # test_data: data folder in format destination:source. | 37 # test_data: data folder in format destination:source. |
36 # host_driven_root: The host-driven test root directory. | 38 # host_driven_root: The host-driven test root directory. |
37 # annotation: Annotation of the tests to include. | 39 # annotation: Annotation of the tests to include. |
38 # exclude_annotation: The annotation of the tests to exclude. | 40 # exclude_annotation: The annotation of the tests to exclude. |
39 I_TEST = collections.namedtuple('InstrumentationTest', [ | 41 I_TEST = collections.namedtuple('InstrumentationTest', [ |
40 'name', 'apk', 'apk_package', 'test_apk', 'test_data', 'host_driven_root', | 42 'name', 'apk', 'apk_package', 'test_apk', 'test_data', 'coverage', |
41 'annotation', 'exclude_annotation', 'extra_flags']) | 43 'host_driven_root', 'annotation', 'exclude_annotation', 'extra_flags']) |
42 | 44 |
43 def I(name, apk, apk_package, test_apk, test_data, host_driven_root=None, | 45 def I(name, apk, apk_package, test_apk, test_data, coverage=True, |
44 annotation=None, exclude_annotation=None, extra_flags=None): | 46 host_driven_root=None, annotation=None, exclude_annotation=None, |
45 return I_TEST(name, apk, apk_package, test_apk, test_data, host_driven_root, | 47 extra_flags=None): |
46 annotation, exclude_annotation, extra_flags) | 48 return I_TEST(name, apk, apk_package, test_apk, test_data, coverage, |
49 host_driven_root, annotation, exclude_annotation, extra_flags) | |
47 | 50 |
48 INSTRUMENTATION_TESTS = dict((suite.name, suite) for suite in [ | 51 INSTRUMENTATION_TESTS = dict((suite.name, suite) for suite in [ |
49 I('ContentShell', | 52 I('ContentShell', |
50 'ContentShell.apk', | 53 'ContentShell.apk', |
51 'org.chromium.content_shell_apk', | 54 'org.chromium.content_shell_apk', |
52 'ContentShellTest', | 55 'ContentShellTest', |
53 'content:content/test/data/android/device_files'), | 56 'content:content/test/data/android/device_files', |
57 annotation='SmallTest'), | |
bulach
2013/08/20 17:42:34
why only SmallTest?
| |
54 I('ChromiumTestShell', | 58 I('ChromiumTestShell', |
55 'ChromiumTestShell.apk', | 59 'ChromiumTestShell.apk', |
56 'org.chromium.chrome.testshell', | 60 'org.chromium.chrome.testshell', |
57 'ChromiumTestShellTest', | 61 'ChromiumTestShellTest', |
58 'chrome:chrome/test/data/android/device_files', | 62 'chrome:chrome/test/data/android/device_files', |
59 constants.CHROMIUM_TEST_SHELL_HOST_DRIVEN_DIR), | 63 host_driven_root=constants.CHROMIUM_TEST_SHELL_HOST_DRIVEN_DIR, |
bulach
2013/08/20 17:42:34
unrelated?
| |
64 annotation='SmallTest'), | |
60 I('AndroidWebView', | 65 I('AndroidWebView', |
61 'AndroidWebView.apk', | 66 'AndroidWebView.apk', |
62 'org.chromium.android_webview.shell', | 67 'org.chromium.android_webview.shell', |
63 'AndroidWebViewTest', | 68 'AndroidWebViewTest', |
64 'webview:android_webview/test/data/device_files'), | 69 'webview:android_webview/test/data/device_files', |
70 annotation='SmallTest'), | |
65 ]) | 71 ]) |
66 | 72 |
67 VALID_TESTS = set(['chromedriver', 'ui', 'unit', 'webkit', 'webkit_layout', | 73 VALID_TESTS = set(['chromedriver', 'ui', 'unit', 'webkit', 'webkit_layout', |
68 'webrtc']) | 74 'webrtc']) |
69 | 75 |
70 RunCmd = bb_utils.RunCmd | 76 RunCmd = bb_utils.RunCmd |
71 | 77 |
72 | 78 |
73 # multiprocessing map_async requires a top-level function for pickle library. | 79 # multiprocessing map_async requires a top-level function for pickle library. |
74 def RebootDeviceSafe(device): | 80 def RebootDeviceSafe(device): |
(...skipping 83 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
158 InstallApk(options, test) | 164 InstallApk(options, test) |
159 args = ['--test-apk', test.test_apk, '--test_data', test.test_data, | 165 args = ['--test-apk', test.test_apk, '--test_data', test.test_data, |
160 '--verbose'] | 166 '--verbose'] |
161 if options.target == 'Release': | 167 if options.target == 'Release': |
162 args.append('--release') | 168 args.append('--release') |
163 if options.asan: | 169 if options.asan: |
164 args.append('--tool=asan') | 170 args.append('--tool=asan') |
165 if options.flakiness_server: | 171 if options.flakiness_server: |
166 args.append('--flakiness-dashboard-server=%s' % | 172 args.append('--flakiness-dashboard-server=%s' % |
167 options.flakiness_server) | 173 options.flakiness_server) |
174 if options.coverage_bucket and test.coverage: | |
175 args.append('--coverage-dir=%s' % constants.GetCoverageDir(options.target)) | |
168 if test.host_driven_root: | 176 if test.host_driven_root: |
169 args.append('--host-driven-root=%s' % test.host_driven_root) | 177 args.append('--host-driven-root=%s' % test.host_driven_root) |
170 if test.annotation: | 178 if test.annotation: |
171 args.extend(['-A', test.annotation]) | 179 args.extend(['-A', test.annotation]) |
172 if test.exclude_annotation: | 180 if test.exclude_annotation: |
173 args.extend(['-E', test.exclude_annotation]) | 181 args.extend(['-E', test.exclude_annotation]) |
174 if test.extra_flags: | 182 if test.extra_flags: |
175 args.extend(test.extra_flags) | 183 args.extend(test.extra_flags) |
176 if python_only: | 184 if python_only: |
177 args.append('-p') | 185 args.append('-p') |
(...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
264 ('provision_devices', ProvisionDevices), | 272 ('provision_devices', ProvisionDevices), |
265 ('device_status_check', DeviceStatusCheck) | 273 ('device_status_check', DeviceStatusCheck) |
266 ] | 274 ] |
267 | 275 |
268 | 276 |
269 def RunUnitTests(options): | 277 def RunUnitTests(options): |
270 RunTestSuites(options, gtest_config.STABLE_TEST_SUITES) | 278 RunTestSuites(options, gtest_config.STABLE_TEST_SUITES) |
271 | 279 |
272 | 280 |
273 def RunInstrumentationTests(options): | 281 def RunInstrumentationTests(options): |
282 coverage_dir = constants.GetCoverageDir(options.target) | |
283 if options.coverage_bucket: | |
284 shutil.rmtree(coverage_dir, ignore_errors=True) | |
285 os.mkdir(coverage_dir) | |
286 | |
274 for test in INSTRUMENTATION_TESTS.itervalues(): | 287 for test in INSTRUMENTATION_TESTS.itervalues(): |
275 RunInstrumentationSuite(options, test) | 288 RunInstrumentationSuite(options, test) |
276 | 289 |
290 if options.coverage_bucket: | |
291 print 'generate_emma_html perms: %s' % RunCmd( | |
292 ['ls', '-l', 'build/android/generate_emma_html.py']) | |
bulach
2013/08/20 17:42:34
not needed?
| |
293 RunCmd(['build/android/generate_emma_html.py', | |
294 '--coverage-dir', coverage_dir, | |
295 '--metadata-dir', os.path.join(constants.DIR_SOURCE_ROOT, | |
296 'out', options.target), | |
297 '--output', os.path.join(coverage_dir, 'coverage.html')]) | |
298 | |
299 revision_file = os.path.join(CHROME_SRC, 'out', options.target, | |
300 REVISION_FILENAME) | |
301 # If the revision file doesn't exist, we're just testing the bot | |
302 if os.path.exists(revision_file): | |
303 with open(revision_file) as f: | |
304 revision = f.read() | |
305 else: | |
306 revision = 'testing' | |
307 bot_id = os.environ.get('BUILDBOT_ID') | |
308 if not bot_id: | |
309 print 'BUILDBOT_ID requried in the environment to upload coverage data.' | |
310 return | |
311 | |
312 boto_config = os.path.join(bb_utils.BB_BUILD_DIR, 'site_config', '.boto') | |
313 os.environ['AWS_CREDENTIAL_FILE'] = boto_config | |
314 os.environ['BOTO_CONFIG'] = boto_config | |
315 RunCmd([os.path.join(bb_utils.BB_BUILD_DIR, 'third_party', | |
316 'gsutil', 'gsutil'), 'cp', '-R', | |
317 coverage_dir, 'gs://%s/%s/%s' % | |
318 (options.coverage_bucket, bot_id, revision)]) | |
319 bb_annotations.PrintLink( | |
320 'Coverage report', | |
321 'https://storage.googleapis.com/%s/%s/coverage.html' | |
322 % (options.coverage_bucket, revision)) | |
323 | |
277 | 324 |
278 def RunWebkitTests(options): | 325 def RunWebkitTests(options): |
279 RunTestSuites(options, ['webkit_unit_tests']) | 326 RunTestSuites(options, ['webkit_unit_tests']) |
280 RunWebkitLint(options.target) | 327 RunWebkitLint(options.target) |
281 | 328 |
282 | 329 |
283 def RunWebRTCTests(options): | 330 def RunWebRTCTests(options): |
284 RunTestSuites(options, gtest_config.WEBRTC_TEST_SUITES) | 331 RunTestSuites(options, gtest_config.WEBRTC_TEST_SUITES) |
285 | 332 |
286 | 333 |
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
348 help='Run experiemental tests') | 395 help='Run experiemental tests') |
349 parser.add_option('-f', '--test-filter', metavar='<filter>', default=[], | 396 parser.add_option('-f', '--test-filter', metavar='<filter>', default=[], |
350 action='append', | 397 action='append', |
351 help=('Run a test suite. Test suites: "%s"' % | 398 help=('Run a test suite. Test suites: "%s"' % |
352 '", "'.join(VALID_TESTS))) | 399 '", "'.join(VALID_TESTS))) |
353 parser.add_option('--asan', action='store_true', help='Run tests with asan.') | 400 parser.add_option('--asan', action='store_true', help='Run tests with asan.') |
354 parser.add_option('--install', metavar='<apk name>', | 401 parser.add_option('--install', metavar='<apk name>', |
355 help='Install an apk by name') | 402 help='Install an apk by name') |
356 parser.add_option('--reboot', action='store_true', | 403 parser.add_option('--reboot', action='store_true', |
357 help='Reboot devices before running tests') | 404 help='Reboot devices before running tests') |
405 parser.add_option('--coverage-bucket', | |
406 help=('Bucket name to store coverage results. Coverage is ' | |
407 'only run if this is set.')) | |
358 parser.add_option( | 408 parser.add_option( |
359 '--flakiness-server', | 409 '--flakiness-server', |
360 help='The flakiness dashboard server to which the results should be ' | 410 help='The flakiness dashboard server to which the results should be ' |
361 'uploaded.') | 411 'uploaded.') |
362 parser.add_option( | 412 parser.add_option( |
363 '--auto-reconnect', action='store_true', | 413 '--auto-reconnect', action='store_true', |
364 help='Push script to device which restarts adbd on disconnections.') | 414 help='Push script to device which restarts adbd on disconnections.') |
365 parser.add_option( | 415 parser.add_option( |
366 '--logcat-dump-output', | 416 '--logcat-dump-output', |
367 help='The logcat dump output will be "tee"-ed into this file') | 417 help='The logcat dump output will be "tee"-ed into this file') |
(...skipping 12 matching lines...) Expand all Loading... | |
380 if unknown_tests: | 430 if unknown_tests: |
381 return sys.exit('Unknown tests %s' % list(unknown_tests)) | 431 return sys.exit('Unknown tests %s' % list(unknown_tests)) |
382 | 432 |
383 setattr(options, 'target', options.factory_properties.get('target', 'Debug')) | 433 setattr(options, 'target', options.factory_properties.get('target', 'Debug')) |
384 | 434 |
385 MainTestWrapper(options) | 435 MainTestWrapper(options) |
386 | 436 |
387 | 437 |
388 if __name__ == '__main__': | 438 if __name__ == '__main__': |
389 sys.exit(main(sys.argv)) | 439 sys.exit(main(sys.argv)) |
OLD | NEW |