Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(22)

Side by Side Diff: build/android/buildbot/bb_device_steps.py

Issue 20210002: [Android] Sets up a coverage system for java using EMMA (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Fixes gsutil upload Created 7 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 import collections 6 import collections
7 import glob 7 import glob
8 import multiprocessing 8 import multiprocessing
9 import os 9 import os
10 import shutil 10 import shutil
11 import sys 11 import sys
12 12
13 import bb_utils 13 import bb_utils
14 import bb_annotations 14 import bb_annotations
15 15
16 sys.path.append(os.path.join(os.path.dirname(__file__), '..')) 16 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
17 import provision_devices 17 import provision_devices
18 from pylib import android_commands 18 from pylib import android_commands
19 from pylib import constants 19 from pylib import constants
20 from pylib.gtest import gtest_config 20 from pylib.gtest import gtest_config
21 21
22 sys.path.append(os.path.join( 22 sys.path.append(os.path.join(
23 constants.DIR_SOURCE_ROOT, 'third_party', 'android_testrunner')) 23 constants.DIR_SOURCE_ROOT, 'third_party', 'android_testrunner'))
24 import errors 24 import errors
25 25
26 26
27 SLAVE_SCRIPTS_DIR = os.path.join(bb_utils.BB_BUILD_DIR, 'scripts', 'slave')
27 CHROME_SRC = constants.DIR_SOURCE_ROOT 28 CHROME_SRC = constants.DIR_SOURCE_ROOT
28 LOGCAT_DIR = os.path.join(CHROME_SRC, 'out', 'logcat') 29 LOGCAT_DIR = os.path.join(CHROME_SRC, 'out', 'logcat')
30 REVISION_FILENAME = 'FULL_BUILD_REVISION'
bulach 2013/08/20 17:42:34 what's this? it'd be nice to reuse the one we have
29 31
30 # Describes an instrumation test suite: 32 # Describes an instrumation test suite:
31 # test: Name of test we're running. 33 # test: Name of test we're running.
32 # apk: apk to be installed. 34 # apk: apk to be installed.
33 # apk_package: package for the apk to be installed. 35 # apk_package: package for the apk to be installed.
34 # test_apk: apk to run tests on. 36 # test_apk: apk to run tests on.
35 # test_data: data folder in format destination:source. 37 # test_data: data folder in format destination:source.
36 # host_driven_root: The host-driven test root directory. 38 # host_driven_root: The host-driven test root directory.
37 # annotation: Annotation of the tests to include. 39 # annotation: Annotation of the tests to include.
38 # exclude_annotation: The annotation of the tests to exclude. 40 # exclude_annotation: The annotation of the tests to exclude.
39 I_TEST = collections.namedtuple('InstrumentationTest', [ 41 I_TEST = collections.namedtuple('InstrumentationTest', [
40 'name', 'apk', 'apk_package', 'test_apk', 'test_data', 'host_driven_root', 42 'name', 'apk', 'apk_package', 'test_apk', 'test_data', 'coverage',
41 'annotation', 'exclude_annotation', 'extra_flags']) 43 'host_driven_root', 'annotation', 'exclude_annotation', 'extra_flags'])
42 44
43 def I(name, apk, apk_package, test_apk, test_data, host_driven_root=None, 45 def I(name, apk, apk_package, test_apk, test_data, coverage=True,
44 annotation=None, exclude_annotation=None, extra_flags=None): 46 host_driven_root=None, annotation=None, exclude_annotation=None,
45 return I_TEST(name, apk, apk_package, test_apk, test_data, host_driven_root, 47 extra_flags=None):
46 annotation, exclude_annotation, extra_flags) 48 return I_TEST(name, apk, apk_package, test_apk, test_data, coverage,
49 host_driven_root, annotation, exclude_annotation, extra_flags)
47 50
48 INSTRUMENTATION_TESTS = dict((suite.name, suite) for suite in [ 51 INSTRUMENTATION_TESTS = dict((suite.name, suite) for suite in [
49 I('ContentShell', 52 I('ContentShell',
50 'ContentShell.apk', 53 'ContentShell.apk',
51 'org.chromium.content_shell_apk', 54 'org.chromium.content_shell_apk',
52 'ContentShellTest', 55 'ContentShellTest',
53 'content:content/test/data/android/device_files'), 56 'content:content/test/data/android/device_files',
57 annotation='SmallTest'),
bulach 2013/08/20 17:42:34 why only SmallTest?
54 I('ChromiumTestShell', 58 I('ChromiumTestShell',
55 'ChromiumTestShell.apk', 59 'ChromiumTestShell.apk',
56 'org.chromium.chrome.testshell', 60 'org.chromium.chrome.testshell',
57 'ChromiumTestShellTest', 61 'ChromiumTestShellTest',
58 'chrome:chrome/test/data/android/device_files', 62 'chrome:chrome/test/data/android/device_files',
59 constants.CHROMIUM_TEST_SHELL_HOST_DRIVEN_DIR), 63 host_driven_root=constants.CHROMIUM_TEST_SHELL_HOST_DRIVEN_DIR,
bulach 2013/08/20 17:42:34 unrelated?
64 annotation='SmallTest'),
60 I('AndroidWebView', 65 I('AndroidWebView',
61 'AndroidWebView.apk', 66 'AndroidWebView.apk',
62 'org.chromium.android_webview.shell', 67 'org.chromium.android_webview.shell',
63 'AndroidWebViewTest', 68 'AndroidWebViewTest',
64 'webview:android_webview/test/data/device_files'), 69 'webview:android_webview/test/data/device_files',
70 annotation='SmallTest'),
65 ]) 71 ])
66 72
67 VALID_TESTS = set(['chromedriver', 'ui', 'unit', 'webkit', 'webkit_layout', 73 VALID_TESTS = set(['chromedriver', 'ui', 'unit', 'webkit', 'webkit_layout',
68 'webrtc']) 74 'webrtc'])
69 75
70 RunCmd = bb_utils.RunCmd 76 RunCmd = bb_utils.RunCmd
71 77
72 78
73 # multiprocessing map_async requires a top-level function for pickle library. 79 # multiprocessing map_async requires a top-level function for pickle library.
74 def RebootDeviceSafe(device): 80 def RebootDeviceSafe(device):
(...skipping 83 matching lines...) Expand 10 before | Expand all | Expand 10 after
158 InstallApk(options, test) 164 InstallApk(options, test)
159 args = ['--test-apk', test.test_apk, '--test_data', test.test_data, 165 args = ['--test-apk', test.test_apk, '--test_data', test.test_data,
160 '--verbose'] 166 '--verbose']
161 if options.target == 'Release': 167 if options.target == 'Release':
162 args.append('--release') 168 args.append('--release')
163 if options.asan: 169 if options.asan:
164 args.append('--tool=asan') 170 args.append('--tool=asan')
165 if options.flakiness_server: 171 if options.flakiness_server:
166 args.append('--flakiness-dashboard-server=%s' % 172 args.append('--flakiness-dashboard-server=%s' %
167 options.flakiness_server) 173 options.flakiness_server)
174 if options.coverage_bucket and test.coverage:
175 args.append('--coverage-dir=%s' % constants.GetCoverageDir(options.target))
168 if test.host_driven_root: 176 if test.host_driven_root:
169 args.append('--host-driven-root=%s' % test.host_driven_root) 177 args.append('--host-driven-root=%s' % test.host_driven_root)
170 if test.annotation: 178 if test.annotation:
171 args.extend(['-A', test.annotation]) 179 args.extend(['-A', test.annotation])
172 if test.exclude_annotation: 180 if test.exclude_annotation:
173 args.extend(['-E', test.exclude_annotation]) 181 args.extend(['-E', test.exclude_annotation])
174 if test.extra_flags: 182 if test.extra_flags:
175 args.extend(test.extra_flags) 183 args.extend(test.extra_flags)
176 if python_only: 184 if python_only:
177 args.append('-p') 185 args.append('-p')
(...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after
264 ('provision_devices', ProvisionDevices), 272 ('provision_devices', ProvisionDevices),
265 ('device_status_check', DeviceStatusCheck) 273 ('device_status_check', DeviceStatusCheck)
266 ] 274 ]
267 275
268 276
269 def RunUnitTests(options): 277 def RunUnitTests(options):
270 RunTestSuites(options, gtest_config.STABLE_TEST_SUITES) 278 RunTestSuites(options, gtest_config.STABLE_TEST_SUITES)
271 279
272 280
273 def RunInstrumentationTests(options): 281 def RunInstrumentationTests(options):
282 coverage_dir = constants.GetCoverageDir(options.target)
283 if options.coverage_bucket:
284 shutil.rmtree(coverage_dir, ignore_errors=True)
285 os.mkdir(coverage_dir)
286
274 for test in INSTRUMENTATION_TESTS.itervalues(): 287 for test in INSTRUMENTATION_TESTS.itervalues():
275 RunInstrumentationSuite(options, test) 288 RunInstrumentationSuite(options, test)
276 289
290 if options.coverage_bucket:
291 print 'generate_emma_html perms: %s' % RunCmd(
292 ['ls', '-l', 'build/android/generate_emma_html.py'])
bulach 2013/08/20 17:42:34 not needed?
293 RunCmd(['build/android/generate_emma_html.py',
294 '--coverage-dir', coverage_dir,
295 '--metadata-dir', os.path.join(constants.DIR_SOURCE_ROOT,
296 'out', options.target),
297 '--output', os.path.join(coverage_dir, 'coverage.html')])
298
299 revision_file = os.path.join(CHROME_SRC, 'out', options.target,
300 REVISION_FILENAME)
301 # If the revision file doesn't exist, we're just testing the bot
302 if os.path.exists(revision_file):
303 with open(revision_file) as f:
304 revision = f.read()
305 else:
306 revision = 'testing'
307 bot_id = os.environ.get('BUILDBOT_ID')
308 if not bot_id:
309 print 'BUILDBOT_ID requried in the environment to upload coverage data.'
310 return
311
312 boto_config = os.path.join(bb_utils.BB_BUILD_DIR, 'site_config', '.boto')
313 os.environ['AWS_CREDENTIAL_FILE'] = boto_config
314 os.environ['BOTO_CONFIG'] = boto_config
315 RunCmd([os.path.join(bb_utils.BB_BUILD_DIR, 'third_party',
316 'gsutil', 'gsutil'), 'cp', '-R',
317 coverage_dir, 'gs://%s/%s/%s' %
318 (options.coverage_bucket, bot_id, revision)])
319 bb_annotations.PrintLink(
320 'Coverage report',
321 'https://storage.googleapis.com/%s/%s/coverage.html'
322 % (options.coverage_bucket, revision))
323
277 324
278 def RunWebkitTests(options): 325 def RunWebkitTests(options):
279 RunTestSuites(options, ['webkit_unit_tests']) 326 RunTestSuites(options, ['webkit_unit_tests'])
280 RunWebkitLint(options.target) 327 RunWebkitLint(options.target)
281 328
282 329
283 def RunWebRTCTests(options): 330 def RunWebRTCTests(options):
284 RunTestSuites(options, gtest_config.WEBRTC_TEST_SUITES) 331 RunTestSuites(options, gtest_config.WEBRTC_TEST_SUITES)
285 332
286 333
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after
348 help='Run experiemental tests') 395 help='Run experiemental tests')
349 parser.add_option('-f', '--test-filter', metavar='<filter>', default=[], 396 parser.add_option('-f', '--test-filter', metavar='<filter>', default=[],
350 action='append', 397 action='append',
351 help=('Run a test suite. Test suites: "%s"' % 398 help=('Run a test suite. Test suites: "%s"' %
352 '", "'.join(VALID_TESTS))) 399 '", "'.join(VALID_TESTS)))
353 parser.add_option('--asan', action='store_true', help='Run tests with asan.') 400 parser.add_option('--asan', action='store_true', help='Run tests with asan.')
354 parser.add_option('--install', metavar='<apk name>', 401 parser.add_option('--install', metavar='<apk name>',
355 help='Install an apk by name') 402 help='Install an apk by name')
356 parser.add_option('--reboot', action='store_true', 403 parser.add_option('--reboot', action='store_true',
357 help='Reboot devices before running tests') 404 help='Reboot devices before running tests')
405 parser.add_option('--coverage-bucket',
406 help=('Bucket name to store coverage results. Coverage is '
407 'only run if this is set.'))
358 parser.add_option( 408 parser.add_option(
359 '--flakiness-server', 409 '--flakiness-server',
360 help='The flakiness dashboard server to which the results should be ' 410 help='The flakiness dashboard server to which the results should be '
361 'uploaded.') 411 'uploaded.')
362 parser.add_option( 412 parser.add_option(
363 '--auto-reconnect', action='store_true', 413 '--auto-reconnect', action='store_true',
364 help='Push script to device which restarts adbd on disconnections.') 414 help='Push script to device which restarts adbd on disconnections.')
365 parser.add_option( 415 parser.add_option(
366 '--logcat-dump-output', 416 '--logcat-dump-output',
367 help='The logcat dump output will be "tee"-ed into this file') 417 help='The logcat dump output will be "tee"-ed into this file')
(...skipping 12 matching lines...) Expand all
380 if unknown_tests: 430 if unknown_tests:
381 return sys.exit('Unknown tests %s' % list(unknown_tests)) 431 return sys.exit('Unknown tests %s' % list(unknown_tests))
382 432
383 setattr(options, 'target', options.factory_properties.get('target', 'Debug')) 433 setattr(options, 'target', options.factory_properties.get('target', 'Debug'))
384 434
385 MainTestWrapper(options) 435 MainTestWrapper(options)
386 436
387 437
388 if __name__ == '__main__': 438 if __name__ == '__main__':
389 sys.exit(main(sys.argv)) 439 sys.exit(main(sys.argv))
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698