Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(95)

Side by Side Diff: build/android/buildbot/bb_device_steps.py

Issue 23345003: [Android] Buildbot changes for EMMA code coverage (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Fixes getting revision, boto config, unnecessary setup Created 7 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | build/android/buildbot/bb_run_bot.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 import collections 6 import collections
7 import glob 7 import glob
8 import multiprocessing 8 import multiprocessing
9 import os 9 import os
10 import shutil 10 import shutil
11 import sys 11 import sys
12 import time
12 13
13 import bb_utils 14 import bb_utils
14 import bb_annotations 15 import bb_annotations
15 16
16 sys.path.append(os.path.join(os.path.dirname(__file__), '..')) 17 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
17 import provision_devices 18 import provision_devices
18 from pylib import android_commands 19 from pylib import android_commands
19 from pylib import constants 20 from pylib import constants
20 from pylib.gtest import gtest_config 21 from pylib.gtest import gtest_config
21 22
22 sys.path.append(os.path.join( 23 sys.path.append(os.path.join(
23 constants.DIR_SOURCE_ROOT, 'third_party', 'android_testrunner')) 24 constants.DIR_SOURCE_ROOT, 'third_party', 'android_testrunner'))
24 import errors 25 import errors
25 26
26 27
27 CHROME_SRC = constants.DIR_SOURCE_ROOT 28 CHROME_SRC = constants.DIR_SOURCE_ROOT
28 LOGCAT_DIR = os.path.join(CHROME_SRC, 'out', 'logcat') 29 LOGCAT_DIR = os.path.join(CHROME_SRC, 'out', 'logcat')
30 REVISION_FILENAME = 'FULL_BUILD_REVISION'
29 31
30 # Describes an instrumation test suite: 32 # Describes an instrumation test suite:
31 # test: Name of test we're running. 33 # test: Name of test we're running.
32 # apk: apk to be installed. 34 # apk: apk to be installed.
33 # apk_package: package for the apk to be installed. 35 # apk_package: package for the apk to be installed.
34 # test_apk: apk to run tests on. 36 # test_apk: apk to run tests on.
35 # test_data: data folder in format destination:source. 37 # test_data: data folder in format destination:source.
36 # host_driven_root: The host-driven test root directory. 38 # host_driven_root: The host-driven test root directory.
37 # annotation: Annotation of the tests to include. 39 # annotation: Annotation of the tests to include.
38 # exclude_annotation: The annotation of the tests to exclude. 40 # exclude_annotation: The annotation of the tests to exclude.
39 I_TEST = collections.namedtuple('InstrumentationTest', [ 41 I_TEST = collections.namedtuple('InstrumentationTest', [
40 'name', 'apk', 'apk_package', 'test_apk', 'test_data', 'host_driven_root', 42 'name', 'apk', 'apk_package', 'test_apk', 'test_data', 'coverage',
41 'annotation', 'exclude_annotation', 'extra_flags']) 43 'host_driven_root', 'annotation', 'exclude_annotation', 'extra_flags'])
42 44
43 def I(name, apk, apk_package, test_apk, test_data, host_driven_root=None, 45 def I(name, apk, apk_package, test_apk, test_data, coverage=True,
frankf 2013/08/21 00:42:26 Didn't we decide all test apks should have coverag
gkanwar1 2013/08/21 00:55:15 Yep, this is a holdover from setting just ContentS
44 annotation=None, exclude_annotation=None, extra_flags=None): 46 host_driven_root=None, annotation=None, exclude_annotation=None,
45 return I_TEST(name, apk, apk_package, test_apk, test_data, host_driven_root, 47 extra_flags=None):
46 annotation, exclude_annotation, extra_flags) 48 return I_TEST(name, apk, apk_package, test_apk, test_data, coverage,
49 host_driven_root, annotation, exclude_annotation, extra_flags)
47 50
48 INSTRUMENTATION_TESTS = dict((suite.name, suite) for suite in [ 51 INSTRUMENTATION_TESTS = dict((suite.name, suite) for suite in [
49 I('ContentShell', 52 I('ContentShell',
50 'ContentShell.apk', 53 'ContentShell.apk',
51 'org.chromium.content_shell_apk', 54 'org.chromium.content_shell_apk',
52 'ContentShellTest', 55 'ContentShellTest',
53 'content:content/test/data/android/device_files'), 56 'content:content/test/data/android/device_files'),
54 I('ChromiumTestShell', 57 I('ChromiumTestShell',
55 'ChromiumTestShell.apk', 58 'ChromiumTestShell.apk',
56 'org.chromium.chrome.testshell', 59 'org.chromium.chrome.testshell',
(...skipping 101 matching lines...) Expand 10 before | Expand all | Expand 10 after
158 InstallApk(options, test) 161 InstallApk(options, test)
159 args = ['--test-apk', test.test_apk, '--test_data', test.test_data, 162 args = ['--test-apk', test.test_apk, '--test_data', test.test_data,
160 '--verbose'] 163 '--verbose']
161 if options.target == 'Release': 164 if options.target == 'Release':
162 args.append('--release') 165 args.append('--release')
163 if options.asan: 166 if options.asan:
164 args.append('--tool=asan') 167 args.append('--tool=asan')
165 if options.flakiness_server: 168 if options.flakiness_server:
166 args.append('--flakiness-dashboard-server=%s' % 169 args.append('--flakiness-dashboard-server=%s' %
167 options.flakiness_server) 170 options.flakiness_server)
171 if options.coverage_bucket and test.coverage:
172 args.append('--coverage-dir=%s' % options.coverage_dir)
168 if test.host_driven_root: 173 if test.host_driven_root:
169 args.append('--host-driven-root=%s' % test.host_driven_root) 174 args.append('--host-driven-root=%s' % test.host_driven_root)
170 if test.annotation: 175 if test.annotation:
171 args.extend(['-A', test.annotation]) 176 args.extend(['-A', test.annotation])
172 if test.exclude_annotation: 177 if test.exclude_annotation:
173 args.extend(['-E', test.exclude_annotation]) 178 args.extend(['-E', test.exclude_annotation])
174 if test.extra_flags: 179 if test.extra_flags:
175 args.extend(test.extra_flags) 180 args.extend(test.extra_flags)
176 if python_only: 181 if python_only:
177 args.append('-p') 182 args.append('-p')
(...skipping 109 matching lines...) Expand 10 before | Expand all | Expand 10 after
287 return [ 292 return [
288 ('chromedriver', RunChromeDriverTests), 293 ('chromedriver', RunChromeDriverTests),
289 ('unit', RunUnitTests), 294 ('unit', RunUnitTests),
290 ('ui', RunInstrumentationTests), 295 ('ui', RunInstrumentationTests),
291 ('webkit', RunWebkitTests), 296 ('webkit', RunWebkitTests),
292 ('webkit_layout', RunWebkitLayoutTests), 297 ('webkit_layout', RunWebkitLayoutTests),
293 ('webrtc', RunWebRTCTests), 298 ('webrtc', RunWebRTCTests),
294 ] 299 ]
295 300
296 301
302 def UploadCoverageData(options, path, coverage_type):
frankf 2013/08/21 00:42:26 I'd just inline this
gkanwar1 2013/08/21 00:55:15 This is separate because the native coverage is go
303 revision = options.build_properties.get('got_revision', 'testing')
304 bot_id = options.build_properties.get('buildername', 'testing')
305
306 timehash = hash(time.time())
frankf 2013/08/21 00:42:26 Perhaps use something like hashlib.sha1(str(random
gkanwar1 2013/08/21 00:55:15 Done.
307
308 RunCmd([bb_utils.GSUTIL_PATH, 'cp', '-R', path, 'gs://%s/%s/%s/%s/%s' %
309 (options.coverage_bucket, coverage_type, bot_id, revision, timehash)])
310 bb_annotations.PrintLink(
311 'Coverage report',
312 'https://storage.googleapis.com/%s/%s/%s/%s/%s/index.html'
313 % (options.coverage_bucket, coverage_type, bot_id, revision, timehash))
frankf 2013/08/21 00:42:26 Not sure if having coverage_type in path is useful
gkanwar1 2013/08/21 00:55:15 We're going to be having a native and java coverag
314
315
316 def GenerateJavaCoverageReport(options):
317 bb_annotations.PrintNamedStep('java_coverage_report')
318
319 coverage_html = os.path.join(options.coverage_dir, 'coverage_html')
320 RunCmd(['build/android/generate_emma_html.py',
321 '--coverage-dir', options.coverage_dir,
322 '--metadata-dir', os.path.join(CHROME_SRC, 'out', options.target),
323 '--output', os.path.join(coverage_html, 'coverage.html')])
324 UploadCoverageData(options, coverage_html, 'java')
325
326
297 def LogcatDump(options): 327 def LogcatDump(options):
298 # Print logcat, kill logcat monitor 328 # Print logcat, kill logcat monitor
299 bb_annotations.PrintNamedStep('logcat_dump') 329 bb_annotations.PrintNamedStep('logcat_dump')
300 logcat_file = os.path.join(CHROME_SRC, 'out', options.target, 'full_log') 330 logcat_file = os.path.join(CHROME_SRC, 'out', options.target, 'full_log')
301 with open(logcat_file, 'w') as f: 331 with open(logcat_file, 'w') as f:
302 RunCmd([ 332 RunCmd([
303 os.path.join(CHROME_SRC, 'build', 'android', 'adb_logcat_printer.py'), 333 os.path.join(CHROME_SRC, 'build', 'android', 'adb_logcat_printer.py'),
304 LOGCAT_DIR], stdout=f) 334 LOGCAT_DIR], stdout=f)
305 RunCmd(['cat', logcat_file]) 335 RunCmd(['cat', logcat_file])
306 336
(...skipping 15 matching lines...) Expand all
322 for _, cmd in GetDeviceSetupStepCmds(): 352 for _, cmd in GetDeviceSetupStepCmds():
323 cmd(options) 353 cmd(options)
324 354
325 if options.install: 355 if options.install:
326 test_obj = INSTRUMENTATION_TESTS[options.install] 356 test_obj = INSTRUMENTATION_TESTS[options.install]
327 InstallApk(options, test_obj, print_step=True) 357 InstallApk(options, test_obj, print_step=True)
328 358
329 if options.test_filter: 359 if options.test_filter:
330 bb_utils.RunSteps(options.test_filter, GetTestStepCmds(), options) 360 bb_utils.RunSteps(options.test_filter, GetTestStepCmds(), options)
331 361
362 if options.coverage_bucket:
363 GenerateJavaCoverageReport(options)
364
332 if options.experimental: 365 if options.experimental:
333 RunTestSuites(options, gtest_config.EXPERIMENTAL_TEST_SUITES) 366 RunTestSuites(options, gtest_config.EXPERIMENTAL_TEST_SUITES)
334 367
335 finally: 368 finally:
336 # Run all post test steps 369 # Run all post test steps
337 LogcatDump(options) 370 LogcatDump(options)
338 GenerateTestReport(options) 371 GenerateTestReport(options)
339 # KillHostHeartbeat() has logic to check if heartbeat process is running, 372 # KillHostHeartbeat() has logic to check if heartbeat process is running,
340 # and kills only if it finds the process is running on the host. 373 # and kills only if it finds the process is running on the host.
341 provision_devices.KillHostHeartbeat() 374 provision_devices.KillHostHeartbeat()
342 375
343 376
344 def GetDeviceStepsOptParser(): 377 def GetDeviceStepsOptParser():
345 parser = bb_utils.GetParser() 378 parser = bb_utils.GetParser()
346 parser.add_option('--experimental', action='store_true', 379 parser.add_option('--experimental', action='store_true',
347 help='Run experiemental tests') 380 help='Run experiemental tests')
348 parser.add_option('-f', '--test-filter', metavar='<filter>', default=[], 381 parser.add_option('-f', '--test-filter', metavar='<filter>', default=[],
349 action='append', 382 action='append',
350 help=('Run a test suite. Test suites: "%s"' % 383 help=('Run a test suite. Test suites: "%s"' %
351 '", "'.join(VALID_TESTS))) 384 '", "'.join(VALID_TESTS)))
352 parser.add_option('--asan', action='store_true', help='Run tests with asan.') 385 parser.add_option('--asan', action='store_true', help='Run tests with asan.')
353 parser.add_option('--install', metavar='<apk name>', 386 parser.add_option('--install', metavar='<apk name>',
354 help='Install an apk by name') 387 help='Install an apk by name')
355 parser.add_option('--reboot', action='store_true', 388 parser.add_option('--reboot', action='store_true',
356 help='Reboot devices before running tests') 389 help='Reboot devices before running tests')
390 parser.add_option('--coverage-bucket',
391 help=('Bucket name to store coverage results. Coverage is '
392 'only run if this is set.'))
357 parser.add_option( 393 parser.add_option(
358 '--flakiness-server', 394 '--flakiness-server',
359 help='The flakiness dashboard server to which the results should be ' 395 help='The flakiness dashboard server to which the results should be '
360 'uploaded.') 396 'uploaded.')
361 parser.add_option( 397 parser.add_option(
362 '--auto-reconnect', action='store_true', 398 '--auto-reconnect', action='store_true',
363 help='Push script to device which restarts adbd on disconnections.') 399 help='Push script to device which restarts adbd on disconnections.')
364 parser.add_option( 400 parser.add_option(
365 '--logcat-dump-output', 401 '--logcat-dump-output',
366 help='The logcat dump output will be "tee"-ed into this file') 402 help='The logcat dump output will be "tee"-ed into this file')
367 403
368 return parser 404 return parser
369 405
370 406
371 def main(argv): 407 def main(argv):
372 parser = GetDeviceStepsOptParser() 408 parser = GetDeviceStepsOptParser()
373 options, args = parser.parse_args(argv[1:]) 409 options, args = parser.parse_args(argv[1:])
374 410
375 if args: 411 if args:
376 return sys.exit('Unused args %s' % args) 412 return sys.exit('Unused args %s' % args)
377 413
378 unknown_tests = set(options.test_filter) - VALID_TESTS 414 unknown_tests = set(options.test_filter) - VALID_TESTS
379 if unknown_tests: 415 if unknown_tests:
380 return sys.exit('Unknown tests %s' % list(unknown_tests)) 416 return sys.exit('Unknown tests %s' % list(unknown_tests))
381 417
382 setattr(options, 'target', options.factory_properties.get('target', 'Debug')) 418 setattr(options, 'target', options.factory_properties.get('target', 'Debug'))
419 if options.coverage_bucket:
420 setattr(options, 'coverage_dir',
421 os.path.join(CHROME_SRC, 'out', options.target, 'coverage'))
383 422
384 MainTestWrapper(options) 423 MainTestWrapper(options)
385 424
386 425
387 if __name__ == '__main__': 426 if __name__ == '__main__':
388 sys.exit(main(sys.argv)) 427 sys.exit(main(sys.argv))
OLDNEW
« no previous file with comments | « no previous file | build/android/buildbot/bb_run_bot.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698