Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(460)

Side by Side Diff: scripts/slave/recipe_modules/chromium_tests/api.py

Issue 1474473004: Pass mb_mastername and mb_buildername as parameters to override the default. (Closed) Base URL: https://chromium.googlesource.com/chromium/tools/build.git@master
Patch Set: Fix nit and docstrings after rebase. Created 5 years ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | scripts/slave/recipe_modules/filter/api.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright 2014 The Chromium Authors. All rights reserved. 1 # Copyright 2014 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 import ast 5 import ast
6 import contextlib 6 import contextlib
7 import copy 7 import copy
8 import itertools 8 import itertools
9 import json 9 import json
10 10
(...skipping 339 matching lines...) Expand 10 before | Expand all | Expand 10 after
350 command(lambda name: '%s (with patch)' % name) 350 command(lambda name: '%s (with patch)' % name)
351 except self.m.step.StepFailure: 351 except self.m.step.StepFailure:
352 self.deapply_patch(update_step) 352 self.deapply_patch(update_step)
353 command(lambda name: '%s (without patch)' % name) 353 command(lambda name: '%s (without patch)' % name)
354 raise 354 raise
355 else: 355 else:
356 command(lambda name: name) 356 command(lambda name: name)
357 357
358 358
359 def compile(self, mastername, buildername, update_step, master_dict, 359 def compile(self, mastername, buildername, update_step, master_dict,
360 test_spec): 360 test_spec, mb_mastername=None, mb_buildername=None):
361 """Runs compile and related steps for given builder.""" 361 """Runs compile and related steps for given builder."""
362 compile_targets, tests_including_triggered = \ 362 compile_targets, tests_including_triggered = \
363 self.get_compile_targets_and_tests( 363 self.get_compile_targets_and_tests(
364 mastername, 364 mastername,
365 buildername, 365 buildername,
366 master_dict, test_spec) 366 master_dict, test_spec)
367 self.compile_specific_targets( 367 self.compile_specific_targets(
368 mastername, buildername, update_step, master_dict, 368 mastername, buildername, update_step, master_dict,
369 compile_targets, tests_including_triggered) 369 compile_targets, tests_including_triggered,
370 mb_mastername=mb_mastername, mb_buildername=mb_buildername)
370 371
371 def compile_specific_targets( 372 def compile_specific_targets(
372 self, mastername, buildername, update_step, master_dict, 373 self, mastername, buildername, update_step, master_dict,
373 compile_targets, tests_including_triggered, override_bot_type=None): 374 compile_targets, tests_including_triggered,
375 mb_mastername=None, mb_buildername=None, override_bot_type=None):
374 """Runs compile and related steps for given builder. 376 """Runs compile and related steps for given builder.
375 377
376 Allows finer-grained control about exact compile targets used.""" 378 Allows finer-grained control about exact compile targets used.
379
380 We don't use the given `mastername` and `buildername` to run MB, because
381 they may be the values of the continuous builder the trybot may be
382 configured to match; instead we need to use the actual mastername and
383 buildername we're running on (Default to the "mastername" and
384 "buildername" in the build properties -- self.m.properties, but could be
385 overridden by `mb_mastername` and `mb_buildername`), because it may be
386 configured with different MB settings.
387
388 However, recipes used by Findit for culprit finding may still set
389 (mb_mastername, mb_buildername) = (mastername, buildername) to exactly match
390 a given continuous builder."""
377 391
378 bot_config = master_dict.get('builders', {}).get(buildername) 392 bot_config = master_dict.get('builders', {}).get(buildername)
379 master_config = master_dict.get('settings', {}) 393 master_config = master_dict.get('settings', {})
380 bot_type = override_bot_type or bot_config.get('bot_type', 'builder_tester') 394 bot_type = override_bot_type or bot_config.get('bot_type', 'builder_tester')
381 395
382 self.m.chromium.cleanup_temp() 396 self.m.chromium.cleanup_temp()
383 if self.m.chromium.c.TARGET_PLATFORM == 'android': 397 if self.m.chromium.c.TARGET_PLATFORM == 'android':
384 self.m.chromium_android.clean_local_files() 398 self.m.chromium_android.clean_local_files()
385 self.m.chromium_android.run_tree_truth() 399 self.m.chromium_android.run_tree_truth()
386 400
387 if bot_type in ['builder', 'builder_tester']: 401 if bot_type in ['builder', 'builder_tester']:
388 isolated_targets = [ 402 isolated_targets = [
389 t.isolate_target for t in tests_including_triggered if t.uses_swarming 403 t.isolate_target for t in tests_including_triggered if t.uses_swarming
390 ] 404 ]
391 405
392 if isolated_targets: 406 if isolated_targets:
393 self.m.isolate.clean_isolated_files(self.m.chromium.output_dir) 407 self.m.isolate.clean_isolated_files(self.m.chromium.output_dir)
394 408
395 if self.m.chromium.c.project_generator.tool == 'mb': 409 if self.m.chromium.c.project_generator.tool == 'mb':
396 if bot_config.get('chromium_config') == 'chromium_win_clang': 410 if bot_config.get('chromium_config') == 'chromium_win_clang':
397 self.m.chromium.update_clang() 411 self.m.chromium.update_clang()
398 412
399 try: 413 try:
400 self.transient_check(update_step, lambda transform_name: 414 self.transient_check(update_step, lambda transform_name:
401 self.run_mb_and_compile(compile_targets, isolated_targets, 415 self.run_mb_and_compile(compile_targets, isolated_targets,
402 name_suffix=transform_name(''))) 416 name_suffix=transform_name(''),
417 mb_mastername=mb_mastername,
418 mb_buildername=mb_buildername))
403 except self.m.step.StepFailure: 419 except self.m.step.StepFailure:
404 self.m.tryserver.set_compile_failure_tryjob_result() 420 self.m.tryserver.set_compile_failure_tryjob_result()
405 raise 421 raise
406 422
407 if isolated_targets: 423 if isolated_targets:
408 self.m.isolate.remove_build_metadata() 424 self.m.isolate.remove_build_metadata()
409 # 'compile' just prepares all information needed for the isolation, 425 # 'compile' just prepares all information needed for the isolation,
410 # and the isolation is a separate step. 426 # and the isolation is a separate step.
411 self.m.isolate.isolate_tests( 427 self.m.isolate.isolate_tests(
412 self.m.chromium.output_dir, 428 self.m.chromium.output_dir,
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after
460 }) 476 })
461 477
462 if bot_config.get('archive_build') and not self.m.tryserver.is_tryserver: 478 if bot_config.get('archive_build') and not self.m.tryserver.is_tryserver:
463 self.m.chromium.archive_build( 479 self.m.chromium.archive_build(
464 'archive_build', 480 'archive_build',
465 bot_config['gs_bucket'], 481 bot_config['gs_bucket'],
466 bot_config.get('gs_acl'), 482 bot_config.get('gs_acl'),
467 mode='dev' 483 mode='dev'
468 ) 484 )
469 485
470 def run_mb_and_compile(self, compile_targets, isolated_targets, name_suffix): 486 def run_mb_and_compile(self, compile_targets, isolated_targets, name_suffix,
487 mb_mastername=None, mb_buildername=None):
471 if self.m.chromium.c.project_generator.tool == 'mb': 488 if self.m.chromium.c.project_generator.tool == 'mb':
472 # We don't use the mastername and buildername passed in, because 489 mb_mastername = mb_mastername or self.m.properties['mastername']
473 # those may be the values of the continuous builder the trybot may 490 mb_buildername = mb_buildername or self.m.properties['buildername']
474 # be configured to match; we need to use the actual mastername 491 self.m.chromium.run_mb(mb_mastername, mb_buildername,
475 # and buildername we're running on, because it may be configured
476 # with different MB settings.
477 real_mastername = self.m.properties['mastername']
478 real_buildername = self.m.properties['buildername']
479 self.m.chromium.run_mb(real_mastername, real_buildername,
480 isolated_targets=isolated_targets, 492 isolated_targets=isolated_targets,
481 name='generate_build_files%s' % name_suffix) 493 name='generate_build_files%s' % name_suffix)
482 494
483 self.m.chromium.compile(compile_targets, name='compile%s' % name_suffix) 495 self.m.chromium.compile(compile_targets, name='compile%s' % name_suffix)
484 496
485 def tests_for_builder(self, mastername, buildername, update_step, master_dict, 497 def tests_for_builder(self, mastername, buildername, update_step, master_dict,
486 override_bot_type=None): 498 override_bot_type=None):
487 got_revision = update_step.presentation.properties['got_revision'] 499 got_revision = update_step.presentation.properties['got_revision']
488 500
489 bot_config = master_dict.get('builders', {}).get(buildername) 501 bot_config = master_dict.get('builders', {}).get(buildername)
(...skipping 101 matching lines...) Expand 10 before | Expand all | Expand 10 after
591 # mapping for their own purposes. 603 # mapping for their own purposes.
592 first_solution_name = self.m.gclient.c.solutions[0].name 604 first_solution_name = self.m.gclient.c.solutions[0].name
593 rev_property = self.m.gclient.c.got_revision_mapping[first_solution_name] 605 rev_property = self.m.gclient.c.got_revision_mapping[first_solution_name]
594 self.m.gclient.c.revisions[first_solution_name] = str( 606 self.m.gclient.c.revisions[first_solution_name] = str(
595 bot_update_json['properties'][rev_property]) 607 bot_update_json['properties'][rev_property])
596 self.m.bot_update.ensure_checkout( 608 self.m.bot_update.ensure_checkout(
597 force=True, patch=False, update_presentation=False) 609 force=True, patch=False, update_presentation=False)
598 self.m.chromium.runhooks(name='runhooks (without patch)') 610 self.m.chromium.runhooks(name='runhooks (without patch)')
599 611
600 def run_tests_on_tryserver(self, mastername, api, tests, bot_update_step, 612 def run_tests_on_tryserver(self, mastername, api, tests, bot_update_step,
601 affected_files): 613 affected_files, mb_mastername=None,
614 mb_buildername=None):
602 def deapply_patch_fn(failing_tests): 615 def deapply_patch_fn(failing_tests):
603 self.deapply_patch(bot_update_step) 616 self.deapply_patch(bot_update_step)
604 compile_targets = list(itertools.chain( 617 compile_targets = list(itertools.chain(
605 *[t.compile_targets(api) for t in failing_tests])) 618 *[t.compile_targets(api) for t in failing_tests]))
606 if compile_targets: 619 if compile_targets:
607 # Remove duplicate targets. 620 # Remove duplicate targets.
608 compile_targets = sorted(set(compile_targets)) 621 compile_targets = sorted(set(compile_targets))
609 failing_swarming_tests = [ 622 failing_swarming_tests = [
610 t.isolate_target for t in failing_tests if t.uses_swarming] 623 t.isolate_target for t in failing_tests if t.uses_swarming]
611 if failing_swarming_tests: 624 if failing_swarming_tests:
612 self.m.isolate.clean_isolated_files(self.m.chromium.output_dir) 625 self.m.isolate.clean_isolated_files(self.m.chromium.output_dir)
613 self.run_mb_and_compile(compile_targets, failing_swarming_tests, 626 self.run_mb_and_compile(compile_targets, failing_swarming_tests,
614 ' (without patch)') 627 ' (without patch)',
628 mb_mastername=mb_mastername,
629 mb_buildername=mb_buildername)
615 if failing_swarming_tests: 630 if failing_swarming_tests:
616 self.m.isolate.isolate_tests(self.m.chromium.output_dir, 631 self.m.isolate.isolate_tests(self.m.chromium.output_dir,
617 verbose=True) 632 verbose=True)
618 633
619 deapply_patch = True 634 deapply_patch = True
620 for path in RECIPE_CONFIG_PATHS: 635 for path in RECIPE_CONFIG_PATHS:
621 if any([f.startswith(path) for f in affected_files]): 636 if any([f.startswith(path) for f in affected_files]):
622 deapply_patch = False 637 deapply_patch = False
623 break 638 break
624 639
625 with self.wrap_chromium_tests(mastername, tests): 640 with self.wrap_chromium_tests(mastername, tests):
626 if deapply_patch: 641 if deapply_patch:
627 self.m.test_utils.determine_new_failures(api, tests, deapply_patch_fn) 642 self.m.test_utils.determine_new_failures(api, tests, deapply_patch_fn)
628 else: 643 else:
629 failing_tests = self.m.test_utils.run_tests_with_patch(api, tests) 644 failing_tests = self.m.test_utils.run_tests_with_patch(api, tests)
630 if failing_tests: 645 if failing_tests:
631 self.m.python.failing_step('test results', 'TESTS FAILED') 646 self.m.python.failing_step('test results', 'TESTS FAILED')
632 647
633 def analyze(self, affected_files, test_targets, additional_compile_targets, 648 def analyze(self, affected_files, test_targets, additional_compile_targets,
634 config_file_name, additional_names=None): 649 config_file_name, mb_mastername=None, mb_buildername=None,
650 additional_names=None):
635 """Runs "analyze" step to determine targets affected by the patch. 651 """Runs "analyze" step to determine targets affected by the patch.
636 652
637 Returns a tuple of: 653 Returns a tuple of:
638 - list of targets that are needed to run tests (see filter recipe module) 654 - list of targets that are needed to run tests (see filter recipe module)
639 - list of targets that need to be compiled (see filter recipe module)""" 655 - list of targets that need to be compiled (see filter recipe module)"""
640 656
641 if additional_names is None: 657 if additional_names is None:
642 additional_names = ['chromium'] 658 additional_names = ['chromium']
643 659
644 use_mb = (self.m.chromium.c.project_generator.tool == 'mb') 660 use_mb = (self.m.chromium.c.project_generator.tool == 'mb')
645 build_output_dir = '//out/%s' % self.m.chromium.c.build_config_fs 661 build_output_dir = '//out/%s' % self.m.chromium.c.build_config_fs
646 self.m.filter.does_patch_require_compile( 662 self.m.filter.does_patch_require_compile(
647 affected_files, 663 affected_files,
648 test_targets=test_targets, 664 test_targets=test_targets,
649 additional_compile_targets=additional_compile_targets, 665 additional_compile_targets=additional_compile_targets,
650 additional_names=additional_names, 666 additional_names=additional_names,
651 config_file_name=config_file_name, 667 config_file_name=config_file_name,
652 use_mb=use_mb, 668 use_mb=use_mb,
669 mb_mastername=mb_mastername,
670 mb_buildername=mb_buildername,
653 build_output_dir=build_output_dir, 671 build_output_dir=build_output_dir,
654 cros_board=self.m.chromium.c.TARGET_CROS_BOARD) 672 cros_board=self.m.chromium.c.TARGET_CROS_BOARD)
655 673
656 compile_targets = self.m.filter.compile_targets[:] 674 compile_targets = self.m.filter.compile_targets[:]
657 675
658 # Add crash_service to compile_targets. This is done after filtering compile 676 # Add crash_service to compile_targets. This is done after filtering compile
659 # targets out because crash_service should always be there on windows. 677 # targets out because crash_service should always be there on windows.
660 # TODO(akuegel): Need to solve this in a better way. crbug.com/478053 678 # TODO(akuegel): Need to solve this in a better way. crbug.com/478053
661 if (self.m.platform.is_win and compile_targets and 679 if (self.m.platform.is_win and compile_targets and
662 'crash_service' not in compile_targets): 680 'crash_service' not in compile_targets):
(...skipping 151 matching lines...) Expand 10 before | Expand all | Expand 10 after
814 result_text = 'MB is enabled for this builder at this revision.' 832 result_text = 'MB is enabled for this builder at this revision.'
815 log_name = 'Builder MB-ready' 833 log_name = 'Builder MB-ready'
816 self.m.step.active_result.presentation.logs[log_name] = [result_text] 834 self.m.step.active_result.presentation.logs[log_name] = [result_text]
817 return False 835 return False
818 except (self.m.step.StepFailure, KeyError): 836 except (self.m.step.StepFailure, KeyError):
819 result_text = 'MB is not enabled for this builder at this revision.' 837 result_text = 'MB is not enabled for this builder at this revision.'
820 log_name = 'Builder NOT MB-ready' 838 log_name = 'Builder NOT MB-ready'
821 self.m.step.active_result.presentation.logs[log_name] = [result_text] 839 self.m.step.active_result.presentation.logs[log_name] = [result_text]
822 self.m.step.active_result.presentation.status = self.m.step.WARNING 840 self.m.step.active_result.presentation.status = self.m.step.WARNING
823 return True 841 return True
OLDNEW
« no previous file with comments | « no previous file | scripts/slave/recipe_modules/filter/api.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698