| OLD | NEW |
| 1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import ast | 5 import ast |
| 6 import contextlib | 6 import contextlib |
| 7 import copy | 7 import copy |
| 8 import itertools | 8 import itertools |
| 9 import json | 9 import json |
| 10 | 10 |
| (...skipping 339 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 350 command(lambda name: '%s (with patch)' % name) | 350 command(lambda name: '%s (with patch)' % name) |
| 351 except self.m.step.StepFailure: | 351 except self.m.step.StepFailure: |
| 352 self.deapply_patch(update_step) | 352 self.deapply_patch(update_step) |
| 353 command(lambda name: '%s (without patch)' % name) | 353 command(lambda name: '%s (without patch)' % name) |
| 354 raise | 354 raise |
| 355 else: | 355 else: |
| 356 command(lambda name: name) | 356 command(lambda name: name) |
| 357 | 357 |
| 358 | 358 |
| 359 def compile(self, mastername, buildername, update_step, master_dict, | 359 def compile(self, mastername, buildername, update_step, master_dict, |
| 360 test_spec): | 360 test_spec, mb_mastername=None, mb_buildername=None): |
| 361 """Runs compile and related steps for given builder.""" | 361 """Runs compile and related steps for given builder.""" |
| 362 compile_targets, tests_including_triggered = \ | 362 compile_targets, tests_including_triggered = \ |
| 363 self.get_compile_targets_and_tests( | 363 self.get_compile_targets_and_tests( |
| 364 mastername, | 364 mastername, |
| 365 buildername, | 365 buildername, |
| 366 master_dict, test_spec) | 366 master_dict, test_spec) |
| 367 self.compile_specific_targets( | 367 self.compile_specific_targets( |
| 368 mastername, buildername, update_step, master_dict, | 368 mastername, buildername, update_step, master_dict, |
| 369 compile_targets, tests_including_triggered) | 369 compile_targets, tests_including_triggered, |
| 370 mb_mastername=mb_mastername, mb_buildername=mb_buildername) |
| 370 | 371 |
| 371 def compile_specific_targets( | 372 def compile_specific_targets( |
| 372 self, mastername, buildername, update_step, master_dict, | 373 self, mastername, buildername, update_step, master_dict, |
| 373 compile_targets, tests_including_triggered, override_bot_type=None): | 374 compile_targets, tests_including_triggered, |
| 375 mb_mastername=None, mb_buildername=None, override_bot_type=None): |
| 374 """Runs compile and related steps for given builder. | 376 """Runs compile and related steps for given builder. |
| 375 | 377 |
| 376 Allows finer-grained control about exact compile targets used.""" | 378 Allows finer-grained control about exact compile targets used. |
| 379 |
| 380 We don't use the given `mastername` and `buildername` to run MB, because |
| 381 they may be the values of the continuous builder the trybot may be |
| 382 configured to match; instead we need to use the actual mastername and |
| 383 buildername we're running on (Default to the "mastername" and |
| 384 "buildername" in the build properties -- self.m.properties, but could be |
| 385 overridden by `mb_mastername` and `mb_buildername`), because it may be |
| 386 configured with different MB settings. |
| 387 |
| 388 However, recipes used by Findit for culprit finding may still set |
| 389 (mb_mastername, mb_buildername) = (mastername, buildername) to exactly match |
| 390 a given continuous builder.""" |
| 377 | 391 |
| 378 bot_config = master_dict.get('builders', {}).get(buildername) | 392 bot_config = master_dict.get('builders', {}).get(buildername) |
| 379 master_config = master_dict.get('settings', {}) | 393 master_config = master_dict.get('settings', {}) |
| 380 bot_type = override_bot_type or bot_config.get('bot_type', 'builder_tester') | 394 bot_type = override_bot_type or bot_config.get('bot_type', 'builder_tester') |
| 381 | 395 |
| 382 self.m.chromium.cleanup_temp() | 396 self.m.chromium.cleanup_temp() |
| 383 if self.m.chromium.c.TARGET_PLATFORM == 'android': | 397 if self.m.chromium.c.TARGET_PLATFORM == 'android': |
| 384 self.m.chromium_android.clean_local_files() | 398 self.m.chromium_android.clean_local_files() |
| 385 self.m.chromium_android.run_tree_truth() | 399 self.m.chromium_android.run_tree_truth() |
| 386 | 400 |
| 387 if bot_type in ['builder', 'builder_tester']: | 401 if bot_type in ['builder', 'builder_tester']: |
| 388 isolated_targets = [ | 402 isolated_targets = [ |
| 389 t.isolate_target for t in tests_including_triggered if t.uses_swarming | 403 t.isolate_target for t in tests_including_triggered if t.uses_swarming |
| 390 ] | 404 ] |
| 391 | 405 |
| 392 if isolated_targets: | 406 if isolated_targets: |
| 393 self.m.isolate.clean_isolated_files(self.m.chromium.output_dir) | 407 self.m.isolate.clean_isolated_files(self.m.chromium.output_dir) |
| 394 | 408 |
| 395 if self.m.chromium.c.project_generator.tool == 'mb': | 409 if self.m.chromium.c.project_generator.tool == 'mb': |
| 396 if bot_config.get('chromium_config') == 'chromium_win_clang': | 410 if bot_config.get('chromium_config') == 'chromium_win_clang': |
| 397 self.m.chromium.update_clang() | 411 self.m.chromium.update_clang() |
| 398 | 412 |
| 399 try: | 413 try: |
| 400 self.transient_check(update_step, lambda transform_name: | 414 self.transient_check(update_step, lambda transform_name: |
| 401 self.run_mb_and_compile(compile_targets, isolated_targets, | 415 self.run_mb_and_compile(compile_targets, isolated_targets, |
| 402 name_suffix=transform_name(''))) | 416 name_suffix=transform_name(''), |
| 417 mb_mastername=mb_mastername, |
| 418 mb_buildername=mb_buildername)) |
| 403 except self.m.step.StepFailure: | 419 except self.m.step.StepFailure: |
| 404 self.m.tryserver.set_compile_failure_tryjob_result() | 420 self.m.tryserver.set_compile_failure_tryjob_result() |
| 405 raise | 421 raise |
| 406 | 422 |
| 407 if isolated_targets: | 423 if isolated_targets: |
| 408 self.m.isolate.remove_build_metadata() | 424 self.m.isolate.remove_build_metadata() |
| 409 # 'compile' just prepares all information needed for the isolation, | 425 # 'compile' just prepares all information needed for the isolation, |
| 410 # and the isolation is a separate step. | 426 # and the isolation is a separate step. |
| 411 self.m.isolate.isolate_tests( | 427 self.m.isolate.isolate_tests( |
| 412 self.m.chromium.output_dir, | 428 self.m.chromium.output_dir, |
| (...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 460 }) | 476 }) |
| 461 | 477 |
| 462 if bot_config.get('archive_build') and not self.m.tryserver.is_tryserver: | 478 if bot_config.get('archive_build') and not self.m.tryserver.is_tryserver: |
| 463 self.m.chromium.archive_build( | 479 self.m.chromium.archive_build( |
| 464 'archive_build', | 480 'archive_build', |
| 465 bot_config['gs_bucket'], | 481 bot_config['gs_bucket'], |
| 466 bot_config.get('gs_acl'), | 482 bot_config.get('gs_acl'), |
| 467 mode='dev' | 483 mode='dev' |
| 468 ) | 484 ) |
| 469 | 485 |
| 470 def run_mb_and_compile(self, compile_targets, isolated_targets, name_suffix): | 486 def run_mb_and_compile(self, compile_targets, isolated_targets, name_suffix, |
| 487 mb_mastername=None, mb_buildername=None): |
| 471 if self.m.chromium.c.project_generator.tool == 'mb': | 488 if self.m.chromium.c.project_generator.tool == 'mb': |
| 472 # We don't use the mastername and buildername passed in, because | 489 mb_mastername = mb_mastername or self.m.properties['mastername'] |
| 473 # those may be the values of the continuous builder the trybot may | 490 mb_buildername = mb_buildername or self.m.properties['buildername'] |
| 474 # be configured to match; we need to use the actual mastername | 491 self.m.chromium.run_mb(mb_mastername, mb_buildername, |
| 475 # and buildername we're running on, because it may be configured | |
| 476 # with different MB settings. | |
| 477 real_mastername = self.m.properties['mastername'] | |
| 478 real_buildername = self.m.properties['buildername'] | |
| 479 self.m.chromium.run_mb(real_mastername, real_buildername, | |
| 480 isolated_targets=isolated_targets, | 492 isolated_targets=isolated_targets, |
| 481 name='generate_build_files%s' % name_suffix) | 493 name='generate_build_files%s' % name_suffix) |
| 482 | 494 |
| 483 self.m.chromium.compile(compile_targets, name='compile%s' % name_suffix) | 495 self.m.chromium.compile(compile_targets, name='compile%s' % name_suffix) |
| 484 | 496 |
| 485 def tests_for_builder(self, mastername, buildername, update_step, master_dict, | 497 def tests_for_builder(self, mastername, buildername, update_step, master_dict, |
| 486 override_bot_type=None): | 498 override_bot_type=None): |
| 487 got_revision = update_step.presentation.properties['got_revision'] | 499 got_revision = update_step.presentation.properties['got_revision'] |
| 488 | 500 |
| 489 bot_config = master_dict.get('builders', {}).get(buildername) | 501 bot_config = master_dict.get('builders', {}).get(buildername) |
| (...skipping 101 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 591 # mapping for their own purposes. | 603 # mapping for their own purposes. |
| 592 first_solution_name = self.m.gclient.c.solutions[0].name | 604 first_solution_name = self.m.gclient.c.solutions[0].name |
| 593 rev_property = self.m.gclient.c.got_revision_mapping[first_solution_name] | 605 rev_property = self.m.gclient.c.got_revision_mapping[first_solution_name] |
| 594 self.m.gclient.c.revisions[first_solution_name] = str( | 606 self.m.gclient.c.revisions[first_solution_name] = str( |
| 595 bot_update_json['properties'][rev_property]) | 607 bot_update_json['properties'][rev_property]) |
| 596 self.m.bot_update.ensure_checkout( | 608 self.m.bot_update.ensure_checkout( |
| 597 force=True, patch=False, update_presentation=False) | 609 force=True, patch=False, update_presentation=False) |
| 598 self.m.chromium.runhooks(name='runhooks (without patch)') | 610 self.m.chromium.runhooks(name='runhooks (without patch)') |
| 599 | 611 |
| 600 def run_tests_on_tryserver(self, mastername, api, tests, bot_update_step, | 612 def run_tests_on_tryserver(self, mastername, api, tests, bot_update_step, |
| 601 affected_files): | 613 affected_files, mb_mastername=None, |
| 614 mb_buildername=None): |
| 602 def deapply_patch_fn(failing_tests): | 615 def deapply_patch_fn(failing_tests): |
| 603 self.deapply_patch(bot_update_step) | 616 self.deapply_patch(bot_update_step) |
| 604 compile_targets = list(itertools.chain( | 617 compile_targets = list(itertools.chain( |
| 605 *[t.compile_targets(api) for t in failing_tests])) | 618 *[t.compile_targets(api) for t in failing_tests])) |
| 606 if compile_targets: | 619 if compile_targets: |
| 607 # Remove duplicate targets. | 620 # Remove duplicate targets. |
| 608 compile_targets = sorted(set(compile_targets)) | 621 compile_targets = sorted(set(compile_targets)) |
| 609 failing_swarming_tests = [ | 622 failing_swarming_tests = [ |
| 610 t.isolate_target for t in failing_tests if t.uses_swarming] | 623 t.isolate_target for t in failing_tests if t.uses_swarming] |
| 611 if failing_swarming_tests: | 624 if failing_swarming_tests: |
| 612 self.m.isolate.clean_isolated_files(self.m.chromium.output_dir) | 625 self.m.isolate.clean_isolated_files(self.m.chromium.output_dir) |
| 613 self.run_mb_and_compile(compile_targets, failing_swarming_tests, | 626 self.run_mb_and_compile(compile_targets, failing_swarming_tests, |
| 614 ' (without patch)') | 627 ' (without patch)', |
| 628 mb_mastername=mb_mastername, |
| 629 mb_buildername=mb_buildername) |
| 615 if failing_swarming_tests: | 630 if failing_swarming_tests: |
| 616 self.m.isolate.isolate_tests(self.m.chromium.output_dir, | 631 self.m.isolate.isolate_tests(self.m.chromium.output_dir, |
| 617 verbose=True) | 632 verbose=True) |
| 618 | 633 |
| 619 deapply_patch = True | 634 deapply_patch = True |
| 620 for path in RECIPE_CONFIG_PATHS: | 635 for path in RECIPE_CONFIG_PATHS: |
| 621 if any([f.startswith(path) for f in affected_files]): | 636 if any([f.startswith(path) for f in affected_files]): |
| 622 deapply_patch = False | 637 deapply_patch = False |
| 623 break | 638 break |
| 624 | 639 |
| 625 with self.wrap_chromium_tests(mastername, tests): | 640 with self.wrap_chromium_tests(mastername, tests): |
| 626 if deapply_patch: | 641 if deapply_patch: |
| 627 self.m.test_utils.determine_new_failures(api, tests, deapply_patch_fn) | 642 self.m.test_utils.determine_new_failures(api, tests, deapply_patch_fn) |
| 628 else: | 643 else: |
| 629 failing_tests = self.m.test_utils.run_tests_with_patch(api, tests) | 644 failing_tests = self.m.test_utils.run_tests_with_patch(api, tests) |
| 630 if failing_tests: | 645 if failing_tests: |
| 631 self.m.python.failing_step('test results', 'TESTS FAILED') | 646 self.m.python.failing_step('test results', 'TESTS FAILED') |
| 632 | 647 |
| 633 def analyze(self, affected_files, test_targets, additional_compile_targets, | 648 def analyze(self, affected_files, test_targets, additional_compile_targets, |
| 634 config_file_name, additional_names=None): | 649 config_file_name, mb_mastername=None, mb_buildername=None, |
| 650 additional_names=None): |
| 635 """Runs "analyze" step to determine targets affected by the patch. | 651 """Runs "analyze" step to determine targets affected by the patch. |
| 636 | 652 |
| 637 Returns a tuple of: | 653 Returns a tuple of: |
| 638 - list of targets that are needed to run tests (see filter recipe module) | 654 - list of targets that are needed to run tests (see filter recipe module) |
| 639 - list of targets that need to be compiled (see filter recipe module)""" | 655 - list of targets that need to be compiled (see filter recipe module)""" |
| 640 | 656 |
| 641 if additional_names is None: | 657 if additional_names is None: |
| 642 additional_names = ['chromium'] | 658 additional_names = ['chromium'] |
| 643 | 659 |
| 644 use_mb = (self.m.chromium.c.project_generator.tool == 'mb') | 660 use_mb = (self.m.chromium.c.project_generator.tool == 'mb') |
| 645 build_output_dir = '//out/%s' % self.m.chromium.c.build_config_fs | 661 build_output_dir = '//out/%s' % self.m.chromium.c.build_config_fs |
| 646 self.m.filter.does_patch_require_compile( | 662 self.m.filter.does_patch_require_compile( |
| 647 affected_files, | 663 affected_files, |
| 648 test_targets=test_targets, | 664 test_targets=test_targets, |
| 649 additional_compile_targets=additional_compile_targets, | 665 additional_compile_targets=additional_compile_targets, |
| 650 additional_names=additional_names, | 666 additional_names=additional_names, |
| 651 config_file_name=config_file_name, | 667 config_file_name=config_file_name, |
| 652 use_mb=use_mb, | 668 use_mb=use_mb, |
| 669 mb_mastername=mb_mastername, |
| 670 mb_buildername=mb_buildername, |
| 653 build_output_dir=build_output_dir, | 671 build_output_dir=build_output_dir, |
| 654 cros_board=self.m.chromium.c.TARGET_CROS_BOARD) | 672 cros_board=self.m.chromium.c.TARGET_CROS_BOARD) |
| 655 | 673 |
| 656 compile_targets = self.m.filter.compile_targets[:] | 674 compile_targets = self.m.filter.compile_targets[:] |
| 657 | 675 |
| 658 # Add crash_service to compile_targets. This is done after filtering compile | 676 # Add crash_service to compile_targets. This is done after filtering compile |
| 659 # targets out because crash_service should always be there on windows. | 677 # targets out because crash_service should always be there on windows. |
| 660 # TODO(akuegel): Need to solve this in a better way. crbug.com/478053 | 678 # TODO(akuegel): Need to solve this in a better way. crbug.com/478053 |
| 661 if (self.m.platform.is_win and compile_targets and | 679 if (self.m.platform.is_win and compile_targets and |
| 662 'crash_service' not in compile_targets): | 680 'crash_service' not in compile_targets): |
| (...skipping 151 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 814 result_text = 'MB is enabled for this builder at this revision.' | 832 result_text = 'MB is enabled for this builder at this revision.' |
| 815 log_name = 'Builder MB-ready' | 833 log_name = 'Builder MB-ready' |
| 816 self.m.step.active_result.presentation.logs[log_name] = [result_text] | 834 self.m.step.active_result.presentation.logs[log_name] = [result_text] |
| 817 return False | 835 return False |
| 818 except (self.m.step.StepFailure, KeyError): | 836 except (self.m.step.StepFailure, KeyError): |
| 819 result_text = 'MB is not enabled for this builder at this revision.' | 837 result_text = 'MB is not enabled for this builder at this revision.' |
| 820 log_name = 'Builder NOT MB-ready' | 838 log_name = 'Builder NOT MB-ready' |
| 821 self.m.step.active_result.presentation.logs[log_name] = [result_text] | 839 self.m.step.active_result.presentation.logs[log_name] = [result_text] |
| 822 self.m.step.active_result.presentation.status = self.m.step.WARNING | 840 self.m.step.active_result.presentation.status = self.m.step.WARNING |
| 823 return True | 841 return True |
| OLD | NEW |