| OLD | NEW |
| 1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import contextlib | 5 import contextlib |
| 6 import copy | 6 import copy |
| 7 import itertools | 7 import itertools |
| 8 import json | 8 import json |
| 9 | 9 |
| 10 from recipe_engine.types import freeze | 10 from recipe_engine.types import freeze |
| (...skipping 12 matching lines...) Expand all Loading... |
| 23 ] | 23 ] |
| 24 | 24 |
| 25 | 25 |
| 26 class ChromiumTestsApi(recipe_api.RecipeApi): | 26 class ChromiumTestsApi(recipe_api.RecipeApi): |
| 27 def __init__(self, *args, **kwargs): | 27 def __init__(self, *args, **kwargs): |
| 28 super(ChromiumTestsApi, self).__init__(*args, **kwargs) | 28 super(ChromiumTestsApi, self).__init__(*args, **kwargs) |
| 29 self._builders = {} | 29 self._builders = {} |
| 30 self.add_builders(builders.BUILDERS) | 30 self.add_builders(builders.BUILDERS) |
| 31 self._precommit_mode = False | 31 self._precommit_mode = False |
| 32 | 32 |
| 33 # Keep track of working directory (which contains the checkout). | |
| 34 # None means "default value". | |
| 35 self._working_dir = None | |
| 36 | |
| 37 @property | 33 @property |
| 38 def builders(self): | 34 def builders(self): |
| 39 return self._builders | 35 return self._builders |
| 40 | 36 |
| 41 @property | 37 @property |
| 42 def steps(self): | 38 def steps(self): |
| 43 return steps | 39 return steps |
| 44 | 40 |
| 45 @property | 41 @property |
| 46 def trybots(self): | 42 def trybots(self): |
| (...skipping 76 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 123 # (3) 'HEAD' for forced builds with unspecified 'revision'. | 119 # (3) 'HEAD' for forced builds with unspecified 'revision'. |
| 124 # TODO(machenbach): Use parent_got_cr_revision on testers with component | 120 # TODO(machenbach): Use parent_got_cr_revision on testers with component |
| 125 # builds to match also the chromium revision from the builder. | 121 # builds to match also the chromium revision from the builder. |
| 126 component_rev = self.m.properties.get('revision') or 'HEAD' | 122 component_rev = self.m.properties.get('revision') or 'HEAD' |
| 127 if bot_type == 'tester': | 123 if bot_type == 'tester': |
| 128 component_rev = self.m.properties.get( | 124 component_rev = self.m.properties.get( |
| 129 'parent_got_revision', component_rev) | 125 'parent_got_revision', component_rev) |
| 130 dep = bot_config.get('set_component_rev') | 126 dep = bot_config.get('set_component_rev') |
| 131 self.m.gclient.c.revisions[dep['name']] = dep['rev_str'] % component_rev | 127 self.m.gclient.c.revisions[dep['name']] = dep['rev_str'] % component_rev |
| 132 | 128 |
| 133 def get_checkout_dir(self, bot_config): | 129 # TODO(phajdan.jr): fix callers and remove chromium_tests.get_checkout_dir. |
| 134 try: | 130 def get_checkout_dir(self, bot_config): # pragma: no cover |
| 135 builder_cache = self.m.path['builder_cache'] | 131 return self.m.chromium_checkout.get_checkout_dir(bot_config) |
| 136 except KeyError: # no-op if builder cache is not set up. | |
| 137 return None | |
| 138 else: | |
| 139 sanitized_buildername = ''.join( | |
| 140 c if c.isalnum() else '_' for c in self.m.properties['buildername']) | |
| 141 checkout_dir = builder_cache.join( | |
| 142 bot_config.get('checkout_dir', sanitized_buildername)) | |
| 143 self.m.shutil.makedirs('checkout path', checkout_dir) | |
| 144 return checkout_dir | |
| 145 | 132 |
| 133 # TODO(phajdan.jr): fix callers and remove chromium_tests.ensure_checkout. |
| 146 def ensure_checkout(self, bot_config, root_solution_revision=None, | 134 def ensure_checkout(self, bot_config, root_solution_revision=None, |
| 147 force=False): | 135 force=False): |
| 148 if self.m.platform.is_win: | 136 return self.m.chromium_checkout.ensure_checkout( # pragma: no cover |
| 149 self.m.chromium.taskkill() | 137 bot_config, root_solution_revision, force) |
| 150 | |
| 151 kwargs = {} | |
| 152 self._working_dir = self.get_checkout_dir(bot_config) | |
| 153 if self._working_dir: | |
| 154 kwargs['cwd'] = self._working_dir | |
| 155 | |
| 156 # Bot Update re-uses the gclient configs. | |
| 157 update_step = self.m.bot_update.ensure_checkout( | |
| 158 patch_root=bot_config.get('patch_root'), | |
| 159 root_solution_revision=root_solution_revision, | |
| 160 clobber=bot_config.get('clobber', False), | |
| 161 force=force, **kwargs) | |
| 162 assert update_step.json.output['did_run'] | |
| 163 # HACK(dnj): Remove after 'crbug.com/398105' has landed | |
| 164 self.m.chromium.set_build_properties(update_step.json.output['properties']) | |
| 165 | |
| 166 return update_step | |
| 167 | 138 |
| 168 def set_up_swarming(self, bot_config): | 139 def set_up_swarming(self, bot_config): |
| 169 if not bot_config.get('enable_swarming'): | 140 if not bot_config.get('enable_swarming'): |
| 170 return | 141 return |
| 171 self.m.isolate.set_isolate_environment(self.m.chromium.c) | 142 self.m.isolate.set_isolate_environment(self.m.chromium.c) |
| 172 self.m.swarming.check_client_version() | 143 self.m.swarming.check_client_version() |
| 173 for key, value in bot_config.get('swarming_dimensions', {}).iteritems(): | 144 for key, value in bot_config.get('swarming_dimensions', {}).iteritems(): |
| 174 self.m.swarming.set_default_dimension(key, value) | 145 self.m.swarming.set_default_dimension(key, value) |
| 175 | 146 |
| 176 def runhooks(self, update_step): | 147 def runhooks(self, update_step): |
| (...skipping 10 matching lines...) Expand all Loading... |
| 187 # TODO(phajdan.jr): remove create_bot_db_from_master_dict. It adds another | 158 # TODO(phajdan.jr): remove create_bot_db_from_master_dict. It adds another |
| 188 # entry point to _add_master_dict_and_test_spec which can really complicate | 159 # entry point to _add_master_dict_and_test_spec which can really complicate |
| 189 # things. | 160 # things. |
| 190 def create_bot_db_from_master_dict(self, mastername, master_dict): | 161 def create_bot_db_from_master_dict(self, mastername, master_dict): |
| 191 bot_db = bdb_module.BotConfigAndTestDB() | 162 bot_db = bdb_module.BotConfigAndTestDB() |
| 192 bot_db._add_master_dict_and_test_spec(mastername, master_dict, {}) | 163 bot_db._add_master_dict_and_test_spec(mastername, master_dict, {}) |
| 193 return bot_db | 164 return bot_db |
| 194 | 165 |
| 195 def prepare_checkout(self, bot_config, root_solution_revision=None, | 166 def prepare_checkout(self, bot_config, root_solution_revision=None, |
| 196 force=False): | 167 force=False): |
| 197 update_step = self.ensure_checkout( | 168 update_step = self.m.chromium_checkout.ensure_checkout( |
| 198 bot_config, root_solution_revision, force=force) | 169 bot_config, root_solution_revision, force=force) |
| 199 | 170 |
| 200 if (self.m.chromium.c.compile_py.compiler and | 171 if (self.m.chromium.c.compile_py.compiler and |
| 201 'goma' in self.m.chromium.c.compile_py.compiler): | 172 'goma' in self.m.chromium.c.compile_py.compiler): |
| 202 self.m.chromium.ensure_goma(canary=bot_config.get('goma_canary', False)) | 173 self.m.chromium.ensure_goma(canary=bot_config.get('goma_canary', False)) |
| 203 | 174 |
| 204 self.set_up_swarming(bot_config) | 175 self.set_up_swarming(bot_config) |
| 205 self.runhooks(update_step) | 176 self.runhooks(update_step) |
| 206 | 177 |
| 207 bot_db = bdb_module.BotConfigAndTestDB() | 178 bot_db = bdb_module.BotConfigAndTestDB() |
| (...skipping 329 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 537 def _make_legacy_build_url(self, master_config, mastername): | 508 def _make_legacy_build_url(self, master_config, mastername): |
| 538 return self.m.archive.legacy_download_url( | 509 return self.m.archive.legacy_download_url( |
| 539 master_config.get('build_gs_bucket'), | 510 master_config.get('build_gs_bucket'), |
| 540 extra_url_components=( | 511 extra_url_components=( |
| 541 None if mastername.startswith('chromium.perf') | 512 None if mastername.startswith('chromium.perf') |
| 542 else self.m.properties['mastername'])) | 513 else self.m.properties['mastername'])) |
| 543 | 514 |
| 544 @contextlib.contextmanager | 515 @contextlib.contextmanager |
| 545 def wrap_chromium_tests(self, bot_config, tests=None): | 516 def wrap_chromium_tests(self, bot_config, tests=None): |
| 546 context = { | 517 context = { |
| 547 'cwd': self._working_dir, | 518 'cwd': self.m.chromium_checkout.working_dir, |
| 548 'env': self.m.chromium.get_env(), | 519 'env': self.m.chromium.get_env(), |
| 549 } | 520 } |
| 550 with self.m.step.context(context): | 521 with self.m.step.context(context): |
| 551 bot_type = bot_config.get('bot_type', 'builder_tester') | 522 bot_type = bot_config.get('bot_type', 'builder_tester') |
| 552 | 523 |
| 553 if bot_type in ('tester', 'builder_tester'): | 524 if bot_type in ('tester', 'builder_tester'): |
| 554 isolated_targets = [ | 525 isolated_targets = [ |
| 555 t.isolate_target(self.m) for t in tests if t.uses_swarming] | 526 t.isolate_target(self.m) for t in tests if t.uses_swarming] |
| 556 if isolated_targets: | 527 if isolated_targets: |
| 557 self.m.isolate.find_isolated_tests(self.m.chromium.output_dir) | 528 self.m.isolate.find_isolated_tests(self.m.chromium.output_dir) |
| (...skipping 81 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 639 # is also imporant that we do not assume that corresponding revision is | 610 # is also imporant that we do not assume that corresponding revision is |
| 640 # stored in the 'got_revision' as some gclient configs change the default | 611 # stored in the 'got_revision' as some gclient configs change the default |
| 641 # mapping for their own purposes. | 612 # mapping for their own purposes. |
| 642 first_solution_name = self.m.gclient.c.solutions[0].name | 613 first_solution_name = self.m.gclient.c.solutions[0].name |
| 643 rev_property = self.m.gclient.c.got_revision_mapping[first_solution_name] | 614 rev_property = self.m.gclient.c.got_revision_mapping[first_solution_name] |
| 644 self.m.gclient.c.revisions[first_solution_name] = str( | 615 self.m.gclient.c.revisions[first_solution_name] = str( |
| 645 bot_update_json['properties'][rev_property]) | 616 bot_update_json['properties'][rev_property]) |
| 646 self._resolve_fixed_revisions(bot_update_json) | 617 self._resolve_fixed_revisions(bot_update_json) |
| 647 | 618 |
| 648 kwargs = {} | 619 kwargs = {} |
| 649 if self._working_dir: | 620 if self.m.chromium_checkout.working_dir: |
| 650 kwargs['cwd'] = self._working_dir | 621 kwargs['cwd'] = self.m.chromium_checkout.working_dir |
| 651 | 622 |
| 652 self.m.bot_update.ensure_checkout( | 623 self.m.bot_update.ensure_checkout( |
| 653 force=True, patch=False, update_presentation=False, **kwargs) | 624 force=True, patch=False, update_presentation=False, **kwargs) |
| 654 self.m.chromium.runhooks(name='runhooks (without patch)') | 625 self.m.chromium.runhooks(name='runhooks (without patch)') |
| 655 | 626 |
| 656 def run_tests_on_tryserver(self, bot_config, api, tests, bot_update_step, | 627 def run_tests_on_tryserver(self, bot_config, api, tests, bot_update_step, |
| 657 affected_files, mb_mastername=None, | 628 affected_files, mb_mastername=None, |
| 658 mb_buildername=None): | 629 mb_buildername=None): |
| 659 def deapply_patch_fn(failing_tests): | 630 def deapply_patch_fn(failing_tests): |
| 660 self.deapply_patch(bot_update_step) | 631 self.deapply_patch(bot_update_step) |
| (...skipping 26 matching lines...) Expand all Loading... |
| 687 if deapply_patch: | 658 if deapply_patch: |
| 688 self.m.test_utils.determine_new_failures(api, tests, deapply_patch_fn) | 659 self.m.test_utils.determine_new_failures(api, tests, deapply_patch_fn) |
| 689 else: | 660 else: |
| 690 failing_tests = self.m.test_utils.run_tests_with_patch(api, tests) | 661 failing_tests = self.m.test_utils.run_tests_with_patch(api, tests) |
| 691 if failing_tests: | 662 if failing_tests: |
| 692 self.m.python.failing_step( | 663 self.m.python.failing_step( |
| 693 'test results', | 664 'test results', |
| 694 'TESTS FAILED; retries without patch disabled (%s)' | 665 'TESTS FAILED; retries without patch disabled (%s)' |
| 695 % deapply_patch_reason) | 666 % deapply_patch_reason) |
| 696 | 667 |
| 668 # TODO(phajdan.jr): fix callers and remove get_files_affected_by_patch. |
| 697 def get_files_affected_by_patch(self, relative_to='src/', cwd=None): | 669 def get_files_affected_by_patch(self, relative_to='src/', cwd=None): |
| 698 """Returns list of POSIX paths of files affected by patch for "analyze". | 670 return self.m.chromium_checkout.get_files_affected_by_patch( # pragma: no c
over |
| 699 | 671 relative_to, cwd) |
| 700 Paths are relative to `relative_to` which for analyze should be 'src/'. | |
| 701 """ | |
| 702 patch_root = self.m.gclient.calculate_patch_root( | |
| 703 self.m.properties.get('patch_project')) | |
| 704 if not cwd: | |
| 705 cwd = self._working_dir.join(patch_root) if self._working_dir else None | |
| 706 files = self.m.tryserver.get_files_affected_by_patch(patch_root, cwd=cwd) | |
| 707 for i, path in enumerate(files): | |
| 708 path = str(path) | |
| 709 assert path.startswith(relative_to) | |
| 710 files[i] = path[len(relative_to):] | |
| 711 return files | |
| 712 | 672 |
| 713 # TODO(phajdan.jr): fix callers and remove chromium_tests.configure_swarming. | 673 # TODO(phajdan.jr): fix callers and remove chromium_tests.configure_swarming. |
| 714 def configure_swarming(self, project_name, precommit, mastername=None): | 674 def configure_swarming(self, project_name, precommit, mastername=None): |
| 715 return self.m.chromium_swarming.configure_swarming( # pragma: no cover | 675 return self.m.chromium_swarming.configure_swarming( # pragma: no cover |
| 716 project_name, precommit, mastername) | 676 project_name, precommit, mastername) |
| 717 | 677 |
| 718 def _build_gs_archive_url(self, mastername, master_config, buildername): | 678 def _build_gs_archive_url(self, mastername, master_config, buildername): |
| 719 """Returns the archive URL to pass to self.m.archive.zip_and_upload_build. | 679 """Returns the archive URL to pass to self.m.archive.zip_and_upload_build. |
| 720 | 680 |
| 721 Most builders on most masters use a standard format for the build archive | 681 Most builders on most masters use a standard format for the build archive |
| (...skipping 96 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 818 | 778 |
| 819 if not tests: | 779 if not tests: |
| 820 return | 780 return |
| 821 | 781 |
| 822 api.chromium_swarming.configure_swarming( | 782 api.chromium_swarming.configure_swarming( |
| 823 'chromium', precommit=False, mastername=mastername) | 783 'chromium', precommit=False, mastername=mastername) |
| 824 test_runner = api.chromium_tests.create_test_runner( | 784 test_runner = api.chromium_tests.create_test_runner( |
| 825 api, tests, serialize_tests=bot_config.get('serialize_tests')) | 785 api, tests, serialize_tests=bot_config.get('serialize_tests')) |
| 826 with api.chromium_tests.wrap_chromium_tests(bot_config, tests): | 786 with api.chromium_tests.wrap_chromium_tests(bot_config, tests): |
| 827 test_runner() | 787 test_runner() |
| OLD | NEW |