OLD | NEW |
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import contextlib | 5 import contextlib |
6 import copy | 6 import copy |
7 import itertools | 7 import itertools |
8 import json | 8 import json |
9 | 9 |
10 from recipe_engine.types import freeze | 10 from recipe_engine.types import freeze |
(...skipping 12 matching lines...) Expand all Loading... |
23 ] | 23 ] |
24 | 24 |
25 | 25 |
26 class ChromiumTestsApi(recipe_api.RecipeApi): | 26 class ChromiumTestsApi(recipe_api.RecipeApi): |
27 def __init__(self, *args, **kwargs): | 27 def __init__(self, *args, **kwargs): |
28 super(ChromiumTestsApi, self).__init__(*args, **kwargs) | 28 super(ChromiumTestsApi, self).__init__(*args, **kwargs) |
29 self._builders = {} | 29 self._builders = {} |
30 self.add_builders(builders.BUILDERS) | 30 self.add_builders(builders.BUILDERS) |
31 self._precommit_mode = False | 31 self._precommit_mode = False |
32 | 32 |
33 # Keep track of working directory (which contains the checkout). | |
34 # None means "default value". | |
35 self._working_dir = None | |
36 | |
37 @property | 33 @property |
38 def builders(self): | 34 def builders(self): |
39 return self._builders | 35 return self._builders |
40 | 36 |
41 @property | 37 @property |
42 def steps(self): | 38 def steps(self): |
43 return steps | 39 return steps |
44 | 40 |
45 @property | 41 @property |
46 def trybots(self): | 42 def trybots(self): |
(...skipping 76 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
123 # (3) 'HEAD' for forced builds with unspecified 'revision'. | 119 # (3) 'HEAD' for forced builds with unspecified 'revision'. |
124 # TODO(machenbach): Use parent_got_cr_revision on testers with component | 120 # TODO(machenbach): Use parent_got_cr_revision on testers with component |
125 # builds to match also the chromium revision from the builder. | 121 # builds to match also the chromium revision from the builder. |
126 component_rev = self.m.properties.get('revision') or 'HEAD' | 122 component_rev = self.m.properties.get('revision') or 'HEAD' |
127 if bot_type == 'tester': | 123 if bot_type == 'tester': |
128 component_rev = self.m.properties.get( | 124 component_rev = self.m.properties.get( |
129 'parent_got_revision', component_rev) | 125 'parent_got_revision', component_rev) |
130 dep = bot_config.get('set_component_rev') | 126 dep = bot_config.get('set_component_rev') |
131 self.m.gclient.c.revisions[dep['name']] = dep['rev_str'] % component_rev | 127 self.m.gclient.c.revisions[dep['name']] = dep['rev_str'] % component_rev |
132 | 128 |
133 def get_checkout_dir(self, bot_config): | 129 # TODO(phajdan.jr): fix callers and remove chromium_tests.get_checkout_dir. |
134 try: | 130 def get_checkout_dir(self, bot_config): # pragma: no cover |
135 builder_cache = self.m.path['builder_cache'] | 131 return self.m.chromium_checkout.get_checkout_dir(bot_config) |
136 except KeyError: # no-op if builder cache is not set up. | |
137 return None | |
138 else: | |
139 sanitized_buildername = ''.join( | |
140 c if c.isalnum() else '_' for c in self.m.properties['buildername']) | |
141 checkout_dir = builder_cache.join( | |
142 bot_config.get('checkout_dir', sanitized_buildername)) | |
143 self.m.shutil.makedirs('checkout path', checkout_dir) | |
144 return checkout_dir | |
145 | 132 |
| 133 # TODO(phajdan.jr): fix callers and remove chromium_tests.ensure_checkout. |
146 def ensure_checkout(self, bot_config, root_solution_revision=None, | 134 def ensure_checkout(self, bot_config, root_solution_revision=None, |
147 force=False): | 135 force=False): |
148 if self.m.platform.is_win: | 136 return self.m.chromium_checkout.ensure_checkout( # pragma: no cover |
149 self.m.chromium.taskkill() | 137 bot_config, root_solution_revision, force) |
150 | |
151 kwargs = {} | |
152 self._working_dir = self.get_checkout_dir(bot_config) | |
153 if self._working_dir: | |
154 kwargs['cwd'] = self._working_dir | |
155 | |
156 # Bot Update re-uses the gclient configs. | |
157 update_step = self.m.bot_update.ensure_checkout( | |
158 patch_root=bot_config.get('patch_root'), | |
159 root_solution_revision=root_solution_revision, | |
160 clobber=bot_config.get('clobber', False), | |
161 force=force, **kwargs) | |
162 assert update_step.json.output['did_run'] | |
163 # HACK(dnj): Remove after 'crbug.com/398105' has landed | |
164 self.m.chromium.set_build_properties(update_step.json.output['properties']) | |
165 | |
166 return update_step | |
167 | 138 |
168 def set_up_swarming(self, bot_config): | 139 def set_up_swarming(self, bot_config): |
169 if not bot_config.get('enable_swarming'): | 140 if not bot_config.get('enable_swarming'): |
170 return | 141 return |
171 self.m.isolate.set_isolate_environment(self.m.chromium.c) | 142 self.m.isolate.set_isolate_environment(self.m.chromium.c) |
172 self.m.swarming.check_client_version() | 143 self.m.swarming.check_client_version() |
173 for key, value in bot_config.get('swarming_dimensions', {}).iteritems(): | 144 for key, value in bot_config.get('swarming_dimensions', {}).iteritems(): |
174 self.m.swarming.set_default_dimension(key, value) | 145 self.m.swarming.set_default_dimension(key, value) |
175 | 146 |
176 def runhooks(self, update_step): | 147 def runhooks(self, update_step): |
(...skipping 10 matching lines...) Expand all Loading... |
187 # TODO(phajdan.jr): remove create_bot_db_from_master_dict. It adds another | 158 # TODO(phajdan.jr): remove create_bot_db_from_master_dict. It adds another |
188 # entry point to _add_master_dict_and_test_spec which can really complicate | 159 # entry point to _add_master_dict_and_test_spec which can really complicate |
189 # things. | 160 # things. |
190 def create_bot_db_from_master_dict(self, mastername, master_dict): | 161 def create_bot_db_from_master_dict(self, mastername, master_dict): |
191 bot_db = bdb_module.BotConfigAndTestDB() | 162 bot_db = bdb_module.BotConfigAndTestDB() |
192 bot_db._add_master_dict_and_test_spec(mastername, master_dict, {}) | 163 bot_db._add_master_dict_and_test_spec(mastername, master_dict, {}) |
193 return bot_db | 164 return bot_db |
194 | 165 |
195 def prepare_checkout(self, bot_config, root_solution_revision=None, | 166 def prepare_checkout(self, bot_config, root_solution_revision=None, |
196 force=False): | 167 force=False): |
197 update_step = self.ensure_checkout( | 168 update_step = self.m.chromium_checkout.ensure_checkout( |
198 bot_config, root_solution_revision, force=force) | 169 bot_config, root_solution_revision, force=force) |
199 | 170 |
200 if (self.m.chromium.c.compile_py.compiler and | 171 if (self.m.chromium.c.compile_py.compiler and |
201 'goma' in self.m.chromium.c.compile_py.compiler): | 172 'goma' in self.m.chromium.c.compile_py.compiler): |
202 self.m.chromium.ensure_goma(canary=bot_config.get('goma_canary', False)) | 173 self.m.chromium.ensure_goma(canary=bot_config.get('goma_canary', False)) |
203 | 174 |
204 self.set_up_swarming(bot_config) | 175 self.set_up_swarming(bot_config) |
205 self.runhooks(update_step) | 176 self.runhooks(update_step) |
206 | 177 |
207 bot_db = bdb_module.BotConfigAndTestDB() | 178 bot_db = bdb_module.BotConfigAndTestDB() |
(...skipping 329 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
537 def _make_legacy_build_url(self, master_config, mastername): | 508 def _make_legacy_build_url(self, master_config, mastername): |
538 return self.m.archive.legacy_download_url( | 509 return self.m.archive.legacy_download_url( |
539 master_config.get('build_gs_bucket'), | 510 master_config.get('build_gs_bucket'), |
540 extra_url_components=( | 511 extra_url_components=( |
541 None if mastername.startswith('chromium.perf') | 512 None if mastername.startswith('chromium.perf') |
542 else self.m.properties['mastername'])) | 513 else self.m.properties['mastername'])) |
543 | 514 |
544 @contextlib.contextmanager | 515 @contextlib.contextmanager |
545 def wrap_chromium_tests(self, bot_config, tests=None): | 516 def wrap_chromium_tests(self, bot_config, tests=None): |
546 context = { | 517 context = { |
547 'cwd': self._working_dir, | 518 'cwd': self.m.chromium_checkout.working_dir, |
548 'env': self.m.chromium.get_env(), | 519 'env': self.m.chromium.get_env(), |
549 } | 520 } |
550 with self.m.step.context(context): | 521 with self.m.step.context(context): |
551 bot_type = bot_config.get('bot_type', 'builder_tester') | 522 bot_type = bot_config.get('bot_type', 'builder_tester') |
552 | 523 |
553 if bot_type in ('tester', 'builder_tester'): | 524 if bot_type in ('tester', 'builder_tester'): |
554 isolated_targets = [ | 525 isolated_targets = [ |
555 t.isolate_target(self.m) for t in tests if t.uses_swarming] | 526 t.isolate_target(self.m) for t in tests if t.uses_swarming] |
556 if isolated_targets: | 527 if isolated_targets: |
557 self.m.isolate.find_isolated_tests(self.m.chromium.output_dir) | 528 self.m.isolate.find_isolated_tests(self.m.chromium.output_dir) |
(...skipping 81 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
639 # is also imporant that we do not assume that corresponding revision is | 610 # is also imporant that we do not assume that corresponding revision is |
640 # stored in the 'got_revision' as some gclient configs change the default | 611 # stored in the 'got_revision' as some gclient configs change the default |
641 # mapping for their own purposes. | 612 # mapping for their own purposes. |
642 first_solution_name = self.m.gclient.c.solutions[0].name | 613 first_solution_name = self.m.gclient.c.solutions[0].name |
643 rev_property = self.m.gclient.c.got_revision_mapping[first_solution_name] | 614 rev_property = self.m.gclient.c.got_revision_mapping[first_solution_name] |
644 self.m.gclient.c.revisions[first_solution_name] = str( | 615 self.m.gclient.c.revisions[first_solution_name] = str( |
645 bot_update_json['properties'][rev_property]) | 616 bot_update_json['properties'][rev_property]) |
646 self._resolve_fixed_revisions(bot_update_json) | 617 self._resolve_fixed_revisions(bot_update_json) |
647 | 618 |
648 kwargs = {} | 619 kwargs = {} |
649 if self._working_dir: | 620 if self.m.chromium_checkout.working_dir: |
650 kwargs['cwd'] = self._working_dir | 621 kwargs['cwd'] = self.m.chromium_checkout.working_dir |
651 | 622 |
652 self.m.bot_update.ensure_checkout( | 623 self.m.bot_update.ensure_checkout( |
653 force=True, patch=False, update_presentation=False, **kwargs) | 624 force=True, patch=False, update_presentation=False, **kwargs) |
654 self.m.chromium.runhooks(name='runhooks (without patch)') | 625 self.m.chromium.runhooks(name='runhooks (without patch)') |
655 | 626 |
656 def run_tests_on_tryserver(self, bot_config, api, tests, bot_update_step, | 627 def run_tests_on_tryserver(self, bot_config, api, tests, bot_update_step, |
657 affected_files, mb_mastername=None, | 628 affected_files, mb_mastername=None, |
658 mb_buildername=None): | 629 mb_buildername=None): |
659 def deapply_patch_fn(failing_tests): | 630 def deapply_patch_fn(failing_tests): |
660 self.deapply_patch(bot_update_step) | 631 self.deapply_patch(bot_update_step) |
(...skipping 26 matching lines...) Expand all Loading... |
687 if deapply_patch: | 658 if deapply_patch: |
688 self.m.test_utils.determine_new_failures(api, tests, deapply_patch_fn) | 659 self.m.test_utils.determine_new_failures(api, tests, deapply_patch_fn) |
689 else: | 660 else: |
690 failing_tests = self.m.test_utils.run_tests_with_patch(api, tests) | 661 failing_tests = self.m.test_utils.run_tests_with_patch(api, tests) |
691 if failing_tests: | 662 if failing_tests: |
692 self.m.python.failing_step( | 663 self.m.python.failing_step( |
693 'test results', | 664 'test results', |
694 'TESTS FAILED; retries without patch disabled (%s)' | 665 'TESTS FAILED; retries without patch disabled (%s)' |
695 % deapply_patch_reason) | 666 % deapply_patch_reason) |
696 | 667 |
| 668 # TODO(phajdan.jr): fix callers and remove get_files_affected_by_patch. |
697 def get_files_affected_by_patch(self, relative_to='src/', cwd=None): | 669 def get_files_affected_by_patch(self, relative_to='src/', cwd=None): |
698 """Returns list of POSIX paths of files affected by patch for "analyze". | 670 return self.m.chromium_checkout.get_files_affected_by_patch( # pragma: no c
over |
699 | 671 relative_to, cwd) |
700 Paths are relative to `relative_to` which for analyze should be 'src/'. | |
701 """ | |
702 patch_root = self.m.gclient.calculate_patch_root( | |
703 self.m.properties.get('patch_project')) | |
704 if not cwd: | |
705 cwd = self._working_dir.join(patch_root) if self._working_dir else None | |
706 files = self.m.tryserver.get_files_affected_by_patch(patch_root, cwd=cwd) | |
707 for i, path in enumerate(files): | |
708 path = str(path) | |
709 assert path.startswith(relative_to) | |
710 files[i] = path[len(relative_to):] | |
711 return files | |
712 | 672 |
713 # TODO(phajdan.jr): fix callers and remove chromium_tests.configure_swarming. | 673 # TODO(phajdan.jr): fix callers and remove chromium_tests.configure_swarming. |
714 def configure_swarming(self, project_name, precommit, mastername=None): | 674 def configure_swarming(self, project_name, precommit, mastername=None): |
715 return self.m.chromium_swarming.configure_swarming( # pragma: no cover | 675 return self.m.chromium_swarming.configure_swarming( # pragma: no cover |
716 project_name, precommit, mastername) | 676 project_name, precommit, mastername) |
717 | 677 |
718 def _build_gs_archive_url(self, mastername, master_config, buildername): | 678 def _build_gs_archive_url(self, mastername, master_config, buildername): |
719 """Returns the archive URL to pass to self.m.archive.zip_and_upload_build. | 679 """Returns the archive URL to pass to self.m.archive.zip_and_upload_build. |
720 | 680 |
721 Most builders on most masters use a standard format for the build archive | 681 Most builders on most masters use a standard format for the build archive |
(...skipping 96 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
818 | 778 |
819 if not tests: | 779 if not tests: |
820 return | 780 return |
821 | 781 |
822 api.chromium_swarming.configure_swarming( | 782 api.chromium_swarming.configure_swarming( |
823 'chromium', precommit=False, mastername=mastername) | 783 'chromium', precommit=False, mastername=mastername) |
824 test_runner = api.chromium_tests.create_test_runner( | 784 test_runner = api.chromium_tests.create_test_runner( |
825 api, tests, serialize_tests=bot_config.get('serialize_tests')) | 785 api, tests, serialize_tests=bot_config.get('serialize_tests')) |
826 with api.chromium_tests.wrap_chromium_tests(bot_config, tests): | 786 with api.chromium_tests.wrap_chromium_tests(bot_config, tests): |
827 test_runner() | 787 test_runner() |
OLD | NEW |