| OLD | NEW | 
|---|
| 1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. | 
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be | 
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. | 
| 4 | 4 | 
| 5 import contextlib | 5 import contextlib | 
| 6 import copy | 6 import copy | 
| 7 import itertools | 7 import itertools | 
| 8 import json | 8 import json | 
| 9 | 9 | 
| 10 from recipe_engine.types import freeze | 10 from recipe_engine.types import freeze | 
| (...skipping 12 matching lines...) Expand all  Loading... | 
| 23 ] | 23 ] | 
| 24 | 24 | 
| 25 | 25 | 
| 26 class ChromiumTestsApi(recipe_api.RecipeApi): | 26 class ChromiumTestsApi(recipe_api.RecipeApi): | 
| 27   def __init__(self, *args, **kwargs): | 27   def __init__(self, *args, **kwargs): | 
| 28     super(ChromiumTestsApi, self).__init__(*args, **kwargs) | 28     super(ChromiumTestsApi, self).__init__(*args, **kwargs) | 
| 29     self._builders = {} | 29     self._builders = {} | 
| 30     self.add_builders(builders.BUILDERS) | 30     self.add_builders(builders.BUILDERS) | 
| 31     self._precommit_mode = False | 31     self._precommit_mode = False | 
| 32 | 32 | 
| 33     # Keep track of working directory (which contains the checkout). |  | 
| 34     # None means "default value". |  | 
| 35     self._working_dir = None |  | 
| 36 |  | 
| 37   @property | 33   @property | 
| 38   def builders(self): | 34   def builders(self): | 
| 39     return self._builders | 35     return self._builders | 
| 40 | 36 | 
| 41   @property | 37   @property | 
| 42   def steps(self): | 38   def steps(self): | 
| 43     return steps | 39     return steps | 
| 44 | 40 | 
| 45   @property | 41   @property | 
| 46   def trybots(self): | 42   def trybots(self): | 
| (...skipping 76 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
| 123       # (3) 'HEAD' for forced builds with unspecified 'revision'. | 119       # (3) 'HEAD' for forced builds with unspecified 'revision'. | 
| 124       # TODO(machenbach): Use parent_got_cr_revision on testers with component | 120       # TODO(machenbach): Use parent_got_cr_revision on testers with component | 
| 125       # builds to match also the chromium revision from the builder. | 121       # builds to match also the chromium revision from the builder. | 
| 126       component_rev = self.m.properties.get('revision') or 'HEAD' | 122       component_rev = self.m.properties.get('revision') or 'HEAD' | 
| 127       if bot_type == 'tester': | 123       if bot_type == 'tester': | 
| 128         component_rev = self.m.properties.get( | 124         component_rev = self.m.properties.get( | 
| 129             'parent_got_revision', component_rev) | 125             'parent_got_revision', component_rev) | 
| 130       dep = bot_config.get('set_component_rev') | 126       dep = bot_config.get('set_component_rev') | 
| 131       self.m.gclient.c.revisions[dep['name']] = dep['rev_str'] % component_rev | 127       self.m.gclient.c.revisions[dep['name']] = dep['rev_str'] % component_rev | 
| 132 | 128 | 
| 133   def get_checkout_dir(self, bot_config): | 129   # TODO(phajdan.jr): fix callers and remove chromium_tests.get_checkout_dir. | 
| 134     try: | 130   def get_checkout_dir(self, bot_config):  # pragma: no cover | 
| 135       builder_cache = self.m.path['builder_cache'] | 131     return self.m.chromium_checkout.get_checkout_dir(bot_config) | 
| 136     except KeyError:  # no-op if builder cache is not set up. |  | 
| 137       return None |  | 
| 138     else: |  | 
| 139       sanitized_buildername = ''.join( |  | 
| 140           c if c.isalnum() else '_' for c in self.m.properties['buildername']) |  | 
| 141       checkout_dir = builder_cache.join( |  | 
| 142           bot_config.get('checkout_dir', sanitized_buildername)) |  | 
| 143       self.m.shutil.makedirs('checkout path', checkout_dir) |  | 
| 144       return checkout_dir |  | 
| 145 | 132 | 
|  | 133   # TODO(phajdan.jr): fix callers and remove chromium_tests.ensure_checkout. | 
| 146   def ensure_checkout(self, bot_config, root_solution_revision=None, | 134   def ensure_checkout(self, bot_config, root_solution_revision=None, | 
| 147                       force=False): | 135                       force=False): | 
| 148     if self.m.platform.is_win: | 136     return self.m.chromium_checkout.ensure_checkout(  # pragma: no cover | 
| 149       self.m.chromium.taskkill() | 137         bot_config, root_solution_revision, force) | 
| 150 |  | 
| 151     kwargs = {} |  | 
| 152     self._working_dir = self.get_checkout_dir(bot_config) |  | 
| 153     if self._working_dir: |  | 
| 154       kwargs['cwd'] = self._working_dir |  | 
| 155 |  | 
| 156     # Bot Update re-uses the gclient configs. |  | 
| 157     update_step = self.m.bot_update.ensure_checkout( |  | 
| 158         patch_root=bot_config.get('patch_root'), |  | 
| 159         root_solution_revision=root_solution_revision, |  | 
| 160         clobber=bot_config.get('clobber', False), |  | 
| 161         force=force, **kwargs) |  | 
| 162     assert update_step.json.output['did_run'] |  | 
| 163     # HACK(dnj): Remove after 'crbug.com/398105' has landed |  | 
| 164     self.m.chromium.set_build_properties(update_step.json.output['properties']) |  | 
| 165 |  | 
| 166     return update_step |  | 
| 167 | 138 | 
| 168   def set_up_swarming(self, bot_config): | 139   def set_up_swarming(self, bot_config): | 
| 169     if not bot_config.get('enable_swarming'): | 140     if not bot_config.get('enable_swarming'): | 
| 170       return | 141       return | 
| 171     self.m.isolate.set_isolate_environment(self.m.chromium.c) | 142     self.m.isolate.set_isolate_environment(self.m.chromium.c) | 
| 172     self.m.swarming.check_client_version() | 143     self.m.swarming.check_client_version() | 
| 173     for key, value in bot_config.get('swarming_dimensions', {}).iteritems(): | 144     for key, value in bot_config.get('swarming_dimensions', {}).iteritems(): | 
| 174       self.m.swarming.set_default_dimension(key, value) | 145       self.m.swarming.set_default_dimension(key, value) | 
| 175 | 146 | 
| 176   def runhooks(self, update_step): | 147   def runhooks(self, update_step): | 
| (...skipping 10 matching lines...) Expand all  Loading... | 
| 187   # TODO(phajdan.jr): remove create_bot_db_from_master_dict. It adds another | 158   # TODO(phajdan.jr): remove create_bot_db_from_master_dict. It adds another | 
| 188   # entry point to _add_master_dict_and_test_spec which can really complicate | 159   # entry point to _add_master_dict_and_test_spec which can really complicate | 
| 189   # things. | 160   # things. | 
| 190   def create_bot_db_from_master_dict(self, mastername, master_dict): | 161   def create_bot_db_from_master_dict(self, mastername, master_dict): | 
| 191     bot_db = bdb_module.BotConfigAndTestDB() | 162     bot_db = bdb_module.BotConfigAndTestDB() | 
| 192     bot_db._add_master_dict_and_test_spec(mastername, master_dict, {}) | 163     bot_db._add_master_dict_and_test_spec(mastername, master_dict, {}) | 
| 193     return bot_db | 164     return bot_db | 
| 194 | 165 | 
| 195   def prepare_checkout(self, bot_config, root_solution_revision=None, | 166   def prepare_checkout(self, bot_config, root_solution_revision=None, | 
| 196                        force=False): | 167                        force=False): | 
| 197     update_step = self.ensure_checkout( | 168     update_step = self.m.chromium_checkout.ensure_checkout( | 
| 198         bot_config, root_solution_revision, force=force) | 169         bot_config, root_solution_revision, force=force) | 
| 199 | 170 | 
| 200     if (self.m.chromium.c.compile_py.compiler and | 171     if (self.m.chromium.c.compile_py.compiler and | 
| 201         'goma' in self.m.chromium.c.compile_py.compiler): | 172         'goma' in self.m.chromium.c.compile_py.compiler): | 
| 202       self.m.chromium.ensure_goma(canary=bot_config.get('goma_canary', False)) | 173       self.m.chromium.ensure_goma(canary=bot_config.get('goma_canary', False)) | 
| 203 | 174 | 
| 204     self.set_up_swarming(bot_config) | 175     self.set_up_swarming(bot_config) | 
| 205     self.runhooks(update_step) | 176     self.runhooks(update_step) | 
| 206 | 177 | 
| 207     bot_db = bdb_module.BotConfigAndTestDB() | 178     bot_db = bdb_module.BotConfigAndTestDB() | 
| (...skipping 329 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
| 537   def _make_legacy_build_url(self, master_config, mastername): | 508   def _make_legacy_build_url(self, master_config, mastername): | 
| 538     return self.m.archive.legacy_download_url( | 509     return self.m.archive.legacy_download_url( | 
| 539                master_config.get('build_gs_bucket'), | 510                master_config.get('build_gs_bucket'), | 
| 540                extra_url_components=( | 511                extra_url_components=( | 
| 541                    None if mastername.startswith('chromium.perf') | 512                    None if mastername.startswith('chromium.perf') | 
| 542                    else self.m.properties['mastername'])) | 513                    else self.m.properties['mastername'])) | 
| 543 | 514 | 
| 544   @contextlib.contextmanager | 515   @contextlib.contextmanager | 
| 545   def wrap_chromium_tests(self, bot_config, tests=None): | 516   def wrap_chromium_tests(self, bot_config, tests=None): | 
| 546     context = { | 517     context = { | 
| 547         'cwd': self._working_dir, | 518         'cwd': self.m.chromium_checkout.working_dir, | 
| 548         'env': self.m.chromium.get_env(), | 519         'env': self.m.chromium.get_env(), | 
| 549     } | 520     } | 
| 550     with self.m.step.context(context): | 521     with self.m.step.context(context): | 
| 551       bot_type = bot_config.get('bot_type', 'builder_tester') | 522       bot_type = bot_config.get('bot_type', 'builder_tester') | 
| 552 | 523 | 
| 553       if bot_type in ('tester', 'builder_tester'): | 524       if bot_type in ('tester', 'builder_tester'): | 
| 554         isolated_targets = [ | 525         isolated_targets = [ | 
| 555             t.isolate_target(self.m) for t in tests if t.uses_swarming] | 526             t.isolate_target(self.m) for t in tests if t.uses_swarming] | 
| 556         if isolated_targets: | 527         if isolated_targets: | 
| 557           self.m.isolate.find_isolated_tests(self.m.chromium.output_dir) | 528           self.m.isolate.find_isolated_tests(self.m.chromium.output_dir) | 
| (...skipping 81 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
| 639     # is also imporant that we do not assume that corresponding revision is | 610     # is also imporant that we do not assume that corresponding revision is | 
| 640     # stored in the 'got_revision' as some gclient configs change the default | 611     # stored in the 'got_revision' as some gclient configs change the default | 
| 641     # mapping for their own purposes. | 612     # mapping for their own purposes. | 
| 642     first_solution_name = self.m.gclient.c.solutions[0].name | 613     first_solution_name = self.m.gclient.c.solutions[0].name | 
| 643     rev_property = self.m.gclient.c.got_revision_mapping[first_solution_name] | 614     rev_property = self.m.gclient.c.got_revision_mapping[first_solution_name] | 
| 644     self.m.gclient.c.revisions[first_solution_name] = str( | 615     self.m.gclient.c.revisions[first_solution_name] = str( | 
| 645         bot_update_json['properties'][rev_property]) | 616         bot_update_json['properties'][rev_property]) | 
| 646     self._resolve_fixed_revisions(bot_update_json) | 617     self._resolve_fixed_revisions(bot_update_json) | 
| 647 | 618 | 
| 648     kwargs = {} | 619     kwargs = {} | 
| 649     if self._working_dir: | 620     if self.m.chromium_checkout.working_dir: | 
| 650       kwargs['cwd'] = self._working_dir | 621       kwargs['cwd'] = self.m.chromium_checkout.working_dir | 
| 651 | 622 | 
| 652     self.m.bot_update.ensure_checkout( | 623     self.m.bot_update.ensure_checkout( | 
| 653         force=True, patch=False, update_presentation=False, **kwargs) | 624         force=True, patch=False, update_presentation=False, **kwargs) | 
| 654     self.m.chromium.runhooks(name='runhooks (without patch)') | 625     self.m.chromium.runhooks(name='runhooks (without patch)') | 
| 655 | 626 | 
| 656   def run_tests_on_tryserver(self, bot_config, api, tests, bot_update_step, | 627   def run_tests_on_tryserver(self, bot_config, api, tests, bot_update_step, | 
| 657                              affected_files, mb_mastername=None, | 628                              affected_files, mb_mastername=None, | 
| 658                              mb_buildername=None): | 629                              mb_buildername=None): | 
| 659     def deapply_patch_fn(failing_tests): | 630     def deapply_patch_fn(failing_tests): | 
| 660       self.deapply_patch(bot_update_step) | 631       self.deapply_patch(bot_update_step) | 
| (...skipping 26 matching lines...) Expand all  Loading... | 
| 687       if deapply_patch: | 658       if deapply_patch: | 
| 688         self.m.test_utils.determine_new_failures(api, tests, deapply_patch_fn) | 659         self.m.test_utils.determine_new_failures(api, tests, deapply_patch_fn) | 
| 689       else: | 660       else: | 
| 690         failing_tests = self.m.test_utils.run_tests_with_patch(api, tests) | 661         failing_tests = self.m.test_utils.run_tests_with_patch(api, tests) | 
| 691         if failing_tests: | 662         if failing_tests: | 
| 692           self.m.python.failing_step( | 663           self.m.python.failing_step( | 
| 693               'test results', | 664               'test results', | 
| 694               'TESTS FAILED; retries without patch disabled (%s)' | 665               'TESTS FAILED; retries without patch disabled (%s)' | 
| 695                   % deapply_patch_reason) | 666                   % deapply_patch_reason) | 
| 696 | 667 | 
|  | 668   # TODO(phajdan.jr): fix callers and remove get_files_affected_by_patch. | 
| 697   def get_files_affected_by_patch(self, relative_to='src/', cwd=None): | 669   def get_files_affected_by_patch(self, relative_to='src/', cwd=None): | 
| 698     """Returns list of POSIX paths of files affected by patch for "analyze". | 670     return self.m.chromium_checkout.get_files_affected_by_patch(  # pragma: no c
     over | 
| 699 | 671         relative_to, cwd) | 
| 700     Paths are relative to `relative_to` which for analyze should be 'src/'. |  | 
| 701     """ |  | 
| 702     patch_root = self.m.gclient.calculate_patch_root( |  | 
| 703         self.m.properties.get('patch_project')) |  | 
| 704     if not cwd: |  | 
| 705       cwd = self._working_dir.join(patch_root) if self._working_dir else None |  | 
| 706     files = self.m.tryserver.get_files_affected_by_patch(patch_root, cwd=cwd) |  | 
| 707     for i, path in enumerate(files): |  | 
| 708       path = str(path) |  | 
| 709       assert path.startswith(relative_to) |  | 
| 710       files[i] = path[len(relative_to):] |  | 
| 711     return files |  | 
| 712 | 672 | 
| 713   # TODO(phajdan.jr): fix callers and remove chromium_tests.configure_swarming. | 673   # TODO(phajdan.jr): fix callers and remove chromium_tests.configure_swarming. | 
| 714   def configure_swarming(self, project_name, precommit, mastername=None): | 674   def configure_swarming(self, project_name, precommit, mastername=None): | 
| 715     return self.m.chromium_swarming.configure_swarming(  # pragma: no cover | 675     return self.m.chromium_swarming.configure_swarming(  # pragma: no cover | 
| 716         project_name, precommit, mastername) | 676         project_name, precommit, mastername) | 
| 717 | 677 | 
| 718   def _build_gs_archive_url(self, mastername, master_config, buildername): | 678   def _build_gs_archive_url(self, mastername, master_config, buildername): | 
| 719     """Returns the archive URL to pass to self.m.archive.zip_and_upload_build. | 679     """Returns the archive URL to pass to self.m.archive.zip_and_upload_build. | 
| 720 | 680 | 
| 721     Most builders on most masters use a standard format for the build archive | 681     Most builders on most masters use a standard format for the build archive | 
| (...skipping 96 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
| 818 | 778 | 
| 819     if not tests: | 779     if not tests: | 
| 820       return | 780       return | 
| 821 | 781 | 
| 822     api.chromium_swarming.configure_swarming( | 782     api.chromium_swarming.configure_swarming( | 
| 823         'chromium', precommit=False, mastername=mastername) | 783         'chromium', precommit=False, mastername=mastername) | 
| 824     test_runner = api.chromium_tests.create_test_runner( | 784     test_runner = api.chromium_tests.create_test_runner( | 
| 825         api, tests, serialize_tests=bot_config.get('serialize_tests')) | 785         api, tests, serialize_tests=bot_config.get('serialize_tests')) | 
| 826     with api.chromium_tests.wrap_chromium_tests(bot_config, tests): | 786     with api.chromium_tests.wrap_chromium_tests(bot_config, tests): | 
| 827       test_runner() | 787       test_runner() | 
| OLD | NEW | 
|---|