Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(428)

Side by Side Diff: scripts/slave/recipe_modules/chromium_tests/api.py

Issue 2128613005: Archive Linux perf builds for manual bisect (Closed) Base URL: https://chromium.googlesource.com/chromium/tools/build.git@master
Patch Set: Code style change Created 4 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright 2014 The Chromium Authors. All rights reserved. 1 # Copyright 2014 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 import collections 5 import collections
6 import contextlib 6 import contextlib
7 import copy 7 import copy
8 import itertools 8 import itertools
9 import json 9 import json
10 10
(...skipping 194 matching lines...) Expand 10 before | Expand all | Expand 10 after
205 bot_db = bdb_module.BotConfigAndTestDB() 205 bot_db = bdb_module.BotConfigAndTestDB()
206 bot_db._add_master_dict_and_test_spec(mastername, master_dict, {}) 206 bot_db._add_master_dict_and_test_spec(mastername, master_dict, {})
207 return bot_db 207 return bot_db
208 208
209 def prepare_checkout(self, bot_config, root_solution_revision=None, 209 def prepare_checkout(self, bot_config, root_solution_revision=None,
210 force=False): 210 force=False):
211 update_step = self.ensure_checkout( 211 update_step = self.ensure_checkout(
212 bot_config, root_solution_revision, force=force) 212 bot_config, root_solution_revision, force=force)
213 213
214 if (self.m.chromium.c.compile_py.compiler and 214 if (self.m.chromium.c.compile_py.compiler and
215 'goma' in self.m.chromium.c.compile_py.compiler): 215 'goma' in self.m.chromium.c.compile_py.compiler):
216 self.m.chromium.ensure_goma(canary=bot_config.get('goma_canary', False)) 216 self.m.chromium.ensure_goma(canary=bot_config.get('goma_canary', False))
217 217
218 self.set_up_swarming(bot_config) 218 self.set_up_swarming(bot_config)
219 self.runhooks(update_step) 219 self.runhooks(update_step)
220 220
221 bot_db = bdb_module.BotConfigAndTestDB() 221 bot_db = bdb_module.BotConfigAndTestDB()
222 bot_config.initialize_bot_db(self, bot_db, update_step) 222 bot_config.initialize_bot_db(self, bot_db, update_step)
223 223
224 if self.m.chromium.c.lto and \ 224 if self.m.chromium.c.lto and \
225 not self.m.chromium.c.env.LLVM_FORCE_HEAD_REVISION: 225 not self.m.chromium.c.env.LLVM_FORCE_HEAD_REVISION:
(...skipping 173 matching lines...) Expand 10 before | Expand all | Expand 10 after
399 verbose=True, 399 verbose=True,
400 set_swarm_hashes=False) 400 set_swarm_hashes=False)
401 401
402 def archive_build(self, mastername, buildername, update_step, bot_db): 402 def archive_build(self, mastername, buildername, update_step, bot_db):
403 bot_config = bot_db.get_bot_config(mastername, buildername) 403 bot_config = bot_db.get_bot_config(mastername, buildername)
404 404
405 if bot_config.get('bot_type') == 'builder': 405 if bot_config.get('bot_type') == 'builder':
406 if not bot_config.get('cf_archive_build'): 406 if not bot_config.get('cf_archive_build'):
407 master_config = bot_db.get_master_settings(mastername) 407 master_config = bot_db.get_master_settings(mastername)
408 build_revision = update_step.presentation.properties['got_revision'] 408 build_revision = update_step.presentation.properties['got_revision']
409
410 # For archiving 'chromium.perf', the builder also archives a version
411 # without perf test files.
412 if mastername.startswith('chromium.perf'):
413 self.m.archive.zip_and_upload_build(
414 'package build bisect',
stgao 2016/07/12 20:40:40 "for bisect" instead?
miimnk 2016/07/13 01:12:48 Done.
415 self.m.chromium.c.build_config_fs,
416 build_url=self._build_gs_archive_url(
417 mastername, master_config, buildername, strip_symbol = True),
418 build_revision=build_revision,
419 cros_board=self.m.chromium.c.TARGET_CROS_BOARD,
420 update_properties=update_step.presentation.properties,
421 includePerfTestFiles=False,
422 store_by_hash=False
423 )
424
409 self.m.archive.zip_and_upload_build( 425 self.m.archive.zip_and_upload_build(
410 'package build', 426 'package build',
411 self.m.chromium.c.build_config_fs, 427 self.m.chromium.c.build_config_fs,
412 build_url=self._build_gs_archive_url( 428 build_url=self._build_gs_archive_url(
413 mastername, master_config, buildername), 429 mastername, master_config, buildername),
414 build_revision=build_revision, 430 build_revision=build_revision,
415 cros_board=self.m.chromium.c.TARGET_CROS_BOARD, 431 cros_board=self.m.chromium.c.TARGET_CROS_BOARD,
416 # TODO(machenbach): Make asan a configuration switch. 432 # TODO(machenbach): Make asan a configuration switch.
417 package_dsym_files=( 433 package_dsym_files=(
418 self.m.chromium.c.gyp_env.GYP_DEFINES.get('asan') and 434 self.m.chromium.c.gyp_env.GYP_DEFINES.get('asan') and
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after
467 isolated_targets=isolated_targets, 483 isolated_targets=isolated_targets,
468 name='generate_build_files%s' % name_suffix) 484 name='generate_build_files%s' % name_suffix)
469 485
470 self.m.chromium.compile(compile_targets, name='compile%s' % name_suffix) 486 self.m.chromium.compile(compile_targets, name='compile%s' % name_suffix)
471 487
472 def download_and_unzip_build(self, mastername, buildername, update_step, 488 def download_and_unzip_build(self, mastername, buildername, update_step,
473 bot_db, build_archive_url=None, 489 bot_db, build_archive_url=None,
474 build_revision=None, override_bot_type=None): 490 build_revision=None, override_bot_type=None):
475 assert isinstance(bot_db, bdb_module.BotConfigAndTestDB), \ 491 assert isinstance(bot_db, bdb_module.BotConfigAndTestDB), \
476 "bot_db argument %r was not a BotConfigAndTestDB" % (bot_db) 492 "bot_db argument %r was not a BotConfigAndTestDB" % (bot_db)
493
477 # We only want to do this for tester bots (i.e. those which do not compile 494 # We only want to do this for tester bots (i.e. those which do not compile
478 # locally). 495 # locally).
479 bot_type = override_bot_type or bot_db.get_bot_config( 496 bot_type = override_bot_type or bot_db.get_bot_config(
480 mastername, buildername).get('bot_type') 497 mastername, buildername).get('bot_type')
481 if bot_type != 'tester': 498 if bot_type != 'tester':
482 return 499 return
483 500
484 # Protect against hard to debug mismatches between directory names 501 # Protect against hard to debug mismatches between directory names
485 # used to run tests from and extract build to. We've had several cases 502 # used to run tests from and extract build to. We've had several cases
486 # where a stale build directory was used on a tester, and the extracted 503 # where a stale build directory was used on a tester, and the extracted
(...skipping 268 matching lines...) Expand 10 before | Expand all | Expand 10 after
755 self.m.swarming.default_user = requester 772 self.m.swarming.default_user = requester
756 773
757 patch_project = self.m.properties.get('patch_project') 774 patch_project = self.m.properties.get('patch_project')
758 if patch_project: 775 if patch_project:
759 self.m.swarming.add_default_tag('patch_project:%s' % patch_project) 776 self.m.swarming.add_default_tag('patch_project:%s' % patch_project)
760 else: 777 else:
761 self.m.swarming.default_priority = MASTER_SWARMING_PRIORITIES[mastername] 778 self.m.swarming.default_priority = MASTER_SWARMING_PRIORITIES[mastername]
762 self.m.swarming.add_default_tag('purpose:post-commit') 779 self.m.swarming.add_default_tag('purpose:post-commit')
763 self.m.swarming.add_default_tag('purpose:CI') 780 self.m.swarming.add_default_tag('purpose:CI')
764 781
765 def _build_gs_archive_url(self, mastername, master_config, buildername): 782 def _build_gs_archive_url(self, mastername, master_config, buildername, strip_ symbol = False):
766 """Returns the archive URL to pass to self.m.archive.zip_and_upload_build. 783 """Returns the archive URL to pass to self.m.archive.zip_and_upload_build.
767 784
768 Most builders on most masters use a standard format for the build archive 785 Most builders on most masters use a standard format for the build archive
769 URL, but some builders on some masters may specify custom places to upload 786 URL, but some builders on some masters may specify custom places to upload
770 builds to. These special cases include: 787 builds to. These special cases include:
771 'chromium.perf' or 'chromium.perf.fyi': 788 'chromium.perf' or 'chromium.perf.fyi':
772 Exclude the name of the master from the url. 789 Exclude the name of the master from the url.
773 'tryserver.chromium.perf', or 790 'tryserver.chromium.perf', or
774 linux_full_bisect_builder on 'tryserver.chromium.linux': 791 linux_full_bisect_builder on 'tryserver.chromium.linux':
775 Return None so that the archive url specified in factory_properties 792 Return None so that the archive url specified in factory_properties
776 (as set on the master's configuration) is used instead. 793 (as set on the master's configuration) is used instead.
777 """ 794 """
778 if mastername.startswith('chromium.perf'): 795 if mastername.startswith('chromium.perf'):
796 if strip_symbol:
797 return self.m.archive.legacy_upload_url(
798 master_config.get('bisect_build_gs_bucket'),
799 extra_url_components=None)
779 return self.m.archive.legacy_upload_url( 800 return self.m.archive.legacy_upload_url(
780 master_config.get('build_gs_bucket'), 801 master_config.get('build_gs_bucket'),
781 extra_url_components=None) 802 extra_url_components=None)
782 elif (mastername == 'tryserver.chromium.perf' or 803 elif (mastername == 'tryserver.chromium.perf' or
783 (mastername == 'tryserver.chromium.linux' and 804 (mastername == 'tryserver.chromium.linux' and
784 buildername == 'linux_full_bisect_builder')): 805 buildername == 'linux_full_bisect_builder')):
785 return None 806 return None
786 else: 807 else:
787 return self.m.archive.legacy_upload_url( 808 return self.m.archive.legacy_upload_url(
788 master_config.get('build_gs_bucket'), 809 master_config.get('build_gs_bucket'),
(...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after
829 def get_compile_targets_for_scripts(self): 850 def get_compile_targets_for_scripts(self):
830 return self.m.python( 851 return self.m.python(
831 name='get compile targets for scripts', 852 name='get compile targets for scripts',
832 script=self.m.path['checkout'].join( 853 script=self.m.path['checkout'].join(
833 'testing', 'scripts', 'get_compile_targets.py'), 854 'testing', 'scripts', 'get_compile_targets.py'),
834 args=[ 855 args=[
835 '--output', self.m.json.output(), 856 '--output', self.m.json.output(),
836 '--', 857 '--',
837 ] + self.get_common_args_for_scripts(), 858 ] + self.get_common_args_for_scripts(),
838 step_test_data=lambda: self.m.json.test_api.output({})) 859 step_test_data=lambda: self.m.json.test_api.output({}))
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698