Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(113)

Side by Side Diff: infra/bots/recipes/swarm_trigger.py

Issue 2215803002: Move builder_spec, [dm|nanobench]_flags, builder_name_schema to recipes (Closed) Base URL: https://skia.googlesource.com/skia.git@master
Patch Set: Add missing blacklist Created 4 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright 2016 The Chromium Authors. All rights reserved. 1 # Copyright 2016 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 5
6 # Recipe module for Skia Swarming trigger. 6 # Recipe module for Skia Swarming trigger.
7 7
8 8
9 import json 9 import json
10 10
11 11
12 DEPS = [ 12 DEPS = [
13 'core',
14 'build/file', 13 'build/file',
15 'build/gsutil', 14 'build/gsutil',
15 'builder_name_schema',
16 'core',
16 'depot_tools/depot_tools', 17 'depot_tools/depot_tools',
17 'depot_tools/git', 18 'depot_tools/git',
18 'depot_tools/tryserver', 19 'depot_tools/tryserver',
19 'recipe_engine/json', 20 'recipe_engine/json',
20 'recipe_engine/path', 21 'recipe_engine/path',
21 'recipe_engine/properties', 22 'recipe_engine/properties',
22 'recipe_engine/python', 23 'recipe_engine/python',
23 'recipe_engine/raw_io', 24 'recipe_engine/raw_io',
24 'recipe_engine/step', 25 'recipe_engine/step',
25 'recipe_engine/time', 26 'recipe_engine/time',
(...skipping 26 matching lines...) Expand all
52 'Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release', 53 'Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release',
53 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug', 54 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug',
54 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-MSAN', 55 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-MSAN',
55 'Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release', 56 'Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release',
56 'Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release', 57 'Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release',
57 ], 58 ],
58 }, 59 },
59 } 60 }
60 61
61 62
62 def derive_compile_bot_name(builder_name, builder_spec): 63 def derive_compile_bot_name(api):
63 builder_cfg = builder_spec['builder_cfg'] 64 builder_name = api.properties['buildername']
65 builder_cfg = api.builder_name_schema.DictForBuilderName(builder_name)
64 if builder_cfg['role'] == 'Housekeeper': 66 if builder_cfg['role'] == 'Housekeeper':
65 return 'Build-Ubuntu-GCC-x86_64-Release-Shared' 67 return 'Build-Ubuntu-GCC-x86_64-Release-Shared'
66 if builder_cfg['role'] in ('Test', 'Perf'): 68 if builder_cfg['role'] in ('Test', 'Perf'):
67 os = builder_cfg['os'] 69 os = builder_cfg['os']
68 extra_config = builder_cfg.get('extra_config') 70 extra_config = builder_cfg.get('extra_config')
69 if os == 'Android': 71 if os == 'Android':
70 if extra_config == 'Vulkan': 72 if extra_config == 'Vulkan':
71 extra_config = '%s_%s' % (os, 'Vulkan') 73 extra_config = '%s_%s' % (os, 'Vulkan')
72 else: 74 else:
73 extra_config = os 75 extra_config = os
74 os = 'Ubuntu' 76 os = 'Ubuntu'
75 elif os == 'iOS': 77 elif os == 'iOS':
76 extra_config = os 78 extra_config = os
77 os = 'Mac' 79 os = 'Mac'
78 elif 'Win' in os: 80 elif 'Win' in os:
79 os = 'Win' 81 os = 'Win'
80 builder_name = 'Build-%s-%s-%s-%s' % ( 82 return api.builder_name_schema.MakeBuilderName(
81 os, 83 role=api.builder_name_schema.BUILDER_ROLE_BUILD,
82 builder_cfg['compiler'], 84 os=os,
83 builder_cfg['arch'], 85 compiler=builder_cfg['compiler'],
84 builder_cfg['configuration'] 86 target_arch=builder_cfg['arch'],
85 ) 87 configuration=builder_cfg['configuration'],
86 if extra_config: 88 extra_config=extra_config,
87 builder_name += '-%s' % extra_config 89 is_trybot=api.builder_name_schema.IsTrybot(builder_name))
88 if builder_cfg['is_trybot']:
89 builder_name += '-Trybot'
90 return builder_name 90 return builder_name
91 91
92 92
93 def swarm_dimensions(builder_spec): 93 def swarm_dimensions(builder_spec):
94 """Return a dict of keys and values to be used as Swarming bot dimensions.""" 94 """Return a dict of keys and values to be used as Swarming bot dimensions."""
95 dimensions = { 95 dimensions = {
96 'pool': 'Skia', 96 'pool': 'Skia',
97 } 97 }
98 builder_cfg = builder_spec['builder_cfg'] 98 builder_cfg = builder_spec['builder_cfg']
99 dimensions['os'] = builder_cfg.get('os', 'Ubuntu') 99 dimensions['os'] = builder_cfg.get('os', 'Ubuntu')
(...skipping 173 matching lines...) Expand 10 before | Expand all | Expand 10 after
273 store_output=False, 273 store_output=False,
274 extra_isolate_hashes=extra_isolate_hashes) 274 extra_isolate_hashes=extra_isolate_hashes)
275 return api.swarming.collect_swarming_task(task) 275 return api.swarming.collect_swarming_task(task)
276 276
277 277
278 def infra_swarm(api, got_revision, infrabots_dir, extra_isolate_hashes): 278 def infra_swarm(api, got_revision, infrabots_dir, extra_isolate_hashes):
279 # Fake the builder spec. 279 # Fake the builder spec.
280 builder_spec = { 280 builder_spec = {
281 'builder_cfg': { 281 'builder_cfg': {
282 'role': 'Infra', 282 'role': 'Infra',
283 'is_trybot': api.properties['buildername'].endswith('-Trybot'), 283 'is_trybot': api.builder_name_schema.IsTrybot(
284 api.properties['buildername'])
284 } 285 }
285 } 286 }
286 task = trigger_task( 287 task = trigger_task(
287 api, 288 api,
288 'infra', 289 'infra',
289 api.properties['buildername'], 290 api.properties['buildername'],
290 api.properties['mastername'], 291 api.properties['mastername'],
291 api.properties['slavename'], 292 api.properties['slavename'],
292 api.properties['buildnumber'], 293 api.properties['buildnumber'],
293 builder_spec, 294 builder_spec,
294 got_revision, 295 got_revision,
295 infrabots_dir, 296 infrabots_dir,
296 idempotent=False, 297 idempotent=False,
297 store_output=False, 298 store_output=False,
298 extra_isolate_hashes=extra_isolate_hashes) 299 extra_isolate_hashes=extra_isolate_hashes)
299 return api.swarming.collect_swarming_task(task) 300 return api.swarming.collect_swarming_task(task)
300 301
301 302
302 def compile_steps_swarm(api, builder_spec, got_revision, infrabots_dir, 303 def compile_steps_swarm(api, builder_spec, got_revision, infrabots_dir,
303 extra_isolate_hashes, cipd_packages): 304 extra_isolate_hashes, cipd_packages):
304 builder_name = derive_compile_bot_name(api.properties['buildername'], 305 builder_name = derive_compile_bot_name(api)
305 builder_spec)
306 compile_builder_spec = builder_spec 306 compile_builder_spec = builder_spec
307 if builder_name != api.properties['buildername']: 307 if builder_name != api.properties['buildername']:
308 compile_builder_spec = api.core.get_builder_spec( 308 compile_builder_spec = api.vars.get_builder_spec(builder_name)
309 api.path['slave_build'].join('skia'), builder_name)
310 309
311 extra_hashes = extra_isolate_hashes[:] 310 extra_hashes = extra_isolate_hashes[:]
312 311
313 # Windows bots require a toolchain. 312 # Windows bots require a toolchain.
314 if 'Win' in builder_name: 313 if 'Win' in builder_name:
315 version_file = infrabots_dir.join('assets', 'win_toolchain', 'VERSION') 314 version_file = infrabots_dir.join('assets', 'win_toolchain', 'VERSION')
316 version = api.run.readfile(version_file, 315 version = api.run.readfile(version_file,
317 name='read win_toolchain VERSION', 316 name='read win_toolchain VERSION',
318 test_data='0').rstrip() 317 test_data='0').rstrip()
319 version = 'version:%s' % version 318 version = 'version:%s' % version
320 pkg = ('t', 'skia/bots/win_toolchain', version) 319 pkg = ('t', 'skia/bots/win_toolchain', version)
321 cipd_packages.append(pkg) 320 cipd_packages.append(pkg)
322 321
323 if 'Vulkan' in builder_name: 322 if 'Vulkan' in builder_name:
324 cipd_packages.append(cipd_pkg(api, infrabots_dir, 'win_vulkan_sdk')) 323 cipd_packages.append(cipd_pkg(api, infrabots_dir, 'win_vulkan_sdk'))
325 324
326 # Fake these properties for compile tasks so that they can be de-duped. 325 # Fake these properties for compile tasks so that they can be de-duped.
327 master = 'client.skia.compile' 326 master = 'client.skia.compile'
328 slave = 'skiabot-dummy-compile-slave' 327 slave = 'skiabot-dummy-compile-slave'
(...skipping 267 matching lines...) Expand 10 before | Expand all | Expand 10 after
596 extra_hashes = [] 595 extra_hashes = []
597 596
598 # Get ready to compile. 597 # Get ready to compile.
599 compile_cipd_deps = [] 598 compile_cipd_deps = []
600 extra_compile_hashes = [] 599 extra_compile_hashes = []
601 600
602 infrabots_dir = api.path['checkout'].join('infra', 'bots') 601 infrabots_dir = api.path['checkout'].join('infra', 'bots')
603 if 'Infra' in api.properties['buildername']: 602 if 'Infra' in api.properties['buildername']:
604 return infra_swarm(api, got_revision, infrabots_dir, extra_hashes) 603 return infra_swarm(api, got_revision, infrabots_dir, extra_hashes)
605 604
606 builder_spec = api.core.get_builder_spec(api.path['checkout'], 605 builder_spec = api.vars.get_builder_spec(api.properties['buildername'])
607 api.properties['buildername'])
608 builder_cfg = builder_spec['builder_cfg'] 606 builder_cfg = builder_spec['builder_cfg']
609 607
610 if 'RecreateSKPs' in api.properties['buildername']: 608 if 'RecreateSKPs' in api.properties['buildername']:
611 recreate_skps_swarm(api, builder_spec, got_revision, infrabots_dir, 609 recreate_skps_swarm(api, builder_spec, got_revision, infrabots_dir,
612 extra_hashes) 610 extra_hashes)
613 return 611 return
614 if 'Android' in api.properties['buildername']: 612 if 'Android' in api.properties['buildername']:
615 compile_cipd_deps.append(cipd_pkg(api, infrabots_dir, 'android_sdk')) 613 compile_cipd_deps.append(cipd_pkg(api, infrabots_dir, 'android_sdk'))
616 614
617 # Compile. 615 # Compile.
(...skipping 89 matching lines...) Expand 10 before | Expand all | Expand 10 after
707 test += api.path.exists(*paths) 705 test += api.path.exists(*paths)
708 706
709 return test 707 return test
710 708
711 709
712 def GenTests(api): 710 def GenTests(api):
713 for mastername, slaves in TEST_BUILDERS.iteritems(): 711 for mastername, slaves in TEST_BUILDERS.iteritems():
714 for slavename, builders_by_slave in slaves.iteritems(): 712 for slavename, builders_by_slave in slaves.iteritems():
715 for builder in builders_by_slave: 713 for builder in builders_by_slave:
716 yield test_for_bot(api, builder, mastername, slavename) 714 yield test_for_bot(api, builder, mastername, slavename)
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698