Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(93)

Issues Search
    My Issues | Starred     Open | Closed | All

Unified Diff: infra/bots/recipes/swarm_ct_skps.py

Issue 2221413002: Move ct_skps recipe from tools repo to Skia repo (Closed) Base URL: https://skia.googlesource.com/skia@master
Patch Set: Address comments Created 4 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « infra/bots/recipes/swarm_compile.expected/win_retry_failed_compile.json ('k') | infra/bots/recipes/swarm_ct_skps.expected/CT_10k_SKPs_UnknownBuilder.json » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: infra/bots/recipes/swarm_ct_skps.py
diff --git a/infra/bots/recipes/swarm_ct_skps.py b/infra/bots/recipes/swarm_ct_skps.py
new file mode 100644
index 0000000000000000000000000000000000000000..2a18be9210bc8fe83fc874071be7b675743bde72
--- /dev/null
+++ b/infra/bots/recipes/swarm_ct_skps.py
@@ -0,0 +1,470 @@
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+import math
+
+
+DEPS = [
+ 'build/file',
+ 'build/gsutil',
+ 'core',
+ 'ct',
+ 'flavor',
+ 'recipe_engine/json',
+ 'recipe_engine/path',
+ 'recipe_engine/properties',
+ 'recipe_engine/step',
+ 'recipe_engine/time',
+ 'run',
+ 'swarming',
+ 'vars',
+]
+
+
+SKPS_VERSION_FILE = 'skps_version'
+CT_SKPS_ISOLATE = 'ct_skps.isolate'
+
+# Do not batch archive more slaves than this value. This is used to prevent
+# no output timeouts in the 'isolate tests' step.
+MAX_SLAVES_TO_BATCHARCHIVE = 100
+
+TOOL_TO_DEFAULT_SKPS_PER_SLAVE = {
+ 'dm': 10000,
+ 'nanobench': 1000,
+ 'get_images_from_skps': 10000,
+}
+
+# The SKP repository to use.
+DEFAULT_SKPS_CHROMIUM_BUILD = 'fad657e-276e633'
+
+
+def RunSteps(api):
+ # Figure out which repository to use.
+ buildername = api.properties['buildername']
+ if '1k' in buildername:
+ ct_page_type = '10k'
+ num_pages = 1000
+ elif '10k' in buildername:
+ ct_page_type = '10k'
+ num_pages = 10000
+ elif '100k' in buildername:
+ ct_page_type = '100k'
+ num_pages = 100000
+ elif '1m' in buildername:
+ ct_page_type = 'All'
+ num_pages = 1000000
+ else:
+ raise Exception('Do not recognise the buildername %s.' % buildername)
+
+ # Figure out which tool to use.
+ if 'DM' in buildername:
+ skia_tool = 'dm'
+ build_target = 'dm'
+ elif 'BENCH' in buildername:
+ skia_tool = 'nanobench'
+ build_target = 'nanobench'
+ elif 'IMG_DECODE' in buildername:
+ skia_tool = 'get_images_from_skps'
+ build_target = 'tools'
+ else:
+ raise Exception('Do not recognise the buildername %s.' % buildername)
+
+ api.core.setup()
+ api.flavor.compile(build_target)
+
+ # Required paths.
+ infrabots_dir = api.vars.skia_dir.join('infra', 'bots')
+ isolate_dir = infrabots_dir.join('ct')
+ isolate_path = isolate_dir.join(CT_SKPS_ISOLATE)
+
+ api.run.copy_build_products(
+ api.flavor.out_dir,
+ isolate_dir)
+ api.swarming.setup(
+ infrabots_dir.join('tools', 'luci-go'),
+ swarming_rev='')
+
+ skps_chromium_build = api.properties.get(
+ 'skps_chromium_build', DEFAULT_SKPS_CHROMIUM_BUILD)
+
+ # Set build property to make finding SKPs convenient.
+ api.step.active_result.presentation.properties['Location of SKPs'] = (
+ 'https://pantheon.corp.google.com/storage/browser/%s/swarming/skps/%s/%s/'
+ % (api.ct.CT_GS_BUCKET, ct_page_type, skps_chromium_build))
+
+ # Delete swarming_temp_dir to ensure it starts from a clean slate.
+ api.run.rmtree(api.swarming.swarming_temp_dir)
+
+ num_per_slave = api.properties.get(
+ 'num_per_slave',
+ min(TOOL_TO_DEFAULT_SKPS_PER_SLAVE[skia_tool], num_pages))
+ ct_num_slaves = api.properties.get(
+ 'ct_num_slaves',
+ int(math.ceil(float(num_pages) / num_per_slave)))
+
+ # Try to figure out if the SKPs we are going to isolate already exist
+ # locally by reading the SKPS_VERSION_FILE.
+ download_skps = True
+ expected_version_contents = {
+ "chromium_build": skps_chromium_build,
+ "page_type": ct_page_type,
+ "num_slaves": ct_num_slaves,
+ }
+ skps_dir = api.vars.checkout_root.join('skps', buildername)
+ version_file = skps_dir.join(SKPS_VERSION_FILE)
+ if api.path.exists(version_file): # pragma: nocover
+ version_file_contents = api.file.read(
+ "Read %s" % version_file,
+ version_file,
+ test_data=expected_version_contents,
+ infra_step=True)
+ actual_version_contents = api.json.loads(version_file_contents)
+ differences = (set(expected_version_contents.items()) ^
+ set(actual_version_contents.items()))
+ download_skps = len(differences) != 0
+ if download_skps:
+ # Delete and recreate the skps dir.
+ api.run.rmtree(skps_dir)
+ api.file.makedirs(api.path.basename(skps_dir), skps_dir)
+
+ # If a blacklist file exists then specify SKPs to be blacklisted.
+ blacklists_dir = api.vars.skia_dir.join('infra', 'bots', 'ct', 'blacklists')
+ blacklist_file = blacklists_dir.join(
+ '%s_%s_%s.json' % (skia_tool, ct_page_type, skps_chromium_build))
+ blacklist_skps = []
+ if api.path.exists(blacklist_file): # pragma: nocover
+ blacklist_file_contents = api.file.read(
+ "Read %s" % blacklist_file,
+ blacklist_file,
+ infra_step=True)
+ blacklist_skps = api.json.loads(blacklist_file_contents)['blacklisted_skps']
+
+ for slave_num in range(1, ct_num_slaves + 1):
+ if download_skps:
+ # Download SKPs.
+ api.ct.download_swarming_skps(
+ ct_page_type, slave_num, skps_chromium_build,
+ skps_dir,
+ start_range=((slave_num-1)*num_per_slave) + 1,
+ num_skps=num_per_slave)
+
+ # Create this slave's isolated.gen.json file to use for batcharchiving.
+ extra_variables = {
+ 'SLAVE_NUM': str(slave_num),
+ 'TOOL_NAME': skia_tool,
+ 'GIT_HASH': api.vars.got_revision,
+ 'CONFIGURATION': api.vars.configuration,
+ 'BUILDER': buildername,
+ }
+ api.swarming.create_isolated_gen_json(
+ isolate_path, isolate_dir, 'linux', 'ct-%s-%s' % (skia_tool, slave_num),
+ extra_variables, blacklist=blacklist_skps)
+
+ if download_skps:
+ # Since we had to download SKPs create an updated version file.
+ api.file.write("Create %s" % version_file,
+ version_file,
+ api.json.dumps(expected_version_contents),
+ infra_step=True)
+
+ # Batcharchive everything on the isolate server for efficiency.
+ max_slaves_to_batcharchive = MAX_SLAVES_TO_BATCHARCHIVE
+ if '1m' in buildername:
+ # Break up the "isolate tests" step into batches with <100k files due to
+ # https://github.com/luci/luci-go/issues/9
+ max_slaves_to_batcharchive = 5
+ tasks_to_swarm_hashes = []
+ for slave_start_num in xrange(1, ct_num_slaves+1, max_slaves_to_batcharchive):
+ m = min(max_slaves_to_batcharchive, ct_num_slaves)
+ batcharchive_output = api.swarming.batcharchive(
+ targets=['ct-' + skia_tool + '-%s' % num for num in range(
+ slave_start_num, slave_start_num + m)])
+ tasks_to_swarm_hashes.extend(batcharchive_output)
+ # Sort the list to go through tasks in order.
+ tasks_to_swarm_hashes.sort()
+
+ # Trigger all swarming tasks.
+ dimensions={'os': 'Ubuntu-14.04', 'cpu': 'x86-64', 'pool': 'Chrome'}
+ if 'GPU' in buildername:
+ dimensions['gpu'] = '10de:104a'
+ tasks = api.swarming.trigger_swarming_tasks(
+ tasks_to_swarm_hashes, dimensions=dimensions, io_timeout=40*60)
+
+ # Now collect all tasks.
+ failed_tasks = []
+ for task in tasks:
+ try:
+ api.swarming.collect_swarming_task(task)
+
+ if skia_tool == 'nanobench':
+ output_dir = api.swarming.tasks_output_dir.join(
+ task.title).join('0')
+ utc = api.time.utcnow()
+ gs_dest_dir = 'ct/%s/%d/%02d/%02d/%02d/' % (
+ ct_page_type, utc.year, utc.month, utc.day, utc.hour)
+ for json_output in api.file.listdir('output dir', output_dir):
+ api.gsutil.upload(
+ name='upload json output',
+ source=output_dir.join(json_output),
+ bucket='skia-perf',
+ dest=gs_dest_dir,
+ env={'AWS_CREDENTIAL_FILE': None, 'BOTO_CONFIG': None},
+ args=['-R']
+ )
+
+ except api.step.StepFailure as e:
+ failed_tasks.append(e)
+
+ if failed_tasks:
+ raise api.step.StepFailure(
+ 'Failed steps: %s' % ', '.join([f.name for f in failed_tasks]))
+
+
+def GenTests(api):
+ ct_num_slaves = 5
+ num_per_slave = 10
+ skia_revision = 'abc123'
+ mastername = 'client.skia'
+ slavename = 'skiabot-linux-swarm-000'
+ buildnumber = 2
+ path_config = 'kitchen'
+
+ yield(
+ api.test('CT_DM_10k_SKPs') +
+ api.properties(
+ buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_10k_SKPs',
+ mastername=mastername,
+ slavename=slavename,
+ buildnumber=buildnumber,
+ path_config=path_config,
+ swarm_out_dir='[SWARM_OUT_DIR]',
+ ct_num_slaves=ct_num_slaves,
+ num_per_slave=num_per_slave,
+ revision=skia_revision,
+ )
+ )
+
+ yield(
+ api.test('CT_IMG_DECODE_10k_SKPs') +
+ api.properties(
+ buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_'
+ '10k_SKPs',
+ mastername=mastername,
+ slavename=slavename,
+ buildnumber=buildnumber,
+ path_config=path_config,
+ swarm_out_dir='[SWARM_OUT_DIR]',
+ ct_num_slaves=ct_num_slaves,
+ num_per_slave=num_per_slave,
+ revision=skia_revision,
+ )
+ )
+
+ yield(
+ api.test('CT_DM_100k_SKPs') +
+ api.properties(
+ buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_100k_SKPs',
+ mastername=mastername,
+ slavename=slavename,
+ buildnumber=buildnumber,
+ path_config=path_config,
+ swarm_out_dir='[SWARM_OUT_DIR]',
+ ct_num_slaves=ct_num_slaves,
+ num_per_slave=num_per_slave,
+ revision=skia_revision,
+ )
+ )
+
+ yield(
+ api.test('CT_IMG_DECODE_100k_SKPs') +
+ api.properties(
+ buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_'
+ '100k_SKPs',
+ mastername=mastername,
+ slavename=slavename,
+ buildnumber=buildnumber,
+ path_config=path_config,
+ swarm_out_dir='[SWARM_OUT_DIR]',
+ ct_num_slaves=ct_num_slaves,
+ num_per_slave=num_per_slave,
+ revision=skia_revision,
+ )
+ )
+
+ yield(
+ api.test('CT_GPU_BENCH_1k_SKPs') +
+ api.properties(
+ buildername=
+ 'Perf-Ubuntu-GCC-Golo-GPU-GT610-x86_64-Release-CT_BENCH_1k_SKPs',
+ mastername=mastername,
+ slavename=slavename,
+ buildnumber=buildnumber,
+ path_config=path_config,
+ swarm_out_dir='[SWARM_OUT_DIR]',
+ ct_num_slaves=ct_num_slaves,
+ num_per_slave=num_per_slave,
+ revision=skia_revision,
+ ) +
+ api.path.exists(
+ api.path['slave_build'].join('skia'),
+ api.path['slave_build'].join('src')
+ )
+ )
+
+ yield(
+ api.test('CT_CPU_BENCH_10k_SKPs') +
+ api.properties(
+ buildername=
+ 'Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-CT_BENCH_10k_SKPs',
+ mastername=mastername,
+ slavename=slavename,
+ buildnumber=buildnumber,
+ path_config=path_config,
+ swarm_out_dir='[SWARM_OUT_DIR]',
+ ct_num_slaves=ct_num_slaves,
+ num_per_slave=num_per_slave,
+ revision=skia_revision,
+ ) +
+ api.path.exists(
+ api.path['slave_build'].join('skia'),
+ api.path['slave_build'].join('src')
+ )
+ )
+
+ yield(
+ api.test('CT_GPU_BENCH_10k_SKPs') +
+ api.properties(
+ buildername=
+ 'Perf-Ubuntu-GCC-Golo-GPU-GT610-x86_64-Release-CT_BENCH_10k_SKPs',
+ mastername=mastername,
+ slavename=slavename,
+ buildnumber=buildnumber,
+ path_config=path_config,
+ swarm_out_dir='[SWARM_OUT_DIR]',
+ ct_num_slaves=ct_num_slaves,
+ num_per_slave=num_per_slave,
+ revision=skia_revision,
+ ) +
+ api.path.exists(
+ api.path['slave_build'].join('skia'),
+ api.path['slave_build'].join('src')
+ )
+ )
+
+ yield(
+ api.test('CT_DM_1m_SKPs') +
+ api.properties(
+ buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs',
+ mastername=mastername,
+ slavename=slavename,
+ buildnumber=buildnumber,
+ path_config=path_config,
+ swarm_out_dir='[SWARM_OUT_DIR]',
+ ct_num_slaves=ct_num_slaves,
+ num_per_slave=num_per_slave,
+ revision=skia_revision,
+ )
+ )
+
+ yield (
+ api.test('CT_DM_SKPs_UnknownBuilder') +
+ api.properties(
+ buildername=
+ 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_UnknownRepo_SKPs',
+ mastername=mastername,
+ slavename=slavename,
+ buildnumber=buildnumber,
+ path_config=path_config,
+ swarm_out_dir='[SWARM_OUT_DIR]',
+ ct_num_slaves=ct_num_slaves,
+ num_per_slave=num_per_slave,
+ revision=skia_revision,
+ ) +
+ api.expect_exception('Exception')
+ )
+
+ yield (
+ api.test('CT_10k_SKPs_UnknownBuilder') +
+ api.properties(
+ buildername=
+ 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_UnknownTool_10k_SKPs',
+ mastername=mastername,
+ slavename=slavename,
+ buildnumber=buildnumber,
+ path_config=path_config,
+ swarm_out_dir='[SWARM_OUT_DIR]',
+ ct_num_slaves=ct_num_slaves,
+ num_per_slave=num_per_slave,
+ revision=skia_revision,
+ ) +
+ api.expect_exception('Exception')
+ )
+
+ yield(
+ api.test('CT_DM_1m_SKPs_slave3_failure') +
+ api.step_data('ct-dm-3 on Ubuntu-14.04', retcode=1) +
+ api.properties(
+ buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs',
+ mastername=mastername,
+ slavename=slavename,
+ buildnumber=buildnumber,
+ path_config=path_config,
+ swarm_out_dir='[SWARM_OUT_DIR]',
+ ct_num_slaves=ct_num_slaves,
+ num_per_slave=num_per_slave,
+ revision=skia_revision,
+ )
+ )
+
+ yield(
+ api.test('CT_DM_1m_SKPs_2slaves_failure') +
+ api.step_data('ct-dm-1 on Ubuntu-14.04', retcode=1) +
+ api.step_data('ct-dm-3 on Ubuntu-14.04', retcode=1) +
+ api.properties(
+ buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs',
+ mastername=mastername,
+ slavename=slavename,
+ buildnumber=buildnumber,
+ path_config=path_config,
+ swarm_out_dir='[SWARM_OUT_DIR]',
+ ct_num_slaves=ct_num_slaves,
+ num_per_slave=num_per_slave,
+ revision=skia_revision,
+ )
+ )
+
+ yield(
+ api.test('CT_DM_10k_SKPs_Trybot') +
+ api.properties(
+ buildername=
+ 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_10k_SKPs-Trybot',
+ mastername=mastername,
+ slavename=slavename,
+ buildnumber=buildnumber,
+ path_config=path_config,
+ swarm_out_dir='[SWARM_OUT_DIR]',
+ ct_num_slaves=ct_num_slaves,
+ num_per_slave=num_per_slave,
+ rietveld='codereview.chromium.org',
+ issue=1499623002,
+ patchset=1,
+ )
+ )
+
+ yield(
+ api.test('CT_IMG_DECODE_10k_SKPs_Trybot') +
+ api.properties(
+ buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_'
+ '10k_SKPs_Trybot',
+ mastername=mastername,
+ slavename=slavename,
+ buildnumber=buildnumber,
+ path_config=path_config,
+ swarm_out_dir='[SWARM_OUT_DIR]',
+ ct_num_slaves=ct_num_slaves,
+ num_per_slave=num_per_slave,
+ revision=skia_revision,
+ )
+ )
« no previous file with comments | « infra/bots/recipes/swarm_compile.expected/win_retry_failed_compile.json ('k') | infra/bots/recipes/swarm_ct_skps.expected/CT_10k_SKPs_UnknownBuilder.json » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698