Move ct_skps recipe from tools repo to Skia repo

infra/bots/recipes/swarm_ct_skps.py

+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+import math
+
+
+DEPS = [
+  'build/file',
+  'build/gsutil',
+  'core',
+  'ct',
+  'flavor',
+  'recipe_engine/json',
+  'recipe_engine/path',
+  'recipe_engine/properties',
+  'recipe_engine/step',
+  'recipe_engine/time',
+  'run',
+  'swarming',
+  'vars',
+]
+
+
+SKPS_VERSION_FILE = 'skps_version'
+CT_SKPS_ISOLATE = 'ct_skps.isolate'
+
+# Do not batch archive more slaves than this value. This is used to prevent
+# no output timeouts in the 'isolate tests' step.
+MAX_SLAVES_TO_BATCHARCHIVE = 100
+
+TOOL_TO_DEFAULT_SKPS_PER_SLAVE = {
+    'dm': 10000,
+    'nanobench': 1000,
+    'get_images_from_skps': 10000,
+}
+
+# The SKP repository to use.
+DEFAULT_SKPS_CHROMIUM_BUILD = 'fad657e-276e633'
+
+
+def RunSteps(api):
+  # Figure out which repository to use.
+  buildername = api.properties['buildername']
+  if '1k' in buildername:
+    ct_page_type = '10k'
+    num_pages = 1000
+  elif '10k' in buildername:
+    ct_page_type = '10k'
+    num_pages = 10000
+  elif '100k' in buildername:
+    ct_page_type = '100k'
+    num_pages = 100000
+  elif '1m' in buildername:
+    ct_page_type = 'All'
+    num_pages = 1000000
+  else:
+    raise Exception('Do not recognise the buildername %s.' % buildername)
+
+  # Figure out which tool to use.
+  if 'DM' in buildername:
+    skia_tool = 'dm'
+    build_target = 'dm'
+  elif 'BENCH' in buildername:
+    skia_tool = 'nanobench'
+    build_target = 'nanobench'
+  elif 'IMG_DECODE' in buildername:
+    skia_tool = 'get_images_from_skps'
+    build_target = 'tools'
+  else:
+    raise Exception('Do not recognise the buildername %s.' % buildername)
+
+  api.core.setup()
+  api.flavor.compile(build_target)
+
+  # Required paths.
+  infrabots_dir = api.vars.skia_dir.join('infra', 'bots')
+  isolate_dir = infrabots_dir.join('ct')
+  isolate_path = isolate_dir.join(CT_SKPS_ISOLATE)
+
+  api.run.copy_build_products(
+      api.flavor.out_dir,
+      isolate_dir)
+  api.swarming.setup(
+      infrabots_dir.join('tools', 'luci-go'),
+      swarming_rev='')
+
+  skps_chromium_build = api.properties.get(
+      'skps_chromium_build', DEFAULT_SKPS_CHROMIUM_BUILD)
+
+  # Set build property to make finding SKPs convenient.
+  api.step.active_result.presentation.properties['Location of SKPs'] = (
+      'https://pantheon.corp.google.com/storage/browser/%s/swarming/skps/%s/%s/'
+          % (api.ct.CT_GS_BUCKET, ct_page_type, skps_chromium_build))
+
+  # Delete swarming_temp_dir to ensure it starts from a clean slate.
+  api.run.rmtree(api.swarming.swarming_temp_dir)
+
+  num_per_slave = api.properties.get(
+      'num_per_slave',
+      min(TOOL_TO_DEFAULT_SKPS_PER_SLAVE[skia_tool], num_pages))
+  ct_num_slaves = api.properties.get(
+      'ct_num_slaves',
+      int(math.ceil(float(num_pages) / num_per_slave)))
+
+  # Try to figure out if the SKPs we are going to isolate already exist
+  # locally by reading the SKPS_VERSION_FILE.
+  download_skps = True
+  expected_version_contents = {
+      "chromium_build": skps_chromium_build,
+      "page_type": ct_page_type,
+      "num_slaves": ct_num_slaves,
+  }
+  skps_dir = api.vars.checkout_root.join('skps', buildername)
+  version_file = skps_dir.join(SKPS_VERSION_FILE)
+  if api.path.exists(version_file):  # pragma: nocover
+    version_file_contents = api.file.read(
+        "Read %s" % version_file,
+        version_file,
+        test_data=expected_version_contents,
+        infra_step=True)
+    actual_version_contents = api.json.loads(version_file_contents)
+    differences = (set(expected_version_contents.items()) ^
+                   set(actual_version_contents.items()))
+    download_skps = len(differences) != 0
+    if download_skps:
+      # Delete and recreate the skps dir.
+      api.run.rmtree(skps_dir)
+      api.file.makedirs(api.path.basename(skps_dir), skps_dir)
+
+  # If a blacklist file exists then specify SKPs to be blacklisted.
+  blacklists_dir = api.vars.skia_dir.join('infra', 'bots', 'ct', 'blacklists')
+  blacklist_file = blacklists_dir.join(
+      '%s_%s_%s.json' % (skia_tool, ct_page_type, skps_chromium_build))
+  blacklist_skps = []
+  if api.path.exists(blacklist_file):  # pragma: nocover
+    blacklist_file_contents = api.file.read(
+        "Read %s" % blacklist_file,
+        blacklist_file,
+        infra_step=True)
+    blacklist_skps = api.json.loads(blacklist_file_contents)['blacklisted_skps']
+
+  for slave_num in range(1, ct_num_slaves + 1):
+    if download_skps:
+      # Download SKPs.
+      api.ct.download_swarming_skps(
+          ct_page_type, slave_num, skps_chromium_build,
+          skps_dir,
+          start_range=((slave_num-1)*num_per_slave) + 1,
+          num_skps=num_per_slave)
+
+    # Create this slave's isolated.gen.json file to use for batcharchiving.
+    extra_variables = {
+        'SLAVE_NUM': str(slave_num),
+        'TOOL_NAME': skia_tool,
+        'GIT_HASH': api.vars.got_revision,
+        'CONFIGURATION': api.vars.configuration,
+        'BUILDER': buildername,
+    }
+    api.swarming.create_isolated_gen_json(
+        isolate_path, isolate_dir, 'linux', 'ct-%s-%s' % (skia_tool, slave_num),
+        extra_variables, blacklist=blacklist_skps)
+
+  if download_skps:
+    # Since we had to download SKPs create an updated version file.
+    api.file.write("Create %s" % version_file,
+                   version_file,
+                   api.json.dumps(expected_version_contents),
+                   infra_step=True)
+
+  # Batcharchive everything on the isolate server for efficiency.
+  max_slaves_to_batcharchive = MAX_SLAVES_TO_BATCHARCHIVE
+  if '1m' in buildername:
+    # Break up the "isolate tests" step into batches with <100k files due to
+    # https://github.com/luci/luci-go/issues/9
+    max_slaves_to_batcharchive = 5
+  tasks_to_swarm_hashes = []
+  for slave_start_num in xrange(1, ct_num_slaves+1, max_slaves_to_batcharchive):
+    m = min(max_slaves_to_batcharchive, ct_num_slaves)
+    batcharchive_output = api.swarming.batcharchive(
+        targets=['ct-' + skia_tool + '-%s' % num for num in range(
+            slave_start_num, slave_start_num + m)])
+    tasks_to_swarm_hashes.extend(batcharchive_output)
+  # Sort the list to go through tasks in order.
+  tasks_to_swarm_hashes.sort()
+
+  # Trigger all swarming tasks.
+  dimensions={'os': 'Ubuntu-14.04', 'cpu': 'x86-64', 'pool': 'Chrome'}
+  if 'GPU' in buildername:
+    dimensions['gpu'] = '10de:104a'
+  tasks = api.swarming.trigger_swarming_tasks(
+      tasks_to_swarm_hashes, dimensions=dimensions, io_timeout=40*60)
+
+  # Now collect all tasks.
+  failed_tasks = []
+  for task in tasks:
+    try:
+      api.swarming.collect_swarming_task(task)
+
+      if skia_tool == 'nanobench':
+        output_dir = api.swarming.tasks_output_dir.join(
+            task.title).join('0')
+        utc = api.time.utcnow()
+        gs_dest_dir = 'ct/%s/%d/%02d/%02d/%02d/' % (
+            ct_page_type, utc.year, utc.month, utc.day, utc.hour)
+        for json_output in api.file.listdir('output dir', output_dir):
+          api.gsutil.upload(
+              name='upload json output',
+              source=output_dir.join(json_output),
+              bucket='skia-perf',
+              dest=gs_dest_dir,
+              env={'AWS_CREDENTIAL_FILE': None, 'BOTO_CONFIG': None},
+              args=['-R']
+          )
+
+    except api.step.StepFailure as e:
+      failed_tasks.append(e)
+
+  if failed_tasks:
+    raise api.step.StepFailure(
+        'Failed steps: %s' % ', '.join([f.name for f in failed_tasks]))
+
+
+def GenTests(api):
+  ct_num_slaves = 5
+  num_per_slave = 10
+  skia_revision = 'abc123'
+  mastername = 'client.skia'
+  slavename = 'skiabot-linux-swarm-000'
+  buildnumber = 2
+  path_config = 'kitchen'
+
+  yield(
+    api.test('CT_DM_10k_SKPs') +
+    api.properties(
+        buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_10k_SKPs',
+        mastername=mastername,
+        slavename=slavename,
+        buildnumber=buildnumber,
+        path_config=path_config,
+        swarm_out_dir='[SWARM_OUT_DIR]',
+        ct_num_slaves=ct_num_slaves,
+        num_per_slave=num_per_slave,
+        revision=skia_revision,
+    )
+  )
+
+  yield(
+    api.test('CT_IMG_DECODE_10k_SKPs') +
+    api.properties(
+        buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_'
+                    '10k_SKPs',
+        mastername=mastername,
+        slavename=slavename,
+        buildnumber=buildnumber,
+        path_config=path_config,
+        swarm_out_dir='[SWARM_OUT_DIR]',
+        ct_num_slaves=ct_num_slaves,
+        num_per_slave=num_per_slave,
+        revision=skia_revision,
+    )
+  )
+
+  yield(
+    api.test('CT_DM_100k_SKPs') +
+    api.properties(
+        buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_100k_SKPs',
+        mastername=mastername,
+        slavename=slavename,
+        buildnumber=buildnumber,
+        path_config=path_config,
+        swarm_out_dir='[SWARM_OUT_DIR]',
+        ct_num_slaves=ct_num_slaves,
+        num_per_slave=num_per_slave,
+        revision=skia_revision,
+    )
+  )
+
+  yield(
+    api.test('CT_IMG_DECODE_100k_SKPs') +
+    api.properties(
+        buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_'
+                    '100k_SKPs',
+        mastername=mastername,
+        slavename=slavename,
+        buildnumber=buildnumber,
+        path_config=path_config,
+        swarm_out_dir='[SWARM_OUT_DIR]',
+        ct_num_slaves=ct_num_slaves,
+        num_per_slave=num_per_slave,
+        revision=skia_revision,
+    )
+  )
+
+  yield(
+    api.test('CT_GPU_BENCH_1k_SKPs') +
+    api.properties(
+        buildername=
+            'Perf-Ubuntu-GCC-Golo-GPU-GT610-x86_64-Release-CT_BENCH_1k_SKPs',
+        mastername=mastername,
+        slavename=slavename,
+        buildnumber=buildnumber,
+        path_config=path_config,
+        swarm_out_dir='[SWARM_OUT_DIR]',
+        ct_num_slaves=ct_num_slaves,
+        num_per_slave=num_per_slave,
+        revision=skia_revision,
+    ) +
+    api.path.exists(
+        api.path['slave_build'].join('skia'),
+        api.path['slave_build'].join('src')
+    )
+  )
+
+  yield(
+    api.test('CT_CPU_BENCH_10k_SKPs') +
+    api.properties(
+        buildername=
+            'Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-CT_BENCH_10k_SKPs',
+        mastername=mastername,
+        slavename=slavename,
+        buildnumber=buildnumber,
+        path_config=path_config,
+        swarm_out_dir='[SWARM_OUT_DIR]',
+        ct_num_slaves=ct_num_slaves,
+        num_per_slave=num_per_slave,
+        revision=skia_revision,
+    ) +
+    api.path.exists(
+        api.path['slave_build'].join('skia'),
+        api.path['slave_build'].join('src')
+    )
+  )
+
+  yield(
+    api.test('CT_GPU_BENCH_10k_SKPs') +
+    api.properties(
+        buildername=
+            'Perf-Ubuntu-GCC-Golo-GPU-GT610-x86_64-Release-CT_BENCH_10k_SKPs',
+        mastername=mastername,
+        slavename=slavename,
+        buildnumber=buildnumber,
+        path_config=path_config,
+        swarm_out_dir='[SWARM_OUT_DIR]',
+        ct_num_slaves=ct_num_slaves,
+        num_per_slave=num_per_slave,
+        revision=skia_revision,
+    ) +
+    api.path.exists(
+        api.path['slave_build'].join('skia'),
+        api.path['slave_build'].join('src')
+    )
+  )
+
+  yield(
+    api.test('CT_DM_1m_SKPs') +
+    api.properties(
+        buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs',
+        mastername=mastername,
+        slavename=slavename,
+        buildnumber=buildnumber,
+        path_config=path_config,
+        swarm_out_dir='[SWARM_OUT_DIR]',
+        ct_num_slaves=ct_num_slaves,
+        num_per_slave=num_per_slave,
+        revision=skia_revision,
+    )
+  )
+
+  yield (
+    api.test('CT_DM_SKPs_UnknownBuilder') +
+    api.properties(
+        buildername=
+            'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_UnknownRepo_SKPs',
+        mastername=mastername,
+        slavename=slavename,
+        buildnumber=buildnumber,
+        path_config=path_config,
+        swarm_out_dir='[SWARM_OUT_DIR]',
+        ct_num_slaves=ct_num_slaves,
+        num_per_slave=num_per_slave,
+        revision=skia_revision,
+    ) +
+    api.expect_exception('Exception')
+  )
+
+  yield (
+    api.test('CT_10k_SKPs_UnknownBuilder') +
+    api.properties(
+        buildername=
+            'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_UnknownTool_10k_SKPs',
+        mastername=mastername,
+        slavename=slavename,
+        buildnumber=buildnumber,
+        path_config=path_config,
+        swarm_out_dir='[SWARM_OUT_DIR]',
+        ct_num_slaves=ct_num_slaves,
+        num_per_slave=num_per_slave,
+        revision=skia_revision,
+    ) +
+    api.expect_exception('Exception')
+  )
+
+  yield(
+    api.test('CT_DM_1m_SKPs_slave3_failure') +
+    api.step_data('ct-dm-3 on Ubuntu-14.04', retcode=1) +
+    api.properties(
+        buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs',
+        mastername=mastername,
+        slavename=slavename,
+        buildnumber=buildnumber,
+        path_config=path_config,
+        swarm_out_dir='[SWARM_OUT_DIR]',
+        ct_num_slaves=ct_num_slaves,
+        num_per_slave=num_per_slave,
+        revision=skia_revision,
+    )
+  )
+
+  yield(
+    api.test('CT_DM_1m_SKPs_2slaves_failure') +
+    api.step_data('ct-dm-1 on Ubuntu-14.04', retcode=1) +
+    api.step_data('ct-dm-3 on Ubuntu-14.04', retcode=1) +
+    api.properties(
+        buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs',
+        mastername=mastername,
+        slavename=slavename,
+        buildnumber=buildnumber,
+        path_config=path_config,
+        swarm_out_dir='[SWARM_OUT_DIR]',
+        ct_num_slaves=ct_num_slaves,
+        num_per_slave=num_per_slave,
+        revision=skia_revision,
+    )
+  )
+
+  yield(
+    api.test('CT_DM_10k_SKPs_Trybot') +
+    api.properties(
+        buildername=
+            'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_10k_SKPs-Trybot',
+        mastername=mastername,
+        slavename=slavename,
+        buildnumber=buildnumber,
+        path_config=path_config,
+        swarm_out_dir='[SWARM_OUT_DIR]',
+        ct_num_slaves=ct_num_slaves,
+        num_per_slave=num_per_slave,
+        rietveld='codereview.chromium.org',
+        issue=1499623002,
+        patchset=1,
+    )
+  )
+
+  yield(
+    api.test('CT_IMG_DECODE_10k_SKPs_Trybot') +
+    api.properties(
+        buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_'
+                    '10k_SKPs_Trybot',
+        mastername=mastername,
+        slavename=slavename,
+        buildnumber=buildnumber,
+        path_config=path_config,
+        swarm_out_dir='[SWARM_OUT_DIR]',
+        ct_num_slaves=ct_num_slaves,
+        num_per_slave=num_per_slave,
+        revision=skia_revision,
+    )
+  )