Add swarming task for nanobench upload

infra/bots/recipes/swarm_trigger.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 
 # Recipe module for Skia Swarming trigger.
 
 
 import os
 import json
@@ skipping 47 matching lines @@
       'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared',
       'Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind',
       'Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release',
       'Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release',
       'Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release',
     ],
   },
 }
 
 
+UPLOAD_DIMENSIONS = {
+  'pool': 'Skia',
+  'os': 'Linux',
+  'cpu': 'x86-64-avx2',
+  'gpu': 'none',
+}
+
+
 def derive_compile_bot_name(api):
   builder_name = api.properties['buildername']
   builder_cfg = api.builder_name_schema.DictForBuilderName(builder_name)
   if builder_cfg['role'] == 'Housekeeper':
     return 'Build-Ubuntu-GCC-x86_64-Release-Shared'
   if builder_cfg['role'] in ('Test', 'Perf'):
     task_os = builder_cfg['os']
     extra_config = builder_cfg.get('extra_config', '')
     if task_os == 'Android':
       if extra_config == 'Vulkan':
@@ skipping 99 matching lines @@
         os.chmod(f, 0755)
       else:
         os.chmod(f, 0644)
 ''',
       cwd=path)
 
 
 def trigger_task(api, task_name, builder, master, slave, buildnumber,
                  builder_cfg, got_revision, infrabots_dir, idempotent=False,
                  store_output=True, extra_isolate_hashes=None, expiration=None,
-                 hard_timeout=None, io_timeout=None, cipd_packages=None):
+                 hard_timeout=None, io_timeout=None, cipd_packages=None,
+                 recipe_name=None, isolate_file=None, dimensions=None):
   """Trigger the given bot to run as a Swarming task."""
   # TODO(borenet): We're using Swarming directly to run the recipe through
   # recipes.py. Once it's possible to track the state of a Buildbucket build,
   # we should switch to use the trigger recipe module instead.
 
   properties = {
     'buildername': builder,
     'mastername': master,
     'buildnumber': buildnumber,
     'reason': 'Triggered by Skia swarm_trigger Recipe',
     'revision': got_revision,
     'slavename': slave,
     'swarm_out_dir': '${ISOLATED_OUTDIR}',
   }
   if builder_cfg['is_trybot']:
     if api.properties.get('patch_storage') == 'gerrit':
       properties['patch_storage'] = api.properties['patch_storage']
       properties['repository'] = api.properties['repository']
       properties['event.patchSet.ref'] = api.properties['event.patchSet.ref']
       properties['event.change.number'] = api.properties['event.change.number']
     else:
       properties['issue'] = str(api.properties['issue'])
       properties['patchset'] = str(api.properties['patchset'])
       properties['rietveld'] = api.properties['rietveld']
 
   extra_args = [
       '--workdir', '../../..',
-      'swarm_%s' % task_name,
+      recipe_name or 'swarm_%s' % task_name,
   ]
   for k, v in properties.iteritems():
     extra_args.append('%s=%s' % (k, v))
 
   isolate_base_dir = api.path['slave_build']
-  dimensions = swarm_dimensions(builder_cfg)
+  dimensions = dimensions or swarm_dimensions(builder_cfg)
   isolate_blacklist = ['.git', 'out', '*.pyc', '.recipe_deps']
   isolate_vars = {
     'WORKDIR': api.path['slave_build'],
   }
 
-  isolate_file = '%s_skia.isolate' % task_name
+  isolate_file = isolate_file or '%s_skia.isolate' % task_name
   if 'Coverage' == builder_cfg.get('configuration'):
     isolate_file = 'coverage_skia.isolate'
   if 'RecreateSKPs' in builder:
     isolate_file = 'compile_skia.isolate'
   return api.swarming.isolate_and_trigger_task(
       infrabots_dir.join(isolate_file),
       isolate_base_dir,
       '%s_skia' % task_name,
       isolate_vars,
       dimensions,
@@ skipping 222 matching lines @@
       builder_cfg,
       got_revision,
       infrabots_dir,
       extra_isolate_hashes=extra_hashes,
       expiration=expiration,
       hard_timeout=hard_timeout,
       io_timeout=io_timeout,
       cipd_packages=cipd_packages)
 
 
-def perf_steps_collect(api, task, got_revision, is_trybot):
+def perf_steps_collect(api, task, builder_cfg, got_revision, infrabots_dir):
   """Wait for perf steps to finish and upload results."""
   # Wait for nanobench to finish, download the results.
   api.run.rmtree(task.task_output_dir)
-  api.swarming.collect_swarming_task(task)
+  if not api.vars.upload_perf_results:  # pragma: nocover
+    api.swarming.collect_swarming_task(task)
+    return
+
+  perf_hash = api.swarming.collect_swarming_task_isolate_hash(task)
 
   # Upload the results.
-  if api.vars.upload_perf_results:
-    perf_data_dir = api.path['slave_build'].join(
-        'perfdata', api.properties['buildername'], 'data')
-    git_timestamp = api.git.get_timestamp(test_data='1408633190',
-                                          infra_step=True)
-    api.run.rmtree(perf_data_dir)
-    api.file.makedirs('perf_dir', perf_data_dir, infra_step=True)
-    src_results_file = task.task_output_dir.join(
-        '0', 'perfdata', api.properties['buildername'], 'data',
-        'nanobench_%s.json' % got_revision)
-    dst_results_file = perf_data_dir.join(
-        'nanobench_%s_%s.json' % (got_revision, git_timestamp))
-    api.file.copy('perf_results', src_results_file, dst_results_file,
-                  infra_step=True)
+  task = trigger_task(
+      api,
+      'upload_nano_results',
+      api.properties['buildername'],
+      api.properties['mastername'],
+      api.properties['slavename'],
+      api.properties['buildnumber'],
+      builder_cfg,
+      got_revision,
+      infrabots_dir,
+      idempotent=True,
+      store_output=False,
+      cipd_packages=None,
+      extra_isolate_hashes=[perf_hash],
+      recipe_name='upload_nano_results',
+      isolate_file='upload_nano_results.isolate',
+      dimensions=UPLOAD_DIMENSIONS)
 
-    gsutil_path = api.path['slave_build'].join(
-        'skia', 'infra', 'bots', '.recipe_deps', 'depot_tools', 'gsutil.py')
-    upload_args = [api.properties['buildername'], api.properties['buildnumber'],
-                   perf_data_dir, got_revision, gsutil_path]
-    if is_trybot:
-      upload_args.append(get_issue_num(api))
-    api.python(
-        'Upload perf results',
-        script=api.core.resource('upload_bench_results.py'),
-        args=upload_args,
-        cwd=api.path['checkout'],
-        infra_step=True)
+  return api.swarming.collect_swarming_task(task)
 
 
 def test_steps_trigger(api, builder_cfg, got_revision, infrabots_dir,
                        extra_hashes, cipd_packages):
   """Trigger DM via Swarming."""
   expiration, hard_timeout, io_timeout = get_timeouts(builder_cfg)
   return trigger_task(
       api,
       'test',
       api.properties['buildername'],
@@ skipping 221 matching lines @@
                                    infrabots_dir, extra_hashes, cipd_packages)
   is_trybot = builder_cfg['is_trybot']
 
   # Wait for results, then upload them if necessary.
 
   if test_task:
     test_steps_collect(api, test_task,
                        got_revision, is_trybot, builder_cfg)
 
   if perf_task:
-    perf_steps_collect(api, perf_task,
-                       got_revision, is_trybot)
+    perf_steps_collect(api, perf_task, builder_cfg,
+                       got_revision, infrabots_dir)
 
 
 def test_for_bot(api, builder, mastername, slavename, testname=None):
   """Generate a test for the given bot."""
   testname = testname or builder
   test = (
     api.test(testname) +
     api.properties(buildername=builder,
                    mastername=mastername,
                    slavename=slavename,
@@ skipping 13 matching lines @@
     paths.append(api.path['slave_build'].join(
         'skia', 'infra', 'bots', 'assets', 'android_sdk', 'VERSION'))
   if 'Test' in builder and 'Coverage' not in builder:
     test += api.step_data(
         'upload new .isolated file for test_skia',
         stdout=api.raw_io.output('def456 XYZ.isolated'))
   if 'Perf' in builder and '-CT_' not in builder:
     test += api.step_data(
         'upload new .isolated file for perf_skia',
         stdout=api.raw_io.output('def456 XYZ.isolated'))
+    test += api.step_data(
+        'upload new .isolated file for upload_nano_results_skia',
+        stdout=api.raw_io.output('def456 XYZ.isolated'))
   if 'Housekeeper' in builder and 'RecreateSKPs' not in builder:
     test += api.step_data(
         'upload new .isolated file for housekeeper_skia',
         stdout=api.raw_io.output('def456 XYZ.isolated'))
   if 'Win' in builder:
     paths.append(api.path['slave_build'].join(
         'skia', 'infra', 'bots', 'assets', 'win_toolchain', 'VERSION'))
     paths.append(api.path['slave_build'].join(
         'skia', 'infra', 'bots', 'assets', 'win_vulkan_sdk', 'VERSION'))
   paths.append(api.path['slave_build'].join(
@@ skipping 27 matching lines @@
           mastername='client.skia',
           slavename='skiabot-linux-swarm-000',
           buildnumber=5,
           path_config='kitchen',
           revision='abc123',
           **gerrit_kwargs) +
       api.step_data(
           'upload new .isolated file for test_skia',
           stdout=api.raw_io.output('def456 XYZ.isolated'))
   )