[recipes] Smush the rest of builder_spec into appropriate places, other cleanup

infra/bots/recipes/swarm_trigger.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 
 # Recipe module for Skia Swarming trigger.
 
 
+import os
 import json
 
 
 DEPS = [
   'build/file',
   'build/gsutil',
   'builder_name_schema',
   'core',
   'depot_tools/depot_tools',
   'depot_tools/git',
@@ skipping 40 matching lines @@
   },
 }
 
 
 def derive_compile_bot_name(api):
   builder_name = api.properties['buildername']
   builder_cfg = api.builder_name_schema.DictForBuilderName(builder_name)
   if builder_cfg['role'] == 'Housekeeper':
     return 'Build-Ubuntu-GCC-x86_64-Release-Shared'
   if builder_cfg['role'] in ('Test', 'Perf'):
-    os = builder_cfg['os']
+    task_os = builder_cfg['os']
     extra_config = builder_cfg.get('extra_config')
-    if os == 'Android':
+    if task_os == 'Android':
       if extra_config == 'Vulkan':
-        extra_config = '%s_%s' % (os, 'Vulkan')
+        extra_config = '%s_%s' % (task_os, 'Vulkan')
       else:
-        extra_config = os
-      os = 'Ubuntu'
-    elif os == 'iOS':
-      extra_config = os
-      os = 'Mac'
-    elif 'Win' in os:
-      os = 'Win'
+        extra_config = task_os
+      task_os = 'Ubuntu'
+    elif task_os == 'iOS':
+      extra_config = task_os
+      task_os = 'Mac'
+    elif 'Win' in task_os:
+      task_os = 'Win'
     return api.builder_name_schema.MakeBuilderName(
         role=api.builder_name_schema.BUILDER_ROLE_BUILD,
-        os=os,
+        os=task_os,
         compiler=builder_cfg['compiler'],
         target_arch=builder_cfg['arch'],
         configuration=builder_cfg['configuration'],
         extra_config=extra_config,
         is_trybot=api.builder_name_schema.IsTrybot(builder_name))
   return builder_name
 
 
-def swarm_dimensions(builder_spec):
+def swarm_dimensions(builder_cfg):
   """Return a dict of keys and values to be used as Swarming bot dimensions."""
   dimensions = {
     'pool': 'Skia',
   }
-  builder_cfg = builder_spec['builder_cfg']
   dimensions['os'] = builder_cfg.get('os', 'Ubuntu')
   if 'Win' in builder_cfg.get('os', ''):
     dimensions['os'] = 'Windows'
   if builder_cfg['role'] in ('Test', 'Perf'):
     if 'Android' in builder_cfg['os']:
       # For Android, the device type is a better dimension than CPU or GPU.
-      dimensions['device_type'] = builder_spec['product.board']
+      dimensions['device_type'] = {
+        'AndroidOne':    'sprout',
+        'GalaxyS3':      'm0',  #'smdk4x12', Detected incorrectly by swarming?
+        'GalaxyS4':      None,  # TODO(borenet,kjlubick)
+        'NVIDIA_Shield': 'foster',
+        'Nexus10':       'manta',
+        'Nexus5':        'hammerhead',
+        'Nexus6':        'shamu',
+        'Nexus7':        'grouper',
+        'Nexus7v2':      'flo',
+        'Nexus9':        'flounder',
+        'NexusPlayer':   'fugu',
+      }[builder_cfg['model']]
     elif 'iOS' in builder_cfg['os']:
       # For iOS, the device type is a better dimension than CPU or GPU.
-      dimensions['device'] = builder_spec['device_cfg']
+      dimensions['device'] = {
+        'iPad4': 'iPad4,1',
+      }[builder_cfg['model']]
       # TODO(borenet): Replace this hack with something better.
       dimensions['os'] = 'iOS-9.2'
     elif builder_cfg['cpu_or_gpu'] == 'CPU':
       dimensions['gpu'] = 'none'
       dimensions['cpu'] = {
         'AVX':  'x86-64',
         'AVX2': 'x86-64-avx2',
         'SSE4': 'x86-64',
       }[builder_cfg['cpu_or_gpu_value']]
       if ('Win' in builder_cfg['os'] and
@@ skipping 30 matching lines @@
     if os.path.isfile(f):
       if os.access(f, os.X_OK):
         os.chmod(f, 0755)
       else:
         os.chmod(f, 0644)
 ''',
       cwd=path)
 
 
 def trigger_task(api, task_name, builder, master, slave, buildnumber,
-                 builder_spec, got_revision, infrabots_dir, idempotent=False,
+                 builder_cfg, got_revision, infrabots_dir, idempotent=False,
                  store_output=True, extra_isolate_hashes=None, expiration=None,
                  hard_timeout=None, io_timeout=None, cipd_packages=None):
   """Trigger the given bot to run as a Swarming task."""
   # TODO(borenet): We're using Swarming directly to run the recipe through
   # recipes.py. Once it's possible to track the state of a Buildbucket build,
   # we should switch to use the trigger recipe module instead.
 
   properties = {
     'buildername': builder,
     'mastername': master,
     'buildnumber': buildnumber,
     'reason': 'Triggered by Skia swarm_trigger Recipe',
     'revision': got_revision,
     'slavename': slave,
     'swarm_out_dir': '${ISOLATED_OUTDIR}',
   }
-  builder_cfg = builder_spec['builder_cfg']
   if builder_cfg['is_trybot']:
     properties['issue'] = str(api.properties['issue'])
     properties['patchset'] = str(api.properties['patchset'])
     properties['rietveld'] = api.properties['rietveld']
 
   extra_args = [
       '--workdir', '../../..',
       'swarm_%s' % task_name,
   ]
   for k, v in properties.iteritems():
     extra_args.append('%s=%s' % (k, v))
 
   isolate_base_dir = api.path['slave_build']
-  dimensions = swarm_dimensions(builder_spec)
+  dimensions = swarm_dimensions(builder_cfg)
   isolate_blacklist = ['.git', 'out', '*.pyc', '.recipe_deps']
   isolate_vars = {
     'WORKDIR': api.path['slave_build'],
   }
 
   isolate_file = '%s_skia.isolate' % task_name
   if 'Coverage' == builder_cfg.get('configuration'):
     isolate_file = 'coverage_skia.isolate'
   if 'RecreateSKPs' in builder:
     isolate_file = 'compile_skia.isolate'
@@ skipping 31 matching lines @@
 
   # Write a fake .gclient file if none exists. This is required by .isolates.
   dot_gclient = api.path['slave_build'].join('.gclient')
   if not api.path.exists(dot_gclient):
     api.run.writefile(dot_gclient, '')
 
   fix_filemodes(api, api.path['checkout'])
   return got_revision
 
 
-def housekeeper_swarm(api, builder_spec, got_revision, infrabots_dir,
+def housekeeper_swarm(api, builder_cfg, got_revision, infrabots_dir,
                       extra_isolate_hashes):
   task = trigger_task(
       api,
       'housekeeper',
       api.properties['buildername'],
       api.properties['mastername'],
       api.properties['slavename'],
       api.properties['buildnumber'],
-      builder_spec,
+      builder_cfg,
       got_revision,
       infrabots_dir,
       idempotent=False,
       store_output=False,
       extra_isolate_hashes=extra_isolate_hashes)
   return api.swarming.collect_swarming_task(task)
 
 
-def recreate_skps_swarm(api, builder_spec, got_revision, infrabots_dir,
+def recreate_skps_swarm(api, builder_cfg, got_revision, infrabots_dir,
                         extra_isolate_hashes):
   task = trigger_task(
       api,
       'RecreateSKPs',
       api.properties['buildername'],
       api.properties['mastername'],
       api.properties['slavename'],
       api.properties['buildnumber'],
-      builder_spec,
+      builder_cfg,
       got_revision,
       infrabots_dir,
       idempotent=False,
       store_output=False,
       extra_isolate_hashes=extra_isolate_hashes)
   return api.swarming.collect_swarming_task(task)
 
 
 def infra_swarm(api, got_revision, infrabots_dir, extra_isolate_hashes):
-  # Fake the builder spec.
-  builder_spec = {
-    'builder_cfg': {
-      'role': 'Infra',
-      'is_trybot': api.builder_name_schema.IsTrybot(
-          api.properties['buildername'])
-    }
+  # Fake the builder cfg.
+  builder_cfg = {
+    'role': 'Infra',
+    'is_trybot': api.builder_name_schema.IsTrybot(
+         api.properties['buildername'])
   }
   task = trigger_task(
       api,
       'infra',
       api.properties['buildername'],
       api.properties['mastername'],
       api.properties['slavename'],
       api.properties['buildnumber'],
-      builder_spec,
+      builder_cfg,
       got_revision,
       infrabots_dir,
       idempotent=False,
       store_output=False,
       extra_isolate_hashes=extra_isolate_hashes)
   return api.swarming.collect_swarming_task(task)
 
 
-def compile_steps_swarm(api, builder_spec, got_revision, infrabots_dir,
-                        extra_isolate_hashes, cipd_packages):
+def compile_steps_swarm(api, builder_cfg, got_revision, infrabots_dir):
   builder_name = derive_compile_bot_name(api)
-  compile_builder_spec = builder_spec
-  if builder_name != api.properties['buildername']:
-    compile_builder_spec = api.vars.get_builder_spec(builder_name)
+  compile_builder_cfg = api.builder_name_schema.DictForBuilderName(builder_name)
 
-  extra_hashes = extra_isolate_hashes[:]
+  cipd_packages = []
+
+  # Android bots require a toolchain.
+  if 'Android' in api.properties['buildername']:
+    cipd_packages.append(cipd_pkg(api, infrabots_dir, 'android_sdk'))
 
   # Windows bots require a toolchain.
   if 'Win' in builder_name:
     version_file = infrabots_dir.join('assets', 'win_toolchain', 'VERSION')
     version = api.run.readfile(version_file,
                                name='read win_toolchain VERSION',
                                test_data='0').rstrip()
     version = 'version:%s' % version
     pkg = ('t', 'skia/bots/win_toolchain', version)
     cipd_packages.append(pkg)
 
     if 'Vulkan' in builder_name:
       cipd_packages.append(cipd_pkg(api, infrabots_dir, 'win_vulkan_sdk'))
 
   # Fake these properties for compile tasks so that they can be de-duped.
   master = 'client.skia.compile'
   slave = 'skiabot-dummy-compile-slave'
   buildnumber = 1
 
   task = trigger_task(
       api,
       'compile',
       builder_name,
       master,
       slave,
       buildnumber,
-      compile_builder_spec,
+      compile_builder_cfg,
       got_revision,
       infrabots_dir,
       idempotent=True,
       store_output=False,
-      extra_isolate_hashes=extra_hashes,
       cipd_packages=cipd_packages)
 
   # Wait for compile to finish, record the results hash.
   return api.swarming.collect_swarming_task_isolate_hash(task)
 
 
 def get_timeouts(builder_cfg):
   """Some builders require longer than the default timeouts.
 
   Returns tuple of (expiration, hard_timeout, io_timeout). If those values are
   none then default timeouts should be used.
   """
   expiration = None
   hard_timeout = None
   io_timeout = None
   if 'Valgrind' in builder_cfg.get('extra_config', ''):
     expiration = 2*24*60*60
     hard_timeout = 9*60*60
     io_timeout = 60*60
   return expiration, hard_timeout, io_timeout
 
 
-def perf_steps_trigger(api, builder_spec, got_revision, infrabots_dir,
+def gsutil_env(api, boto_file):
+  """Environment variables for gsutil."""
+  home_dir = os.path.expanduser('~')
+  if api.path._test_data.enabled:
+    home_dir = '[HOME]'
+
+  boto_path = None
+  if boto_file:
+    boto_path = api.path.join(home_dir, boto_file)
+  return {'AWS_CREDENTIAL_FILE': boto_path,
+          'BOTO_CONFIG': boto_path}
+
+
+def perf_steps_trigger(api, builder_cfg, got_revision, infrabots_dir,
                        extra_hashes, cipd_packages):
   """Trigger perf tests via Swarming."""
 
-  expiration, hard_timeout, io_timeout = get_timeouts(
-      builder_spec['builder_cfg'])
+  expiration, hard_timeout, io_timeout = get_timeouts(builder_cfg)
   return trigger_task(
       api,
       'perf',
       api.properties['buildername'],
       api.properties['mastername'],
       api.properties['slavename'],
       api.properties['buildnumber'],
-      builder_spec,
+      builder_cfg,
       got_revision,
       infrabots_dir,
       extra_isolate_hashes=extra_hashes,
       expiration=expiration,
       hard_timeout=hard_timeout,
       io_timeout=io_timeout,
       cipd_packages=cipd_packages)
 
 
-def perf_steps_collect(api, task, upload_perf_results, got_revision,
-                       is_trybot):
+def perf_steps_collect(api, task, got_revision, is_trybot):
   """Wait for perf steps to finish and upload results."""
   # Wait for nanobench to finish, download the results.
   api.run.rmtree(task.task_output_dir)
   api.swarming.collect_swarming_task(task)
 
   # Upload the results.
-  if upload_perf_results:
+  if api.vars.upload_perf_results:
     perf_data_dir = api.path['slave_build'].join(
         'perfdata', api.properties['buildername'], 'data')
     git_timestamp = api.git.get_timestamp(test_data='1408633190',
                                           infra_step=True)
     api.run.rmtree(perf_data_dir)
     api.file.makedirs('perf_dir', perf_data_dir, infra_step=True)
     src_results_file = task.task_output_dir.join(
         '0', 'perfdata', api.properties['buildername'], 'data',
         'nanobench_%s.json' % got_revision)
     dst_results_file = perf_data_dir.join(
         'nanobench_%s_%s.json' % (got_revision, git_timestamp))
     api.file.copy('perf_results', src_results_file, dst_results_file,
                   infra_step=True)
 
     gsutil_path = api.path['slave_build'].join(
         'skia', 'infra', 'bots', '.recipe_deps', 'depot_tools', 'third_party',
         'gsutil', 'gsutil')
     upload_args = [api.properties['buildername'], api.properties['buildnumber'],
                    perf_data_dir, got_revision, gsutil_path]
     if is_trybot:
       upload_args.append(api.properties['issue'])
     api.python(
-             'Upload perf results',
-             script=api.core.resource('upload_bench_results.py'),
-             args=upload_args,
-             cwd=api.path['checkout'],
-             infra_step=True)
+        'Upload perf results',
+        script=api.core.resource('upload_bench_results.py'),
+        args=upload_args,
+        cwd=api.path['checkout'],
+        infra_step=True)
 
 
-def test_steps_trigger(api, builder_spec, got_revision, infrabots_dir,
+def test_steps_trigger(api, builder_cfg, got_revision, infrabots_dir,
                        extra_hashes, cipd_packages):
   """Trigger DM via Swarming."""
-  expiration, hard_timeout, io_timeout = get_timeouts(
-      builder_spec['builder_cfg'])
+  expiration, hard_timeout, io_timeout = get_timeouts(builder_cfg)
   return trigger_task(
       api,
       'test',
       api.properties['buildername'],
       api.properties['mastername'],
       api.properties['slavename'],
       api.properties['buildnumber'],
-      builder_spec,
+      builder_cfg,
       got_revision,
       infrabots_dir,
       extra_isolate_hashes=extra_hashes,
       expiration=expiration,
       hard_timeout=hard_timeout,
       io_timeout=io_timeout,
       cipd_packages=cipd_packages)
 
 
-def test_steps_collect(api, task, upload_dm_results, got_revision, is_trybot,
-                       builder_cfg):
+def test_steps_collect(api, task, got_revision, is_trybot, builder_cfg):
   """Collect the test results from Swarming."""
   # Wait for tests to finish, download the results.
   api.run.rmtree(task.task_output_dir)
   api.swarming.collect_swarming_task(task)
 
   # Upload the results.
-  if upload_dm_results:
+  if api.vars.upload_dm_results:
     dm_dir = api.path['slave_build'].join('dm')
     dm_src = task.task_output_dir.join('0', 'dm')
     api.run.rmtree(dm_dir)
     api.file.copytree('dm_dir', dm_src, dm_dir, infra_step=True)
 
     # Upload them to Google Storage.
     api.python(
         'Upload DM Results',
         script=api.core.resource('upload_dm_results.py'),
         args=[
           dm_dir,
           got_revision,
           api.properties['buildername'],
           api.properties['buildnumber'],
           api.properties['issue'] if is_trybot else '',
           api.path['slave_build'].join('skia', 'common', 'py', 'utils'),
         ],
         cwd=api.path['checkout'],
-        env=api.vars.gsutil_env('chromium-skia-gm.boto'),
+        env=gsutil_env(api, 'chromium-skia-gm.boto'),
         infra_step=True)
 
   if builder_cfg['configuration']  == 'Coverage':
     upload_coverage_results(api, task, got_revision, is_trybot)
 
 
 def upload_coverage_results(api, task, got_revision, is_trybot):
   results_dir = task.task_output_dir.join('0')
   git_timestamp = api.git.get_timestamp(test_data='1408633190',
                                         infra_step=True)
@@ skipping 33 matching lines @@
   gsutil_path = api.depot_tools.gsutil_py_path
   upload_args = [api.properties['buildername'], api.properties['buildnumber'],
                  results_dir, got_revision, gsutil_path]
   if is_trybot:
     upload_args.append(api.properties['issue'])
   api.python(
       'upload nanobench coverage results',
       script=api.core.resource('upload_bench_results.py'),
       args=upload_args,
       cwd=api.path['checkout'],
-      env=api.vars.gsutil_env('chromium-skia-gm.boto'),
+      env=gsutil_env(api, 'chromium-skia-gm.boto'),
       infra_step=True)
 
   # Transform the coverage_by_line_${git_hash}.json file received from
   # swarming bot into a coverage_by_line_${git_hash}_${timestamp}.json file.
   src_lbl_file = results_dir.join('coverage_by_line_%s.json' % got_revision)
   dst_lbl_file_basename = 'coverage_by_line_%s_%s.json' % (
       got_revision, git_timestamp)
   dst_lbl_file = results_dir.join(dst_lbl_file_basename)
   api.file.copy('Line-by-line coverage JSON', src_lbl_file, dst_lbl_file,
                 infra_step=True)
@@ skipping 48 matching lines @@
   api.swarming.setup(
       infrabots_dir.join('tools', 'luci-go'),
       swarming_rev='')
 
   # Run gsutil.py to ensure that it's installed.
   api.gsutil(['help'])
 
   extra_hashes = []
 
   # Get ready to compile.
-  compile_cipd_deps = []
-  extra_compile_hashes = []
-
   infrabots_dir = api.path['checkout'].join('infra', 'bots')
   if 'Infra' in api.properties['buildername']:
     return infra_swarm(api, got_revision, infrabots_dir, extra_hashes)
 
-  builder_spec = api.vars.get_builder_spec(api.properties['buildername'])
-  builder_cfg = builder_spec['builder_cfg']
+  builder_cfg = api.builder_name_schema.DictForBuilderName(
+      api.properties['buildername'])
 
   if 'RecreateSKPs' in api.properties['buildername']:
-    recreate_skps_swarm(api, builder_spec, got_revision, infrabots_dir,
+    recreate_skps_swarm(api, builder_cfg, got_revision, infrabots_dir,
                         extra_hashes)
     return
-  if 'Android' in api.properties['buildername']:
-    compile_cipd_deps.append(cipd_pkg(api, infrabots_dir, 'android_sdk'))
 
   # Compile.
-  do_compile_steps = builder_spec.get('do_compile_steps', True)
+  do_compile_steps = True
+  if 'Coverage' in api.properties['buildername']:
+    do_compile_steps = False
   if do_compile_steps:
     extra_hashes.append(compile_steps_swarm(
-        api, builder_spec, got_revision, infrabots_dir, extra_compile_hashes,
-        cipd_packages=compile_cipd_deps))
+        api, builder_cfg, got_revision, infrabots_dir))
 
   if builder_cfg['role'] == 'Housekeeper':
-    housekeeper_swarm(api, builder_spec, got_revision, infrabots_dir,
+    housekeeper_swarm(api, builder_cfg, got_revision, infrabots_dir,
                       extra_hashes)
     return
 
   # Get ready to test/perf.
 
   # CIPD packages needed by test/perf.
   cipd_packages = []
 
-  do_test_steps = builder_spec['do_test_steps']
-  do_perf_steps = builder_spec['do_perf_steps']
+  do_test_steps = (
+      builder_cfg['role'] == api.builder_name_schema.BUILDER_ROLE_TEST)
+  do_perf_steps = (
+      builder_cfg['role'] == api.builder_name_schema.BUILDER_ROLE_PERF)
 
   if not (do_test_steps or do_perf_steps):
     return
 
   # SKPs, SkImages.
   cipd_packages.append(cipd_pkg(api, infrabots_dir, 'skp'))
   cipd_packages.append(cipd_pkg(api, infrabots_dir, 'skimage'))
 
   # Trigger test and perf tasks.
   test_task = None
   perf_task = None
   if do_test_steps:
-    test_task = test_steps_trigger(api, builder_spec, got_revision,
+    test_task = test_steps_trigger(api, builder_cfg, got_revision,
                                    infrabots_dir, extra_hashes, cipd_packages)
   if do_perf_steps:
-    perf_task = perf_steps_trigger(api, builder_spec, got_revision,
+    perf_task = perf_steps_trigger(api, builder_cfg, got_revision,
                                    infrabots_dir, extra_hashes, cipd_packages)
   is_trybot = builder_cfg['is_trybot']
+
+  # Wait for results, then upload them if necessary.
+
   if test_task:
-    test_steps_collect(api, test_task, builder_spec['upload_dm_results'],
+    test_steps_collect(api, test_task,
                        got_revision, is_trybot, builder_cfg)
+
   if perf_task:
-    perf_steps_collect(api, perf_task, builder_spec['upload_perf_results'],
+    perf_steps_collect(api, perf_task,
                        got_revision, is_trybot)
 
 
 def test_for_bot(api, builder, mastername, slavename, testname=None):
   """Generate a test for the given bot."""
   testname = testname or builder
   test = (
     api.test(testname) +
     api.properties(buildername=builder,
                    mastername=mastername,
@@ skipping 38 matching lines @@
   test += api.path.exists(*paths)
 
   return test
 
 
 def GenTests(api):
   for mastername, slaves in TEST_BUILDERS.iteritems():
     for slavename, builders_by_slave in slaves.iteritems():
       for builder in builders_by_slave:
         yield test_for_bot(api, builder, mastername, slavename)