Re-organize Skia recipes

infra/bots/recipes/swarm_trigger.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 
 # Recipe module for Skia Swarming trigger.
 
 
 import json
 
 
 DEPS = [
+  'core',
   'build/file',
   'build/gsutil',
   'depot_tools/depot_tools',
   'depot_tools/git',
   'depot_tools/tryserver',
   'recipe_engine/json',
   'recipe_engine/path',
   'recipe_engine/properties',
   'recipe_engine/python',
   'recipe_engine/raw_io',
   'recipe_engine/step',
   'recipe_engine/time',
-  'skia',
-  'skia_swarming',
+  'run',
+  'swarming',
+  'vars',
 ]
 
 
 TEST_BUILDERS = {
   'client.skia': {
     'skiabot-linux-swarm-000': [
       'Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind',
       'Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot',
       'Build-Mac-Clang-x86_64-Release',
       'Build-Ubuntu-GCC-Arm64-Debug-Android_Vulkan',
@@ skipping 152 matching lines @@
   isolate_blacklist = ['.git', 'out', '*.pyc', '.recipe_deps']
   isolate_vars = {
     'WORKDIR': api.path['slave_build'],
   }
 
   isolate_file = '%s_skia.isolate' % task_name
   if 'Coverage' == builder_cfg.get('configuration'):
     isolate_file = 'coverage_skia.isolate'
   if 'RecreateSKPs' in builder:
     isolate_file = 'compile_skia.isolate'
-  return api.skia_swarming.isolate_and_trigger_task(
+  return api.swarming.isolate_and_trigger_task(
       infrabots_dir.join(isolate_file),
       isolate_base_dir,
       '%s_skia' % task_name,
       isolate_vars,
       dimensions,
       isolate_blacklist=isolate_blacklist,
       extra_isolate_hashes=extra_isolate_hashes,
       idempotent=idempotent,
       store_output=store_output,
       extra_args=extra_args,
@@ skipping 14 matching lines @@
       step_test_data=lambda: api.raw_io.test_api.stream_output('abc123\n'),
   ).stdout.rstrip()
   cmd = ['python', '-c', '"print \'%s\'"' % got_revision]
   res = api.step('got_revision', cmd=cmd)
   res.presentation.properties['got_revision'] = got_revision
   api.path['checkout'] = checkout_path
 
   # Write a fake .gclient file if none exists. This is required by .isolates.
   dot_gclient = api.path['slave_build'].join('.gclient')
   if not api.path.exists(dot_gclient):
-    api.skia._writefile(dot_gclient, '')
+    api.run.writefile(dot_gclient, '')
 
   fix_filemodes(api, api.path['checkout'])
   return got_revision
 
 
 def housekeeper_swarm(api, builder_spec, got_revision, infrabots_dir,
                       extra_isolate_hashes):
   task = trigger_task(
       api,
       'housekeeper',
       api.properties['buildername'],
       api.properties['mastername'],
       api.properties['slavename'],
       api.properties['buildnumber'],
       builder_spec,
       got_revision,
       infrabots_dir,
       idempotent=False,
       store_output=False,
       extra_isolate_hashes=extra_isolate_hashes)
-  return api.skia_swarming.collect_swarming_task(task)
+  return api.swarming.collect_swarming_task(task)
 
 
 def recreate_skps_swarm(api, builder_spec, got_revision, infrabots_dir,
                         extra_isolate_hashes):
   task = trigger_task(
       api,
       'RecreateSKPs',
       api.properties['buildername'],
       api.properties['mastername'],
       api.properties['slavename'],
       api.properties['buildnumber'],
       builder_spec,
       got_revision,
       infrabots_dir,
       idempotent=False,
       store_output=False,
       extra_isolate_hashes=extra_isolate_hashes)
-  return api.skia_swarming.collect_swarming_task(task)
+  return api.swarming.collect_swarming_task(task)
 
 
 def infra_swarm(api, got_revision, infrabots_dir, extra_isolate_hashes):
   # Fake the builder spec.
   builder_spec = {
     'builder_cfg': {
       'role': 'Infra',
       'is_trybot': api.properties['buildername'].endswith('-Trybot'),
     }
   }
   task = trigger_task(
       api,
       'infra',
       api.properties['buildername'],
       api.properties['mastername'],
       api.properties['slavename'],
       api.properties['buildnumber'],
       builder_spec,
       got_revision,
       infrabots_dir,
       idempotent=False,
       store_output=False,
       extra_isolate_hashes=extra_isolate_hashes)
-  return api.skia_swarming.collect_swarming_task(task)
+  return api.swarming.collect_swarming_task(task)
 
 
 def compile_steps_swarm(api, builder_spec, got_revision, infrabots_dir,
                         extra_isolate_hashes, cipd_packages):
   builder_name = derive_compile_bot_name(api.properties['buildername'],
                                          builder_spec)
   compile_builder_spec = builder_spec
   if builder_name != api.properties['buildername']:
-    compile_builder_spec = api.skia.get_builder_spec(
+    compile_builder_spec = api.core.get_builder_spec(
         api.path['slave_build'].join('skia'), builder_name)
 
   extra_hashes = extra_isolate_hashes[:]
 
   # Windows bots require a toolchain.
   if 'Win' in builder_name:
     version_file = infrabots_dir.join('assets', 'win_toolchain', 'VERSION')
-    version = api.skia._readfile(version_file,
+    version = api.run.readfile(version_file,
                                  name='read win_toolchain VERSION',
                                  test_data='0').rstrip()
     version = 'version:%s' % version
     pkg = ('t', 'skia/bots/win_toolchain', version)
     cipd_packages.append(pkg)
 
     if 'Vulkan' in builder_name:
       cipd_packages.append(cipd_pkg(api, infrabots_dir, 'win_vulkan_sdk'))
 
   # Fake these properties for compile tasks so that they can be de-duped.
@@ skipping 10 matching lines @@
       buildnumber,
       compile_builder_spec,
       got_revision,
       infrabots_dir,
       idempotent=True,
       store_output=False,
       extra_isolate_hashes=extra_hashes,
       cipd_packages=cipd_packages)
 
   # Wait for compile to finish, record the results hash.
-  return api.skia_swarming.collect_swarming_task_isolate_hash(task)
+  return api.swarming.collect_swarming_task_isolate_hash(task)
 
 
 def get_timeouts(builder_cfg):
   """Some builders require longer than the default timeouts.
 
   Returns tuple of (expiration, hard_timeout, io_timeout). If those values are
   none then default timeouts should be used.
   """
   expiration = None
   hard_timeout = None
@@ skipping 25 matching lines @@
       expiration=expiration,
       hard_timeout=hard_timeout,
       io_timeout=io_timeout,
       cipd_packages=cipd_packages)
 
 
 def perf_steps_collect(api, task, upload_perf_results, got_revision,
                        is_trybot):
   """Wait for perf steps to finish and upload results."""
   # Wait for nanobench to finish, download the results.
-  api.skia.rmtree(task.task_output_dir)
-  api.skia_swarming.collect_swarming_task(task)
+  api.run.rmtree(task.task_output_dir)
+  api.swarming.collect_swarming_task(task)
 
   # Upload the results.
   if upload_perf_results:
     perf_data_dir = api.path['slave_build'].join(
         'perfdata', api.properties['buildername'], 'data')
     git_timestamp = api.git.get_timestamp(test_data='1408633190',
                                           infra_step=True)
-    api.skia.rmtree(perf_data_dir)
+    api.run.rmtree(perf_data_dir)
     api.file.makedirs('perf_dir', perf_data_dir, infra_step=True)
     src_results_file = task.task_output_dir.join(
         '0', 'perfdata', api.properties['buildername'], 'data',
         'nanobench_%s.json' % got_revision)
     dst_results_file = perf_data_dir.join(
         'nanobench_%s_%s.json' % (got_revision, git_timestamp))
     api.file.copy('perf_results', src_results_file, dst_results_file,
                   infra_step=True)
 
     gsutil_path = api.path['slave_build'].join(
         'skia', 'infra', 'bots', '.recipe_deps', 'depot_tools', 'third_party',
         'gsutil', 'gsutil')
     upload_args = [api.properties['buildername'], api.properties['buildnumber'],
                    perf_data_dir, got_revision, gsutil_path]
     if is_trybot:
       upload_args.append(api.properties['issue'])
     api.python(
              'Upload perf results',
-             script=api.skia.resource('upload_bench_results.py'),
+             script=api.core.resource('upload_bench_results.py'),
              args=upload_args,
              cwd=api.path['checkout'],
              infra_step=True)
 
 
 def test_steps_trigger(api, builder_spec, got_revision, infrabots_dir,
                        extra_hashes, cipd_packages):
   """Trigger DM via Swarming."""
   expiration, hard_timeout, io_timeout = get_timeouts(
       builder_spec['builder_cfg'])
@@ skipping 11 matching lines @@
       expiration=expiration,
       hard_timeout=hard_timeout,
       io_timeout=io_timeout,
       cipd_packages=cipd_packages)
 
 
 def test_steps_collect(api, task, upload_dm_results, got_revision, is_trybot,
                        builder_cfg):
   """Collect the test results from Swarming."""
   # Wait for tests to finish, download the results.
-  api.skia.rmtree(task.task_output_dir)
-  api.skia_swarming.collect_swarming_task(task)
+  api.run.rmtree(task.task_output_dir)
+  api.swarming.collect_swarming_task(task)
 
   # Upload the results.
   if upload_dm_results:
     dm_dir = api.path['slave_build'].join('dm')
     dm_src = task.task_output_dir.join('0', 'dm')
-    api.skia.rmtree(dm_dir)
+    api.run.rmtree(dm_dir)
     api.file.copytree('dm_dir', dm_src, dm_dir, infra_step=True)
 
     # Upload them to Google Storage.
     api.python(
         'Upload DM Results',
-        script=api.skia.resource('upload_dm_results.py'),
+        script=api.core.resource('upload_dm_results.py'),
         args=[
           dm_dir,
           got_revision,
           api.properties['buildername'],
           api.properties['buildnumber'],
           api.properties['issue'] if is_trybot else '',
           api.path['slave_build'].join('skia', 'common', 'py', 'utils'),
         ],
         cwd=api.path['checkout'],
-        env=api.skia.gsutil_env('chromium-skia-gm.boto'),
+        env=api.vars.gsutil_env('chromium-skia-gm.boto'),
         infra_step=True)
 
   if builder_cfg['configuration']  == 'Coverage':
     upload_coverage_results(api, task, got_revision, is_trybot)
 
 
 def upload_coverage_results(api, task, got_revision, is_trybot):
   results_dir = task.task_output_dir.join('0')
   git_timestamp = api.git.get_timestamp(test_data='1408633190',
                                         infra_step=True)
@@ skipping 30 matching lines @@
   api.file.remove('old nanobench JSON', src_nano_file)
 
   # Upload nanobench JSON data.
   gsutil_path = api.depot_tools.gsutil_py_path
   upload_args = [api.properties['buildername'], api.properties['buildnumber'],
                  results_dir, got_revision, gsutil_path]
   if is_trybot:
     upload_args.append(api.properties['issue'])
   api.python(
       'upload nanobench coverage results',
-      script=api.skia.resource('upload_bench_results.py'),
+      script=api.core.resource('upload_bench_results.py'),
       args=upload_args,
       cwd=api.path['checkout'],
-      env=api.skia.gsutil_env('chromium-skia-gm.boto'),
+      env=api.vars.gsutil_env('chromium-skia-gm.boto'),
       infra_step=True)
 
   # Transform the coverage_by_line_${git_hash}.json file received from
   # swarming bot into a coverage_by_line_${git_hash}_${timestamp}.json file.
   src_lbl_file = results_dir.join('coverage_by_line_%s.json' % got_revision)
   dst_lbl_file_basename = 'coverage_by_line_%s_%s.json' % (
       got_revision, git_timestamp)
   dst_lbl_file = results_dir.join(dst_lbl_file_basename)
   api.file.copy('Line-by-line coverage JSON', src_lbl_file, dst_lbl_file,
                 infra_step=True)
   api.file.remove('old line-by-line coverage JSON', src_lbl_file)
 
   # Upload line-by-line coverage data.
   api.gsutil.upload(
       name='upload line-by-line coverage data',
       source=dst_lbl_file,
       bucket='skia-infra',
       dest='/'.join(('coverage-json-v1', gs_json_path,
                      dst_lbl_file_basename)),
       env={'AWS_CREDENTIAL_FILE': None, 'BOTO_CONFIG': None},
   )
 
 
 def cipd_pkg(api, infrabots_dir, asset_name):
   """Find and return the CIPD package info for the given asset."""
   version_file = infrabots_dir.join('assets', asset_name, 'VERSION')
-  version = api.skia._readfile(version_file,
+  version = api.run.readfile(version_file,
                                name='read %s VERSION' % asset_name,
                                test_data='0').rstrip()
   version = 'version:%s' % version
   return (asset_name, 'skia/bots/%s' % asset_name, version)
 
 
 def print_properties(api):
   """Dump out all properties for debugging purposes."""
   props = {}
   for k, v in api.properties.iteritems():
@@ skipping 11 matching lines @@
 ''',
       args=[api.json.input(props)])
 
 
 def RunSteps(api):
   # TODO(borenet): Remove this once SwarmBucket is working.
   print_properties(api)
 
   got_revision = checkout_steps(api)
   infrabots_dir = api.path['checkout'].join('infra', 'bots')
-  api.skia_swarming.setup(
+  api.swarming.setup(
       infrabots_dir.join('tools', 'luci-go'),
       swarming_rev='')
 
   # Run gsutil.py to ensure that it's installed.
   api.gsutil(['help'])
 
   extra_hashes = []
 
   # Get ready to compile.
   compile_cipd_deps = []
   extra_compile_hashes = []
 
   infrabots_dir = api.path['checkout'].join('infra', 'bots')
   if 'Infra' in api.properties['buildername']:
     return infra_swarm(api, got_revision, infrabots_dir, extra_hashes)
 
-  builder_spec = api.skia.get_builder_spec(api.path['checkout'],
+  builder_spec = api.core.get_builder_spec(api.path['checkout'],
                                            api.properties['buildername'])
   builder_cfg = builder_spec['builder_cfg']
 
   if 'RecreateSKPs' in api.properties['buildername']:
     recreate_skps_swarm(api, builder_spec, got_revision, infrabots_dir,
                         extra_hashes)
     return
-
-  # Android bots require an SDK.
   if 'Android' in api.properties['buildername']:
     compile_cipd_deps.append(cipd_pkg(api, infrabots_dir, 'android_sdk'))
 
   # Compile.
   do_compile_steps = builder_spec.get('do_compile_steps', True)
   if do_compile_steps:
     extra_hashes.append(compile_steps_swarm(
         api, builder_spec, got_revision, infrabots_dir, extra_compile_hashes,
         cipd_packages=compile_cipd_deps))
 
@@ skipping 83 matching lines @@
   test += api.path.exists(*paths)
 
   return test
 
 
 def GenTests(api):
   for mastername, slaves in TEST_BUILDERS.iteritems():
     for slavename, builders_by_slave in slaves.iteritems():
       for builder in builders_by_slave:
         yield test_for_bot(api, builder, mastername, slavename)