CT Perf recipe to run benchmarks on the top 1k sites using swarming

scripts/slave/recipe_modules/ct_swarming/api.py

Index: scripts/slave/recipe_modules/ct_swarming/api.py
diff --git a/scripts/slave/recipe_modules/ct_swarming/api.py b/scripts/slave/recipe_modules/ct_swarming/api.py
new file mode 100644
index 0000000000000000000000000000000000000000..1df11d6abd97288aa1e5abb0f4f0ac82a5cf8e53
--- /dev/null
+++ b/scripts/slave/recipe_modules/ct_swarming/api.py
@@ -0,0 +1,171 @@
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+from recipe_engine import recipe_api
+
+
+CT_GS_BUCKET = 'cluster-telemetry'
+
+
+class CTSwarmingApi(recipe_api.RecipeApi):
+  """Provides steps to run CT tasks on swarming bots."""
+
+  def __init__(self, **kwargs):
+    super(CTSwarmingApi, self).__init__(**kwargs)
+    # Path to where artifacts should be downloaded from Google Storage. Will be
+    # populated when checkout_dependencies is called.
+    self._downloads_dir = None
+    # Path where swarming artifacts (isolate file, json output, etc) will be
+    # stored. Will be populated when checkout dependencies is called.
+    self._swarming_temp_dir = None
+    # Directory where the outputs of the swarming tasks will be stored.
+    self._tasks_output_dir = None
+    # Collection of all swarming tasks triggered by this recipe.
+    self._swarming_tasks = []
+
+  @property
+  def downloads_dir(self):
+    return self._downloads_dir

[M-A Ruel, 2015/11/20 18:14:01]

I'd prefer
return self.m.path['checkout'].join('content', 'test', 'ct')

so you can delete lines 19 and 44 and migrate comment at line 17-18 here as a
docstring.

Same thing for _swarming_temp_dir and tasks_output_dir.

[rmistry, 2015/11/23 15:13:48]

Done.

+
+  @property
+  def tasks_output_dir(self):
+    return self._tasks_output_dir
+
+  def checkout_dependencies(self):
+    """Checks out all repositories required for CT to run on swarming bots."""
+    # Checkout chromium and swarming.
+    self.m.chromium.set_config('chromium')
+    self.m.gclient.set_config('chromium')
+    self.m.bot_update.ensure_checkout(force=True)
+    self.m.swarming_client.checkout()
+    # Set the paths required by this recipe module.
+    self._downloads_dir = self.m.path['checkout'].join('content', 'test', 'ct')
+    self._swarming_temp_dir = self.m.path['tmp_base'].join('swarming_temp_dir')
+    self.m.file.makedirs('Create swarming tmp dir', self._swarming_temp_dir)
+    # The directory to store swarming task outputs in.
+    self._tasks_output_dir = self._swarming_temp_dir.join('outputs')
+    # Ensure swarming_client is compatible with what recipes expect.
+    self.m.swarming.check_client_version()
+
+  def download_CT_binary(self, ct_binary_name):
+    """Downloads the specified CT binary from GS into the downloads_dir."""
+    binary_dest = self._downloads_dir.join(ct_binary_name)
+    self.m.gsutil.download(
+        name="download %s" % ct_binary_name,
+        bucket=CT_GS_BUCKET,
+        source='swarming/binaries/%s' % ct_binary_name,
+        dest=binary_dest)
+    # Set executable bit on the binary.
+    self.m.python.inline(
+        name='Set executable bit on %s' % ct_binary_name,
+        program='''
+import os
+import stat
+
+os.chmod('%s', os.stat('%s').st_mode | stat.S_IEXEC)
+''' % (str(binary_dest), str(binary_dest))
+    )
+
+  def download_page_artifacts(self, page_type, slave_num):
+    """Downloads all the artifacts needed to run benchmarks on a page.
+
+    The artifacts are downloaded into subdirectories in the downloads_dir.
+
+    Args:
+      page_type: str. The CT page type. Eg: 1k, 10k.
+      slave_num: int. The number of the slave used to determine which GS
+                 directory to download from. Eg: for the top 1k, slave1 will
+                 contain webpages 1-10, slave2 will contain 11-20.
+    """
+    # Download page sets.
+    page_sets_dir = self._downloads_dir.join('slave%s' % slave_num, 'page_sets')
+    self.m.file.makedirs('Create page_sets dir', page_sets_dir)
+    self.m.gsutil.download(
+        bucket=CT_GS_BUCKET,
+        source='swarming/page_sets/%s/slave%s/*' % (page_type, slave_num),
+        dest=page_sets_dir)
+
+    # Download archives.
+    wpr_dir = page_sets_dir.join('data')
+    self.m.file.makedirs('Create WPR dir', wpr_dir)
+    self.m.gsutil.download(
+        bucket=CT_GS_BUCKET,
+        source='swarming/webpage_archives/%s/slave%s/*' % (page_type,
+                                                           slave_num),
+        dest=wpr_dir)
+
+  def create_isolated_gen_json(self, isolate_path, base_dir, os_type,
+                               slave_num, extra_variables):
+    """Creates an isolated.gen.json file.
+
+    Args:
+      isolate_path: path obj. Path to the isolate file.
+      base_dir: path obj. Dir that is the base of all paths in the isolate file.
+      os_type: str. The OS type to use when archiving the isolate file.
+               Eg: linux.
+      slave_num: int. The slave we want to create isolated.gen.json file for.
+      extra_variables: dict of str to str. The extra vars to pass to isolate.
+                      Eg: {'SLAVE_NUM': '1', 'MASTER': 'ChromiumPerfFYI'}
+
+    Returns:
+      Path to the isolated.gen.json file.
+    """
+    isolated_path = self._swarming_temp_dir.join(
+        'ct-task-%s.isolated' % slave_num)
+    isolate_args = [
+      '--isolate', isolate_path,
+      '--isolated', isolated_path,
+      '--config-variable', 'OS', os_type,
+    ]
+    for k, v in extra_variables.iteritems():
+      isolate_args.extend(['--extra-variable', k, v])
+    isolated_gen_dict = {
+      'version': 1,
+      'dir': base_dir,
+      'args': isolate_args,
+    }
+    isolated_gen_json = self._swarming_temp_dir.join(
+        'slave%s.isolated.gen.json' % slave_num)
+    self.m.file.write(
+        'Write slave%s.isolated.gen.json' % slave_num,
+        isolated_gen_json,
+        self.m.json.dumps(isolated_gen_dict, indent=4),
+    )
+
+  def batcharchive(self, num_slaves):
+    """Calls batcharchive on the specified isolated.gen.json files.
+
+    Args:
+      num_slaves: int. The number of slaves we will batcharchive
+                  isolated.gen.json files for.
+    """
+    self.m.isolate.isolate_tests(
+        build_dir=self._swarming_temp_dir,
+        targets=['slave%s' % num for num in range(1, num_slaves+1)])
+
+  def trigger_swarming_tasks(self, swarm_hashes, task_name_prefix, dimensions):
+    """Triggers swarming tasks using swarm hashes.
+
+    Args:
+      swarm_hashes: list of str. List of swarm hashes from the isolate server.
+      task_name_prefix: The prefix to use when creating task_name.
+      dimensions: dict of str to str. The dimensions to run the task on.
+                  Eg: {'os': 'Ubuntu', 'gpu': '10de'}
+    """
+    task_num = 0
+    for swarm_hash in swarm_hashes:
+      task_num += 1
+      swarming_task = self.m.swarming.task(
+          title='%s-%s' % (task_name_prefix, task_num),
+          isolated_hash=swarm_hash,
+          task_output_dir=self._tasks_output_dir.join('slave%s' % task_num))
+      swarming_task.dimensions = dimensions
+      swarming_task.priority = 90
+      self._swarming_tasks.append(swarming_task)
+    self.m.swarming.trigger(self._swarming_tasks)

[M-A Ruel, 2015/11/20 18:14:01]

You should return the tasks and remove self._swarming_tasks. It is closer to
what recipes_modules/swarming/api.py does (yet still different, but closer).

[rmistry, 2015/11/23 15:13:48]

Done.

+
+  def collect_swarming_tasks(self):

[M-A Ruel, 2015/11/20 18:14:01]

Accept the tasks as a parameter.

[rmistry, 2015/11/23 15:13:48]

Done.

+    """Collects all swarming tasks triggered by this recipe."""
+    return self.m.swarming.collect(self._swarming_tasks)