CT Perf recipe to run benchmarks on the top 1k sites using swarming

scripts/slave/recipe_modules/ct_swarming/api.py

Index: scripts/slave/recipe_modules/ct_swarming/api.py
diff --git a/scripts/slave/recipe_modules/ct_swarming/api.py b/scripts/slave/recipe_modules/ct_swarming/api.py
new file mode 100644
index 0000000000000000000000000000000000000000..82b94db05e20399d854a083b61e5d390c4187aa2
--- /dev/null
+++ b/scripts/slave/recipe_modules/ct_swarming/api.py
@@ -0,0 +1,161 @@
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+from recipe_engine import recipe_api
+
+
+CT_GS_BUCKET = 'cluster-telemetry'
+
+
+class CTSwarmingApi(recipe_api.RecipeApi):
+  """Provides steps to run CT tasks on swarming bots."""
+
+  def __init__(self, **kwargs):
+    super(CTSwarmingApi, self).__init__(**kwargs)
+    # Path to the chromium src directory. Will be populated when
+    # checkout_dependencies is called.
+    self.chromium_src_dir = None

[M-A Ruel, 2015/11/12 17:33:41]

If you want to keep this, I'd prefer

@property
def chromium_dir(self):
  return self.m.path['checkout']

but I don't see this much useful.

[rmistry, 2015/11/13 14:59:58]

Agreed. Done.

+    # Path to where artifacts should be downloaded from Google Storage. Will be
+    # populated when checkout_dependencies is called.
+    self.downloads_dir = None

[M-A Ruel, 2015/11/12 17:33:41]

Make this a property, this reduces aliasing of the root variables.

[rmistry, 2015/11/13 14:59:58]

Done.

+    # Path where swarming artifacts (isolate file, json output, etc) will be
+    # stored. Will be populated when checkout dependencies is caled.
+    self.swarming_temp_dir = None
+    # Keep track of all dirs created during recipe execution. Will be cleaned
+    # up by the cleanup step.
+    self._created_dirs = []
+    # Collection of all swarming tasks triggered by this recipe.
+    self._swarming_tasks = []
+
+  def checkout_dependencies(self):
+    """Checks out all repositories required for CT to run on swarming bots."""
+    # Checkout chromium and swarming.
+    self.m.chromium.set_config('chromium')
+    self.m.gclient.set_config('chromium')
+    self.m.bot_update.ensure_checkout(force=True)
+    self.m.swarming_client.checkout()
+    # Set the paths required by this recipe module.
+    self.chromium_src_dir = self.m.path['checkout']
+    self.downloads_dir = self.chromium_src_dir.join('content', 'test', 'ct')
+    self.swarming_temp_dir = self.m.path['tmp_base'].join('swarming_temp_dir')
+    self._makedirs(self.swarming_temp_dir)
+    # Ensure swarming_client is compatible with what recipes expect.
+    self.m.swarming.check_client_version()
+
+  def _makedirs(self, d):
+    """Creates the specified dir and registers it as a recipe created dir.
+
+    All directories created by this method will be deleted when cleanup() is
+    called.
+    """
+    self.m.file.makedirs('makedirs %s' % d, d)
+    self._created_dirs.append(d)

[M-A Ruel, 2015/11/12 17:33:41]

IIUC, 'tmp_base' is deleted automatically, so no need to keep track of these.

In particular what you want is file.mkdtemp():
https://github.com/luci/recipes-py/blob/master/recipe_modules/path/api.py#L189

[rmistry, 2015/11/13 14:59:58]

Yes looks like tmp_base is deleted automatically.

+
+  def download_CT_binary(self, ct_binary_name):
+    """Downloads the specified CT binary from GS into the downloads_dir."""
+    self.m.gsutil.download(
+        bucket=CT_GS_BUCKET,
+        source='swarming/binaries/%s' % ct_binary_name,
+        dest=self.downloads_dir.join(ct_binary_name))
+
+  def download_page_artifacts(self, page_type, slave_num, local_slave_dir):
+    """Downloads all the artifacts needed to run benchmarks on a page.
+
+    Args:
+      page_type: str. The CT page type. Eg: 1k, 10k.
+      slave_num: int. The number of the slave used to determine which GS
+                 directory to download from. Eg: for the top 1k, slave1 will
+                 contain webpages 1-10, slave2 will contain 11-20.
+      local_slave_dir: path obj. The directory artifacts should be downloaded
+                       to.
+    """
+    # Download page sets.
+    page_sets_dir = local_slave_dir.join('page_sets')
+    self._makedirs(page_sets_dir)
+    self.m.gsutil.download(
+        bucket=CT_GS_BUCKET,
+        source='swarming/page_sets/%s/slave%s/*' % (page_type, slave_num),
+        dest=page_sets_dir)
+
+    # Download archives.
+    wpr_dir = page_sets_dir.join('data')
+    self._makedirs(wpr_dir)
+    self.m.gsutil.download(
+        bucket=CT_GS_BUCKET,
+        source='swarming/webpage_archives/%s/slave%s/*' % (page_type,
+                                                           slave_num),
+        dest=wpr_dir)
+
+  def create_isolated_gen_json(self, isolate_path, base_dir, os_type,
+                               slave_num):
+    """Creates an isolated.gen.json file.
+
+    Args:
+      isolate_path: path obj. Path to the isolate file.
+      base_dir: path obj. Dir that is the base of all paths in the isolate file.
+      os_type: str. The OS type to use when archiving the isolate file.
+               Eg: linux.
+      slave_num: int. The slave we want to create isolated.gen.json file for.
+
+    Returns:
+      Path to the isolated.gen.json file.
+    """
+    isolated_path = self.swarming_temp_dir.join(
+        'ct-task-%s.isolated' % slave_num)
+    isolate_args = [
+      '--isolate', isolate_path,
+      '--isolated', isolated_path,
+      '--config-variable', 'OS', os_type,
+    ]
+    isolated_gen_dict = {
+      'version': 1,
+      'dir': base_dir,
+      'args': isolate_args,
+    }
+    isolated_gen_json = self.swarming_temp_dir.join(
+        'slave%s.isolated.gen.json' % slave_num)
+    with open(str(isolated_gen_json), 'w') as fout:
+      fout.write(self.m.json.dumps(isolated_gen_dict, indent=4))
+    return isolated_gen_json
+
+  def batcharchive(self, num_slaves):
+    """Calls batcharchive on the specified isolated.gen.json files.
+
+    Args:
+      num_slaves: int. The number of slaves we will batcharchive
+                  isolated.gen.json files for.
+    """
+    self.m.isolate.isolate_tests(
+        build_dir=self.swarming_temp_dir,
+        targets=['slave%s' % num for num in range(1, num_slaves+1)])
+
+  def trigger_swarming_tasks(self, swarm_hashes, task_name_prefix, dimensions):
+    """Triggers swarming tasks using swarm hashes.
+
+    Args:
+      swarm_hashes: list of str. List of swarm hashes from the isolate server.
+      task_name_prefix: The prefix to use when creating task_name.
+      dimensions: dict of str to str. The dimensions to run the task on.
+                  Eg: {'os': 'Ubuntu', 'gpu': '10de'}
+    """
+    task_num = 0
+    for swarm_hash in swarm_hashes:
+      task_num += 1
+      swarming_task = self.m.swarming.task(
+          title='%s-%s' % (task_name_prefix, task_num),
+          isolated_hash=swarm_hash)
+      swarming_task.dimensions = dimensions
+      swarming_task.priority = 90
+      self._swarming_tasks.append(swarming_task)
+    self.m.swarming.trigger(self._swarming_tasks)
+
+  def collect_swarming_tasks(self):
+    """Collects all swarming tasks triggered by this recipe."""
+    self.m.swarming.collect(self._swarming_tasks)
+
+  def cleanup(self):
+    """Cleans up all directories created by this recipe module."""
+    for d in self._created_dirs:

[M-A Ruel, 2015/11/12 17:33:41]

This shouldn't be needed in practice, if it is, we should refactor until it's
not needed.

[rmistry, 2015/11/13 14:59:58]

Yes I do not think this is needed, I thought it was because I saw directories
pile up in my local testing when I had commented out the part that cleaned the
local checkout. Removed.

+      self.m.file.rmtree('Removing dir %s' % d, d)