CT Perf recipe to run benchmarks on the top 1k sites using swarming

scripts/slave/recipe_modules/ct_swarming/api.py

Index: scripts/slave/recipe_modules/ct_swarming/api.py
diff --git a/scripts/slave/recipe_modules/ct_swarming/api.py b/scripts/slave/recipe_modules/ct_swarming/api.py
new file mode 100644
index 0000000000000000000000000000000000000000..0c47159870b318ee3dfc65dc3fde54eef5a23ecb
--- /dev/null
+++ b/scripts/slave/recipe_modules/ct_swarming/api.py
@@ -0,0 +1,177 @@
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import urllib

[M-A Ruel, 2015/11/11 23:53:36]

Remove

[rmistry, 2015/11/12 14:39:17]

Done.

+
+from recipe_engine import recipe_api
+
+
+CT_GS_BUCKET = 'cluster-telemetry'
+
+
+class CTSwarmingApi(recipe_api.RecipeApi):
+  """Provides steps to run CT tasks on swarming bots."""
+
+  def __init__(self, **kwargs):
+    super(CTSwarmingApi, self).__init__(**kwargs)
+    # Path to the chromium src directory. Will be populated when
+    # checkout_dependencies is called.
+    self.chromium_src_dir = None
+    # Path to where artifacts should be downloaded from Google Storage. Will be
+    # populated when checkout_dependencies is called.
+    self.downloads_dir = None
+    # Path where swarming artifacts (isolate file, json output, etc) will be
+    # stored. Will be populated when checkout dependencies is caled.
+    self.swarming_temp_dir = None
+    # Keep track of all dirs created during recipe execution. Will be cleaned
+    # up by the cleanup step.
+    self._created_dirs = []
+
+  def checkout_dependencies(self):
+    """Checks out all repositories required for CT to run on swarming bots."""
+    # Checkout chromium and swarming.
+    self.m.chromium.set_config('chromium')
+    self.m.gclient.set_config('chromium')
+    self.m.bot_update.ensure_checkout(force=True)
+    self.m.swarming_client.checkout()
+    # Set the paths required by this recipe module.
+    self.chromium_src_dir = self.m.path['checkout']
+    self.downloads_dir = self.chromium_src_dir.join('content', 'test', 'ct')
+    self.swarming_temp_dir = self.m.path['tmp_base'].join('swarming_temp_dir')
+    self._makedirs(self.swarming_temp_dir)
+
+  def _makedirs(self, d):
+    """Creates the specified dir and registers it as a recipe created dir.
+
+    All directories created by this method will be deleted when cleanup() is
+    called.
+    """
+    self.m.file.makedirs('makedirs %s' % d, d)
+    self._created_dirs.append(d)
+
+  def download_CT_binary(self, ct_binary_name):
+    """Downloads the specified CT binary from GS into the downloads_dir."""
+    self.m.gsutil.download(
+        bucket=CT_GS_BUCKET,
+        source='swarming/binaries/%s' % ct_binary_name,
+        dest=self.downloads_dir.join(ct_binary_name))
+
+  def download_page_artifacts(self, page_type, slave_num, local_slave_dir):
+    """Downloads all the artifacts needed to run benchmarks on a page.
+
+    Args:
+      page_type: str. The CT page type. Eg: 1k, 10k.
+      slave_num: int. The number of the slave used to determine which GS
+                 directory to download from. Eg: for the top 1k, slave1 will
+                 contain webpages 1-10, slave2 will contain 11-20.
+      local_slave_dir: path obj. The directory artifacts should be downloaded
+                       to.
+    """
+    # Download page sets.
+    page_sets_dir = local_slave_dir.join('page_sets')
+    self._makedirs(page_sets_dir)
+    self.m.gsutil.download(
+        bucket=CT_GS_BUCKET,
+        source='swarming/page_sets/%s/slave%s/*' % (page_type, slave_num),
+        dest=page_sets_dir)
+
+    # Download archives.
+    wpr_dir = page_sets_dir.join('data')
+    self._makedirs(wpr_dir)
+    self.m.gsutil.download(
+        bucket=CT_GS_BUCKET,
+        source='swarming/webpage_archives/%s/slave%s/*' % (page_type,
+                                                           slave_num),
+        dest=wpr_dir)
+
+  def create_isolated_gen_json(self, isolate_path, base_dir, os_type,
+                               slave_num):
+    """Creates an isolated.gen.json file.
+
+    Args:
+      isolate_path: path obj. Path to the isolate file.
+      base_dir: path obj. Dir that is the base of all paths in the isolate file.
+      os_type: str. The OS type to use when archiving the isolate file.
+               Eg: linux.
+      slave_num: int. The slave we want to create isolated.gen.json file for.
+
+    Returns:
+      Path to the isolated.gen.json file.
+    """
+    isolated_path = self.swarming_temp_dir.join(
+        'ct-task-%s.isolated' % slave_num)
+    isolate_args = [
+      '--isolate', isolate_path,
+      '--isolated', isolated_path,
+      '--config-variable', 'OS', os_type,
+    ]
+    isolated_gen_dict = {
+      'version': 1,
+      'dir': base_dir,
+      'args': isolate_args,
+    }
+    isolated_gen_json = self.swarming_temp_dir.join(
+        'slave%s.isolated.gen.json' % slave_num)
+    with open(str(isolated_gen_json), 'w') as fout:
+      fout.write(self.m.json.dumps(isolated_gen_dict, indent=4))
+    return isolated_gen_json
+
+  def batcharchive(self, isolated_gen_json_files):
+    """Calls batcharchive on the specified isolated.gen.json files."""
+    batcharchive_args = [
+      'batcharchive',
+      '--isolate-server', self.m.isolate.isolate_server,
+      '--',
+    ]
+    batcharchive_args.extend(str(i) for i in isolated_gen_json_files)
+    self.m.python(
+        'batcharchiving isolated.gen.json for all slaves',
+        self.m.swarming_client.path.join('isolate.py'),
+        batcharchive_args)
+
+  def trigger_swarming_task(self, task_name, slave_num, dimensions):
+    """Triggers swarming task using the slave's isolated and json output file.
+
+    Args:
+      task_name: The name of the swarming task.
+      slave_num: int. The slave we want to trigger swarming tasks for.
+      dimensions: list of str. The dimensions to run the task on.
+                  Eg: ['os Ubuntu', 'gpu 10de']
+    """
+    isolated_path = self.swarming_temp_dir.join(
+        'ct-task-%s.isolated' % slave_num)
+    json_output = self.swarming_temp_dir.join('ct-task-%s.json' % slave_num)

[M-A Ruel, 2015/11/11 23:53:36]

I don't think duplicating swarming.py is a good idea. On the other hand I don't
want to block you.

So rubberstamp lgtm but if I break the python script, which I know I'll do in
the future, you'll keep the pieces.

Also you'll have to switch to the Go implementation of the batcharchive. But to
get something up and running to verify that things works it's fine.

[rmistry, 2015/11/12 00:07:34]

I would like to switch to the Go implementation so that things do not break in
the future. But I have not been able to figure out how to do that, see my
comment here:
https://codereview.chromium.org/1423993007/diff/440001/scripts/slave/recipes/...

+    swarming_trigger_args = [
+      'trigger',
+      '--task-name', task_name,
+      isolated_path,
+      '--swarming', self.m.swarming.swarming_server,
+      '--isolate-server', self.m.isolate.isolate_server,
+      '--dump-json', json_output

[M-A Ruel, 2015/11/11 23:53:36]

Specify: '--priority', '90'

This will run a quite low priority using whatever left over capacity there is.

[rmistry, 2015/11/12 14:39:17]

Done. I brought up a job to see if this worked but it still showed up as
priority 100: https://chromium-swarm.appspot.com/user/task/2b14d294d7c6ef10

I saw the following message in my logs: "Priority was reset to 100". Any
suggestions?

+    ]
+    for d in dimensions:
+      swarming_trigger_args.append('--dimension')
+      swarming_trigger_args.extend(i for i in d.split())
+    self.m.python(
+        'triggering task for slave%s' % slave_num,
+        self.m.swarming_client.path.join('swarming.py'),
+        swarming_trigger_args)
+
+  def collect_swarming_task(self, slave_num):
+    """Collects swarming task for the specified slave."""
+    json_output = self.swarming_temp_dir.join('ct-task-%s.json' % slave_num)
+    swarming_collect_args = [
+      'collect',
+      '--swarming', self.m.swarming.swarming_server,
+      '--json', json_output
+    ]
+    self.m.python(
+        'collecting task for slave%s' % slave_num,
+        self.m.swarming_client.path.join('swarming.py'),
+        swarming_collect_args)
+
+  def cleanup(self):
+    """Cleans up all directories created by this recipe module."""
+    for d in self._created_dirs:
+      self.m.file.rmtree('Removing dir %s' % d, d)