CT Perf recipe to run benchmarks on the top 1k sites using swarming

scripts/slave/recipes/perf/ct_top1k_rr_perf.py

+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+DEPS = [
+  'archive',
+  'bot_update',
+  'chromium',
+  'file',
+  'gclient',
+  'gsutil',
+  'isolate',
+  'json',
+  'path',
+  'platform',
+  'properties',
+  'python',
+  'step',
+  'time',
+  'swarming',
+  'swarming_client',
+  'zip',
+]
+
+
+CT_BUCKET = 'cluster-telemetry'
+CT_PAGE_TYPE = '1k'
+CT_BINARY = 'run_chromium_perf_swarming'
+CT_ISOLATE_TEMPLATE = 'ct_top1k.isolate.tmpl'
+
+# Number of slaves to shard CT runs to.
+# TODO(rmistry): Change the below to 100 when ready to run the full top 1k.
+CT_NUM_SLAVES = 2
+
+
+def _DownloadAndExtractBinary(api):
+  """Downloads the binary from the revision passed to the recipe."""
+  build_archive_url = api.properties['parent_build_archive_url']

[M-A Ruel, 2015/11/09 19:46:45]

I think you could inline this and it'd fit 80 cols ?

[rmistry, 2015/11/10 12:58:06]

Done.

+  api.archive.download_and_unzip_build(
+      step_name='Download and Extract Binary',
+      target='Release',
+      build_url=None,  # This is a required parameter, but has no effect.
+      build_archive_url=build_archive_url)
+
+
+# TODO(rmistry): What priority can I give the below tasks??
+def RunSteps(api):
+  # Figure out which benchmark to use.
+  buildername = api.properties['buildername']
+  if 'Repaint' in buildername:
+    benchmark = 'repaint'
+  elif 'RR' in buildername:
+    benchmark = 'rasterize_and_record_micro'
+  else:
+    raise Exception('Do not recognise the buildername %s.' % buildername)
+
+  # Checkout chromium and swarming.
+  api.chromium.set_config('chromium')
+  api.gclient.set_config('chromium')
+  api.bot_update.ensure_checkout(force=True)
+  api.swarming_client.checkout()
+
+  # Download the prebuilt chromium binary.
+  _DownloadAndExtractBinary(api)
+
+  # Path to the chromium src directory.
+  chromium_src_dir = api.path['checkout']
+  # Path to where artifacts should be downloaded from Google Storage.
+  downloads_dir = chromium_src_dir.join('content', 'test', 'ct')

[M-A Ruel, 2015/11/09 19:46:45]

Is it in the .gitignore? Otherwise it'll get continuously deleted. Is is
possible to keep a local cache to speed things up or it's not an issue in
practice?

[rmistry, 2015/11/10 12:58:06]

I think its ok to get continuously deleted for now. I can look into optimizing
this if needed once things are up and running.

+  # Path where swarming artifacts (isolate file, json output) will be stored.
+  swarming_temp_dir = api.path['tmp_base'].join('swarming_temp_dir')
+  api.file.makedirs('makedirs swarming_temp_dir', swarming_temp_dir)
+
+  # Download Cluster Telemetry binary.
+  ct_binary_path = downloads_dir.join(CT_BINARY)
+  api.gsutil.download(
+      bucket=CT_BUCKET,
+      source='swarming/binaries/%s' % CT_BINARY,
+      dest=ct_binary_path)
+
+  # List that will contain all .isolated.gen.json file locations.
+  isolated_gen_json_files = []
+
+  # Record how long the step took in swarming tasks.
+  swarming_start_time = api.time.time()
+
+  for slave_num in range(1, CT_NUM_SLAVES + 1):
+    slave_dir = downloads_dir.join('slave%s' % slave_num)
+    api.file.makedirs('makedirs slave_dir', slave_dir)
+
+    # Download page sets.
+    page_sets_dir = slave_dir.join('page_sets')
+    api.file.makedirs('makedirs page_sets', page_sets_dir)
+    api.gsutil.download(
+        bucket=CT_BUCKET,
+        source='swarming/page_sets/%s/slave%s/*' % (CT_PAGE_TYPE, slave_num),
+        dest=page_sets_dir)
+
+    # Download archives.
+    wpr_dir = page_sets_dir.join('data')
+    api.file.makedirs('makedirs wpr', wpr_dir)
+    api.gsutil.download(
+        bucket=CT_BUCKET,
+        source='swarming/webpage_archives/%s/slave%s/*' % (CT_PAGE_TYPE,
+                                                           slave_num),
+        dest=wpr_dir)
+
+    # TODO(rmistry): Remove the entire below section after crrev.com/1410353007
+    #                is submitted.
+    api.file.copy(
+        'copy %s' % CT_ISOLATE_TEMPLATE,
+        '/repos/chromium/src/chrome/%s' % CT_ISOLATE_TEMPLATE,
+        chromium_src_dir.join('chrome', CT_ISOLATE_TEMPLATE))
+    for f in ['run_ct_top1k.py', 'path_util.py']:
+      api.file.copy(
+          'copy %s' % f,
+          '/repos/chromium/src/content/test/ct/%s' % f,
+          chromium_src_dir.join('content', 'test', 'ct', f))
+
+    # Create this slave's isolate file from the CT_ISOLATE_TEMPLATE.
+    isolate_dir = chromium_src_dir.join('chrome')
+    isolate_template_path = isolate_dir.join(CT_ISOLATE_TEMPLATE)
+    generated_isolate_path = isolate_dir.join(
+        'slave%s.ct_top1k.isolate' % slave_num)
+    with open(str(generated_isolate_path), 'wb') as fout:
+      with open(str(isolate_template_path), 'rb') as fin:
+        for line in fin:
+          fout.write(line.replace('[[SLAVE_NUM]]', str(slave_num))
+                         .replace('[[MASTER]]', api.properties['mastername'])
+                         .replace('[[BUILDER]]', api.properties['buildername'])
+                         .replace('[[GIT_HASH]]',
+                                  api.properties['git_revision'])
+                         .replace('[[BENCHMARK]]', benchmark))
+
+    # Batcharchive everything on the isolate server for efficiency.
+    isolated_path = swarming_temp_dir.join('ct-1k-task-%s.isolated' % slave_num)
+    isolate_args = [
+      '--isolate', generated_isolate_path,
+      '--isolated', isolated_path,
+      '--config-variable', 'OS', 'linux',
+      # TODO(rmistry): Why do I need PRODUCT_DIR ? fails without it. It also
+      #                requires bitmaptools in PRODUCT_DIR.

[nednguyen, 2015/11/09 23:04:32]

Bitmaptools build is used by telemetry's image processing code. It's required,
unless you can prebuilt the binary on multiple platforms & upload them to cloud
storage so telemetry can fetch the  file.

I honestly don't know how to reference the path to bitmap tool in this context.
Can you just reuse the isolate defined for telemetry+chrome in
https://code.google.com/p/chromium/codesearch#chromium/src/tools/perf/chrome_...

[rmistry, 2015/11/10 12:58:06]

I tried using it and it failed with:
IsolateError: These configuration variables were missing from the command line:
CONFIGURATION_NAME, asan, component, disable_nacl, fastbuild,
icu_use_data_file_flag, kasko, lsan, msan, msvs_version, target_arch, tsan,
use_custom_libcxx, use_instrumented_libraries,
use_prebuilt_instrumented_libraries, v8_use_external_startup_data

thats a lot of missing variables, I do not know how to specify them here.

As a workaround I removed the reference to telemetry.isolate in
https://codereview.chromium.org/1410353007/diff2/220001:240001/chrome/ct_top1...
for now.

+      '--path-variable', 'PRODUCT_DIR', api.path['tmp_base'],
+    ]
+    isolated_gen_dict = {
+      'version': 1,
+      'dir': isolate_dir,
+      'args': isolate_args,
+    }
+    # Write the slave isolated.gen.json file.
+    isolated_gen_json = swarming_temp_dir.join(
+        'slave%s.isolated.gen.json' % slave_num)
+    with open(str(isolated_gen_json), 'w') as fout:
+      fout.write(api.json.dumps(isolated_gen_dict, indent=4))
+    isolated_gen_json_files.append(isolated_gen_json)
+
+  # Batcharchive everything on the isolate server for efficiency.
+  batcharchive_args = [
+    'batcharchive',
+    '--isolate-server', api.isolate.isolate_server,
+    '--',
+  ]
+  for isolated_gen_json in isolated_gen_json_files:

[M-A Ruel, 2015/11/09 19:46:45]

batcharchive_args.extend(str(i) for i in isolated_gen_json_files)

[rmistry, 2015/11/10 12:58:06]

Done.

+    batcharchive_args.append(str(isolated_gen_json))
+  api.python(
+      'batcharchiving isolated.gen.json for all slaves',
+      api.swarming_client.path.join('isolate.py'),

[M-A Ruel, 2015/11/09 19:46:45]

Use luci-go implementation which is an order of magnitude faster. You should be
able to use it via the isolate recipe.

[rmistry, 2015/11/10 12:58:06]

How do I use it using
https://code.google.com/p/chromium/codesearch#chromium/build/scripts/slave/re...
?
(sorry, I must have missed something).

[M-A Ruel, 2015/11/12 00:36:01]

isolate_tests() is the function you need. It is hardcoded to glob all the
*.isolated.gen.json in the output directory but this could be changed if
necessary. Otherwise you can write the temporary gen files in the output
directory. Overall it's kind of hardcoded for our use case so it's fairly
possible that it'll need refactoring.

[rmistry, 2015/11/12 14:39:17]

I could use it without refactoring. Also used trigger and collect from the
swarming recipe. Done. PTAL.

+      batcharchive_args)
+
+  # Trigger all swarming tasks.
+  for slave_num in range(1, CT_NUM_SLAVES + 1):
+    task_name = 'ct-1k-task-%s' % slave_num
+    isolated_path = swarming_temp_dir.join('ct-1k-task-%s.isolated' % slave_num)
+    json_output = swarming_temp_dir.join('ct-1k-task-%s.json' % slave_num)
+    swarming_trigger_args = [
+      'trigger',
+      '--task-name', task_name,
+      isolated_path,
+      '--swarming', api.swarming.swarming_server,
+      '--dimension', 'os', 'Ubuntu',
+      '--dimension', 'gpu', '10de',
+      '--isolate-server', api.isolate.isolate_server,
+      '--dump-json', json_output
+    ]
+    api.python(
+        'triggering task for slave%s' % slave_num,
+        api.swarming_client.path.join('swarming.py'),
+        swarming_trigger_args)
+
+    # We have triggered this slave's swarming task. Cleanup slave artifacts.
+    api.file.rmtree('Remove slave dir', slave_dir)
+
+  # Now collect all tasks.
+  for slave_num in range(1, CT_NUM_SLAVES + 1):
+    json_output = swarming_temp_dir.join('ct-1k-task-%s.json' % slave_num)
+    swarming_collect_args = [
+      'collect',
+      '--swarming', api.swarming.swarming_server,
+      '--json', json_output
+    ]
+    api.python(
+        'collecting task for slave%s' % slave_num,
+        api.swarming_client.path.join('swarming.py'),
+        swarming_collect_args)
+
+  # Cleanup the temporary swarming dir.
+  api.file.rmtree('Remove swarming temp dir', swarming_temp_dir)
+
+  print ('Running isolating, triggering and collecting swarming tasks took a '
+         'total of %s seconds') % (api.time.time() - swarming_start_time)