CT Skia recipe to run DM on the top 10k SKPs using swarming

scripts/slave/recipe_modules/ct_swarming/api.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 
 from recipe_engine import recipe_api
 
 
 CT_GS_BUCKET = 'cluster-telemetry'
 
@@ skipping 51 matching lines @@
     The artifacts are downloaded into subdirectories in the downloads_dir.
 
     Args:
       page_type: str. The CT page type. Eg: 1k, 10k.
       slave_num: int. The number of the slave used to determine which GS
                  directory to download from. Eg: for the top 1k, slave1 will
                  contain webpages 1-10, slave2 will contain 11-20.
     """
     # Download page sets.
     page_sets_dir = self.downloads_dir.join('slave%s' % slave_num, 'page_sets')
-    self.m.file.makedirs('Create page_sets dir', page_sets_dir)
+    self.m.file.makedirs('page_sets dir', page_sets_dir)
     self.m.gsutil.download(
         bucket=CT_GS_BUCKET,
         source='swarming/page_sets/%s/slave%s/*' % (page_type, slave_num),
         dest=page_sets_dir)
 
     # Download archives.
     wpr_dir = page_sets_dir.join('data')
-    self.m.file.makedirs('Create WPR dir', wpr_dir)
+    self.m.file.makedirs('WPR dir', wpr_dir)
     self.m.gsutil.download(
         bucket=CT_GS_BUCKET,
         source='swarming/webpage_archives/%s/slave%s/*' % (page_type,
                                                            slave_num),
         dest=wpr_dir)
 
+  def download_skps(self, page_type, slave_num, skps_chromium_build):
+    """Downloads SKPs corresponding to the specified page type, slave and build.
+
+    The SKPs are downloaded into subdirectories in the downloads_dir.
+
+    Args:
+      page_type: str. The CT page type. Eg: 1k, 10k.
+      slave_num: int. The number of the slave used to determine which GS
+                 directory to download from. Eg: for the top 1k, slave1 will
+                 contain SKPs from webpages 1-10, slave2 will contain 11-20.
+      skps_chromium_build: str. The build the SKPs were captured from.
+    """
+    skps_dir = self.downloads_dir.join('slave%s' % slave_num, 'skps')
+    self.m.file.rmtree('SKPs dir', skps_dir)
+    self.m.file.makedirs('SKPs dir', skps_dir)
+    full_source = 'gs://%s/skps/%s/%s/slave%s/*' % (
+        CT_GS_BUCKET, page_type, skps_chromium_build, slave_num)
+    self.m.gsutil(['-m', 'cp', full_source, skps_dir])
+
   def create_isolated_gen_json(self, isolate_path, base_dir, os_type,
                                slave_num, extra_variables):
     """Creates an isolated.gen.json file.
 
     Args:
       isolate_path: path obj. Path to the isolate file.
       base_dir: path obj. Dir that is the base of all paths in the isolate file.
       os_type: str. The OS type to use when archiving the isolate file.
                Eg: linux.
       slave_num: int. The slave we want to create isolated.gen.json file for.
       extra_variables: dict of str to str. The extra vars to pass to isolate.
                       Eg: {'SLAVE_NUM': '1', 'MASTER': 'ChromiumPerfFYI'}
 
     Returns:
       Path to the isolated.gen.json file.
     """
-    self.m.file.makedirs('Create swarming tmp dir', self.swarming_temp_dir)
+    self.m.file.makedirs('swarming tmp dir', self.swarming_temp_dir)
     isolated_path = self.swarming_temp_dir.join(
         'ct-task-%s.isolated' % slave_num)
     isolate_args = [
       '--isolate', isolate_path,
       '--isolated', isolated_path,
       '--config-variable', 'OS', os_type,
     ]
     for k, v in extra_variables.iteritems():
       isolate_args.extend(['--extra-variable', k, v])
     isolated_gen_dict = {
@@ skipping 43 matching lines @@
     self.m.swarming.trigger(swarming_tasks)
     return swarming_tasks
 
   def collect_swarming_tasks(self, swarming_tasks):
     """Collects all swarming tasks triggered by this recipe.
 
     Args:
       swarming_tasks: list of swarming.SwarmingTask instances.
     """
     return self.m.swarming.collect(swarming_tasks)