Reland "Skia: Maintain a checkout on swarming (compile) bots"

scripts/slave/recipe_modules/skia/api.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 
 import json
 import os
 import re
 import sys
 
 from recipe_engine import recipe_api
+from recipe_engine import config_types
 
 # TODO(luqui): Make this recipe stop depending on common so we can make it
 # independent of build/.
 sys.path.append(
     os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(
         os.path.abspath(__file__))))))
 from common.skia import global_constants
 
 from . import android_flavor
 from . import appurify_flavor
@@ skipping 119 matching lines @@
     fake_spec = None
     if self._test_data.enabled:
       fake_spec = fake_specs.FAKE_SPECS[builder_name]
     builder_spec = self.json_from_file(
       skia_dir.join('tools', 'buildbot_spec.py'),
       skia_dir,
       builder_name,
       fake_spec)
     return builder_spec
 
+  def make_path(self, *path):
+    """Return a Path object for the given path."""
+    key  = 'custom_%s' % '_'.join(path)
+    self.m.path.c.base_paths[key] = tuple(path)
+    return self.m.path[key]
+
   def setup(self, running_in_swarming=False):
     """Prepare the bot to run."""
     # Setup
     self.failed = []
 
     self.builder_name = self.m.properties['buildername']
     self.master_name = self.m.properties['mastername']
     self.slave_name = self.m.properties['slavename']
 
     self.slave_dir = self.m.path['slave_build']
-    self.skia_dir = self.slave_dir.join('skia')
-    self.infrabots_dir = self.skia_dir.join('infra', 'bots')
+    self.checkout_root = self.slave_dir
+    self.default_env = {}
+    self.is_compile_bot = self.builder_name.startswith('Build-')
 
-    self.default_env = {}
     if running_in_swarming:
       self.default_env['CHROME_HEADLESS'] = '1'
-      depot_tools = self.slave_dir.join('depot_tools')
-      self.default_env['PATH'] = '%s:%%(PATH)s' % depot_tools
+      # The 'build' and 'depot_tools' directories are provided through isolate
+      # and aren't in the expected location, so we need to override them.
+      self.m.path.c.base_paths['depot_tools'] = (
+          self.m.path.c.base_paths['slave_build'] + ('depot_tools',))
+      self.default_env['PATH'] = '%s:%%(PATH)s' % self.m.path['depot_tools']
+      self.m.path.c.base_paths['build'] = (
+          self.m.path.c.base_paths['slave_build'] + ('build',))
+      self.default_env['PYTHONPATH'] = self.m.path['build'].join('scripts')
+
+      # Compile bots keep a persistent checkout.
+      if self.is_compile_bot:
+        if 'Win' in self.builder_name:
+          self.checkout_root = self.make_path('C:\\', 'b', 'cache')
+        else:
+          self.checkout_root = self.make_path('/', 'b', 'cache')
+
+    self.skia_dir = self.checkout_root.join('skia')
+    self.infrabots_dir = self.skia_dir.join('infra', 'bots')
 
     # We run through this recipe in one of two ways:
     # 1. Normal bot: run all of the steps.
     # 2. Running as a Swarming task: perform the given task only, with
     #    adaptations for running within Swarming, eg. copying build results
     #    into the correct output directory.
     self.running_in_swarming = running_in_swarming
 
     # Some bots also require a checkout of chromium.
     self._need_chromium_checkout = 'CommandBuffer' in self.builder_name
 
     # Check out the Skia code.
     self.checkout_steps()
 
     # Obtain the spec for this builder from the Skia repo. Use it to set more
     # properties.
     self.builder_spec = self.get_builder_spec(self.skia_dir, self.builder_name)
 
     self.builder_cfg = self.builder_spec['builder_cfg']
     self.role = self.builder_cfg['role']
 
     # Set some important variables.
     self.resource_dir = self.skia_dir.join('resources')
     self.images_dir = self.slave_dir.join('images')
+    self.out_dir = self.skia_dir.join('out', self.builder_name)
     if self.running_in_swarming:
       self.swarming_out_dir = self.m.properties['swarm_out_dir']
-      self.out_dir = self.slave_dir.join('out')
       self.local_skp_dir = self.slave_dir.join('skps')
+      if not self.is_compile_bot:
+        self.out_dir = self.slave_dir.join('out')
     else:
-      self.out_dir = self.m.path['checkout'].join('out', self.builder_name)
       self.local_skp_dir = self.slave_dir.join('playback', 'skps')
     self.tmp_dir = self.m.path['slave_build'].join('tmp')
 
     self.gsutil_env_chromium_skia_gm = self.gsutil_env(BOTO_CHROMIUM_SKIA_GM)
 
     self.device_dirs = None
     self._ccache = None
     self._checked_for_ccache = False
     self.configuration = self.builder_spec['configuration']
     self.default_env.update({'SKIA_OUT': self.out_dir,
@@ skipping 39 matching lines @@
       raise self.m.step.StepFailure('Failed build steps: %s' %
                                     ', '.join([f.name for f in self.failed]))
 
   def _run_once(self, fn, *args, **kwargs):
     if not hasattr(self, '_already_ran'):
       self._already_ran = {}
     if not fn.__name__ in self._already_ran:
       self._already_ran[fn.__name__] = True
       fn(*args, **kwargs)
 
-  def update_repo(self, repo):
+  def update_repo(self, parent_dir, repo):
     """Update an existing repo. This is safe to call without gen_steps."""
-    repo_path = self.m.path['slave_build'].join(repo.name)
+    repo_path = parent_dir.join(repo.name)
     if self.m.path.exists(repo_path):
       if self.m.platform.is_win:
         git = 'git.bat'
       else:
         git = 'git'
       self.m.step('git remote set-url',
                   cmd=[git, 'remote', 'set-url', 'origin', repo.url],
                   cwd=repo_path,
                   infra_step=True)
       self.m.step('git fetch',
                   cmd=[git, 'fetch'],
                   cwd=repo_path,
                   infra_step=True)
       self.m.step('git reset',
                   cmd=[git, 'reset', '--hard', repo.revision],
                   cwd=repo_path,
                   infra_step=True)
       self.m.step('git clean',
                   cmd=[git, 'clean', '-d', '-f'],
                   cwd=repo_path,
                   infra_step=True)
 
   def checkout_steps(self):
     """Run the steps to obtain a checkout of Skia."""
+    cfg_kwargs = {}
     if self.running_in_swarming:
-      # We should've obtained the Skia checkout through isolates, so we don't
-      # need to perform the checkout ourselves.
-      self.m.path['checkout'] = self.m.path['slave_build'].join('skia')
-      self.got_revision = self.m.properties['revision']
-      return
+      if not self.is_compile_bot:
+        # We should've obtained the Skia checkout through isolates, so we don't
+        # need to perform the checkout ourselves.
+        self.m.path['checkout'] = self.skia_dir
+        self.got_revision = self.m.properties['revision']
+        return
+
+      # The gclient cache gets deleted by Swarming, so don't use it.
+      cfg_kwargs['CACHE_DIR'] = None
+
+    # Create the checkout path if necessary.
+    if not self.m.path.exists(self.checkout_root):
+      self.m.file.makedirs('checkout_path', self.checkout_root, infra_step=True)
 
     # Initial cleanup.
-    gclient_cfg = self.m.gclient.make_config()
+    gclient_cfg = self.m.gclient.make_config(**cfg_kwargs)
     skia = gclient_cfg.solutions.add()
     skia.name = 'skia'
     skia.managed = False
     skia.url = global_constants.SKIA_REPO
     skia.revision = self.m.properties.get('revision') or 'origin/master'
-    self.update_repo(skia)
+    self.update_repo(self.checkout_root, skia)
 
     if self._need_chromium_checkout:
       chromium = gclient_cfg.solutions.add()
       chromium.name = 'src'
       chromium.managed = False
       chromium.url = 'https://chromium.googlesource.com/chromium/src.git'
       chromium.revision = 'origin/lkgr'
-      self.update_repo(chromium)
+      self.update_repo(self.checkout_root, chromium)
 
     # Run 'gclient sync'.
     gclient_cfg.got_revision_mapping['skia'] = 'got_revision'
     gclient_cfg.target_os.add('llvm')
-    update_step = self.m.gclient.checkout(gclient_config=gclient_cfg)
+    checkout_kwargs = {}
+    if self.running_in_swarming:
+      checkout_kwargs['env'] = self.default_env
+    update_step = self.m.gclient.checkout(gclient_config=gclient_cfg,
+                                          cwd=self.checkout_root,
+                                          **checkout_kwargs)
 
     self.got_revision = update_step.presentation.properties['got_revision']
     self.m.tryserver.maybe_apply_issue()
 
     if self._need_chromium_checkout:
       self.m.gclient.runhooks()
 
   def copy_build_products(self, src, dst):
     """Copy whitelisted build products from src to dst."""
     self.m.python.inline(
@@ skipping 534 matching lines @@
     # Don't bother to include role, which is always Test.
     # TryBots are uploaded elsewhere so they can use the same key.
     blacklist = ['role', 'is_trybot']
 
     flat = []
     for k in sorted(self.builder_cfg.keys()):
       if k not in blacklist:
         flat.append(k)
         flat.append(self.builder_cfg[k])
     return flat