Revert of Removing old recipe code

scripts/slave/recipe_modules/recipe_tryjob/api.py

+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import collections
+import hashlib
+import json
+import re
+
+from recipe_engine import recipe_api
+
+RECIPE_TRYJOB_BYPASS_REASON_TAG = "Recipe-Tryjob-Bypass-Reason"
+
+def get_recipes_path(project_config):
+  # Returns a tuple of the path components to traverse from the root of the repo
+  # to get to the directory containing recipes.
+  return project_config['recipes_path'][0].split('/')
+
+
+def get_deps(project_config):
+  """ Get the recipe engine deps of a project from its recipes.cfg file. """
+  # "[0]" Since parsing makes every field a list
+  return [dep['project_id'][0] for dep in project_config.get('deps', [])]
+
+
+def get_deps_info(projects, configs):
+  """Calculates dependency information (forward and backwards) given configs."""
+  deps = {p: get_deps(configs[p]) for p in projects}
+
+  # Figure out the backwards version of the deps graph. This allows us to figure
+  # out which projects we need to test given a project. So, given
+  #
+  #     A
+  #    / \
+  #   B   C
+  #
+  # We want to test B and C, if A changes. Recipe projects only know about he
+  # B-> A and C-> A dependencies, so we have to reverse this to get the
+  # information we want.
+  downstream_projects = collections.defaultdict(set)
+  for proj, targets in deps.items():
+    for target in targets:
+      downstream_projects[target].add(proj)
+
+  return deps, downstream_projects
+
+
+RietveldPatch = collections.namedtuple(
+    'RietveldPatch', 'project server issue patchset')
+
+
+def parse_patches(failing_step, patches_raw, rietveld, issue, patchset,
+                  patch_project):
+  """
+  gives mapping of project to patch
+    expect input of
+    project1:https://a.b.c/1342342#ps1,project2:https://d.ce.f/1231231#ps1
+  """
+  result = {}
+
+  if rietveld and issue and patchset and patch_project:
+    # convert to str because recipes don't like unicode as step names
+    result[str(patch_project)] = RietveldPatch(
+        patch_project, rietveld, issue, patchset)
+
+  if not patches_raw:
+    return result
+
+  for patch_raw in patches_raw.split(','):
+    project, url = patch_raw.split(':', 1)
+    server, issue_and_patchset = url.rsplit('/', 1)
+    issue, patchset = issue_and_patchset.split('#')
+    patchset = patchset[2:]
+
+    if project in result:
+      failing_step(
+          "Invalid patchset list",
+          "You have two patches for %r. Patches seen so far: %r" % (
+              project, result)
+      )
+
+    result[project] = RietveldPatch(project, server, issue, patchset)
+
+  return result
+
+
+
+PROJECTS_TO_TRY = [
+  'build',
+  'build_limited_scripts_slave',
+  'recipe_engine',
+  'depot_tools',
+]
+
+PROJECT_TO_CONTINUOUS_WATERFALL = {
+  'build': 'https://build.chromium.org/p/chromium.tools.build/builders/'
+    'recipe-simulation_trusty64',
+  'recipe_engine': 'https://build.chromium.org/p/chromium.infra/builders/'
+    'recipe_engine-recipes-tests',
+  'depot_tools': 'https://build.chromium.org/p/chromium.infra/builders/'
+    'depot_tools-recipes-tests',
+}
+
+FILE_BUG_FOR_CONTINUOUS_LINK = 'https://goo.gl/PoAPOJ'
+
+
+class RecipeTryjobApi(recipe_api.RecipeApi):
+  """
+  This is intended as a utility module for recipe tryjobs. Currently it's just a
+  refactored version of a recipe; eventually some of this, especially the
+  dependency information, will probably get moved into the recipe engine.
+  """
+  def _get_project_config(self, project):
+    """Fetch the project config from luci-config.
+
+    Args:
+      project: The name of the project in luci-config.
+
+    Returns:
+      The recipes.cfg file for that project, as a parsed dictionary. See
+      parse_protobuf for details on the format to expect.
+    """
+    result = self.m.luci_config.get_project_config(project, 'recipes.cfg')
+
+    parsed = self.m.luci_config.parse_textproto(result['content'].split('\n'))
+    return parsed
+
+  def _checkout_projects(self, root_dir, url_mapping, deps,
+                        downstream_projects, patches):
+    """Checks out projects listed in projects into root_dir.
+
+    Args:
+      root_dir: Root directory to check this project out in.
+      url_mapping: Project id to url of git repository.
+      downstream_projects: The mapping from project to dependent projects.
+      patches: Mapping of project id to patch to apply to that project.
+
+    Returns:
+      The projects we want to test, and the locations of those projects
+    """
+    # TODO(martiniss): be smarter about which projects we actually run tests on
+
+    # All the projects we want to test.
+    projs_to_test  = set()
+    # Projects we need to look at dependencies for.
+    queue = set(patches.keys())
+    # luci config project name to file system path of the checkout
+    locations = {}
+
+    while queue:
+      proj = queue.pop()
+      if proj not in projs_to_test:
+        locations[proj] = self._checkout_project(
+            proj, url_mapping[proj], root_dir, patches.get(proj))
+        projs_to_test.add(proj)
+
+        for downstream in downstream_projects[proj]:
+          queue.add(downstream)
+        for upstream in deps[proj]:
+          queue.add(upstream)
+
+    return projs_to_test, locations
+
+  def _checkout_project(self, proj, proj_config, root_dir, patch=None):
+    """
+    Args:
+      proj: luci-config project name to checkout.
+      proj_config: The recipes.cfg configuration for the project.
+      root_dir: The temporary directory to check the project out in.
+      patch: optional patch to apply to checkout.
+
+    Returns:
+      Path to repo on disk.
+    """
+    checkout_path = root_dir.join(proj)
+    repo_path = checkout_path.join(proj)
+    self.m.file.makedirs('%s directory' % proj, repo_path)
+
+    # Not working yet, but maybe??
+    #api.file.rmtree('clean old %s repo' % proj, checkout_path)
+
+    config = self.m.gclient.make_config(
+        GIT_MODE=True, CACHE_DIR=root_dir.join("__cache_dir"))
+    soln = config.solutions.add()
+    soln.name = proj
+    soln.url = proj_config['repo_url']
+
+    kwargs = {
+        'suffix': proj,
+        'gclient_config': config,
+        'force': True,
+        'cwd': checkout_path,
+    }
+    if patch:
+      kwargs['rietveld'] = patch.server
+      kwargs['issue'] = patch.issue
+      kwargs['patchset'] = patch.patchset
+    else:
+      kwargs['patch'] = False
+
+    self.m.bot_update.ensure_checkout(**kwargs)
+    return repo_path
+
+  def get_fail_build_info(self, downstream_projects, patches):
+    fail_build = collections.defaultdict(lambda: True)
+
+    for proj, patch in patches.items():
+      patch_url = "%s/%s" % (patch.server, patch.issue)
+      desc = self.m.git_cl.get_description(
+          patch=patch_url, codereview='rietveld', suffix=proj)
+
+      assert desc.stdout is not None, "CL %s had no description!" % patch_url
+
+      bypass_reason = self.m.tryserver.get_footer(
+          RECIPE_TRYJOB_BYPASS_REASON_TAG, patch_text=desc.stdout)
+
+      fail_build[proj] = not bool(bypass_reason)
+
+    # Propogate Falses down the deps tree
+    queue = list(patches.keys())
+    while queue:
+      item = queue.pop(0)
+
+      if not fail_build[item]:
+        for downstream in downstream_projects.get(item, []):
+          fail_build[downstream] = False
+          queue.append(downstream)
+
+    return fail_build
+
+  def simulation_test(self, proj, proj_config, repo_path, deps):
+    """
+    Args:
+      proj: The luci-config project to simulation_test.
+      proj_config: The recipes.cfg configuration for the project.
+      repo_path: The path to the repository on disk.
+      deps: Mapping from project name to Path. Passed into the recipes.py
+        invocation via the "-O" options.
+
+    Returns the result of running the simulation tests.
+    """
+    recipes_path = get_recipes_path(proj_config) + ['recipes.py']
+    recipes_py_loc = repo_path.join(*recipes_path)
+    args = []
+    for dep_name, location in deps.items():
+      args += ['-O', '%s=%s' % (dep_name, location)]
+    args += ['--package', repo_path.join('infra', 'config', 'recipes.cfg')]
+
+    args += ['simulation_test']
+
+    return self._python('%s tests' % proj, recipes_py_loc, args)
+
+  def _python(self, name, script, args, **kwargs):
+    """Call python from infra's virtualenv.
+
+    This is needed because of the coverage module, which is not installed by
+    default, but which infra's python has installed."""
+    return self.m.step(name, [
+        self.m.path['checkout'].join('ENV', 'bin', 'python'),
+        '-u', script] + args, **kwargs)
+
+  def run_tryjob(self, patches_raw, rietveld, issue, patchset, patch_project):
+    patches = parse_patches(
+        self.m.python.failing_step, patches_raw, rietveld, issue, patchset,
+        patch_project)
+
+    root_dir = self.m.path['slave_build']
+
+    # Needed to set up the infra checkout, for _python
+    self.m.gclient.set_config('infra')
+    self.m.gclient.c.solutions[0].revision = 'origin/master'
+    self.m.gclient.checkout()
+    self.m.gclient.runhooks()
+
+    url_mapping = self.m.luci_config.get_projects()
+
+    # TODO(martiniss): use luci-config smarter; get recipes.cfg directly, rather
+    # than in two steps.
+    # luci config project name to recipe config namedtuple
+    recipe_configs = {}
+
+    # List of all the projects we care about testing. luci-config names
+    all_projects = set(p for p in url_mapping if p in PROJECTS_TO_TRY)
+
+    recipe_configs = {
+        p: self._get_project_config(p) for p in all_projects}
+
+    deps, downstream_projects = get_deps_info(all_projects, recipe_configs)
+    should_fail_build_mapping = self.get_fail_build_info(
+        downstream_projects, patches)
+
+    projs_to_test, locations = self._checkout_projects(
+        root_dir, url_mapping, deps, downstream_projects, patches)
+
+    bad_projects = []
+    for proj in projs_to_test:
+      deps_locs = {dep: locations[dep] for dep in deps[proj]}
+
+      try:
+        result = self.simulation_test(
+          proj, recipe_configs[proj], locations[proj], deps_locs)
+      except recipe_api.StepFailure as f:
+        result = f.result
+        if should_fail_build_mapping.get(proj, True):
+          bad_projects.append(proj)
+      finally:
+        link = PROJECT_TO_CONTINUOUS_WATERFALL.get(proj)
+        if link:
+          result.presentation.links['reference builder'] = link
+        else:
+          result.presentation.links[
+              'no reference builder; file a bug to get one?'] = (
+                  FILE_BUG_FOR_CONTINUOUS_LINK)
+
+
+    if bad_projects:
+      raise recipe_api.StepFailure(
+          "One or more projects failed tests: %s" % (
+              ','.join(bad_projects)))
+
+