Rebuild when build-time/run-time deps are upgraded.

pym/_emerge/depgraph.py

 # Copyright 1999-2011 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import print_function
 
 import difflib
 import gc
 import logging
 import re
 import sys
 import textwrap
+from collections import deque
 from itertools import chain
 
 import portage
 from portage import os, OrderedDict
 from portage import _unicode_decode
 from portage.const import PORTAGE_PACKAGE_ATOM
 from portage.dbapi import dbapi
 from portage.dep import Atom, extract_affecting_use, check_required_use, human_readable_required_use, _repo_separator
 from portage.eapi import eapi_has_strong_blocks, eapi_has_required_use
 from portage.exception import InvalidAtom, InvalidDependString
@@ skipping 98 matching lines @@
                 self._required_set_names = set(["world"])
 
                 atoms = ' '.join(myopts.get("--exclude", [])).split()
                 self.excluded_pkgs = _wildcard_set(atoms)
                 atoms = ' '.join(myopts.get("--reinstall-atoms", [])).split()
                 self.reinstall_atoms = _wildcard_set(atoms)
                 atoms = ' '.join(myopts.get("--nousepkg-atoms", [])).split()
                 self.nousepkg_atoms = _wildcard_set(atoms)
                 atoms = ' '.join(myopts.get("--useoldpkg-atoms", [])).split()
                 self.useoldpkg_atoms = _wildcard_set(atoms)
+                atoms = ' '.join(myopts.get("--norebuild-atoms", [])).split()
+                self.norebuild_atoms = _wildcard_set(atoms)
+
+                self.rebuild = myopts.get('--rebuild', 'n') != 'n'
 
 class _depgraph_sets(object):
         def __init__(self):
                 # contains all sets added to the graph
                 self.sets = {}
                 # contains non-set atoms given as arguments
                 self.sets['__non_set_args__'] = InternalPackageSet(allow_repo=True)
                 # contains all atoms from all sets added to the graph, including
                 # atoms given as arguments
                 self.atoms = InternalPackageSet(allow_repo=True)
                 self.atom_arg_map = {}
 
+class _rebuild_config(object):
+        def __init__(self, frozen_config, backtrack_parameters):
+                self._graph = digraph()
+                self._frozen_config = frozen_config
+                self.rebuild_list = backtrack_parameters.rebuild_list.copy()
+                self.orig_rebuild_list = self.rebuild_list.copy()
+                self.reinstall_list = backtrack_parameters.reinstall_list.copy()
+
+        def add(self, dep_pkg, dep):
+                parent = dep.collapsed_parent
+                priority = dep.collapsed_priority
+                norebuild_atoms = self._frozen_config.norebuild_atoms
+                if (self._frozen_config.rebuild and isinstance(parent, Package) and
+                        parent.built and (priority.buildtime or priority.runtime) and
+                        isinstance(dep_pkg, Package) and
+                        not norebuild_atoms.findAtomForPackage(parent)):
+                        self._graph.add(dep_pkg, parent, priority)
+
+        def _trigger_rebuild(self, parent, build_deps, runtime_deps):
+                root_slot = (parent.root, parent.slot_atom)
+                if root_slot in self.rebuild_list:
+                        return False
+                trees = self._frozen_config.trees
+                children = set(build_deps).intersection(runtime_deps)
+                reinstall = False
+                for slot_atom in children:
+                        kids = set([build_deps[slot_atom], runtime_deps[slot_atom]])
+                        for dep_pkg in kids:
+                                dep_root_slot = (dep_pkg.root, slot_atom)
+                                if (not dep_pkg.built and
+                                        dep_root_slot not in self.orig_rebuild_list):
+                                        # There's no binary package for dep_pkg, so any binary
+                                        # package for this parent would be invalid. Force rebuild.
+                                        self.rebuild_list.add(root_slot)
+                                        return True
+                                elif ("--usepkg" in self._frozen_config.myopts and
+                                        (dep_root_slot in self.reinstall_list or
+                                        dep_root_slot in self.rebuild_list or
+                                        not dep_pkg.installed)):
+
+                                        # A direct rebuild dependency is being installed. We
+                                        # should update the parent as well to the latest binary,
+                                        # if that binary is valid.
+                                        #
+                                        # To validate the binary, we check whether all of the
+                                        # rebuild dependencies are present on the same binhost.
+                                        #
+                                        # 1) If parent is present on the binhost, but one of its
+                                        #    rebuild dependencies is not, then the parent should
+                                        #    be rebuilt from source.
+                                        # 2) Otherwise, the parent binary is assumed to be valid,
+                                        #    because all of its rebuild dependencies are
+                                        #    consistent.
+                                        bintree = trees[parent.root]["bintree"]
+                                        uri = bintree.get_pkgindex_uri(parent.cpv)
+                                        dep_uri = bintree.get_pkgindex_uri(dep_pkg.cpv)
+                                        bindb = bintree.dbapi
+
+                                        if uri and uri != dep_uri:
+                                                # 1) Remote binary package is invalid because it was
+                                                #    built without dep_pkg. Force rebuild.
+                                                self.rebuild_list.add(root_slot)
+                                                return True
+                                        elif (parent.installed and
+                                                root_slot not in self.reinstall_list):
+                                                inst_build_time = parent.metadata.get("BUILD_TIME")
+                                                try:
+                                                        bin_build_time, = bindb.aux_get(parent.cpv,
+                                                                ["BUILD_TIME"])
+                                                except KeyError:
+                                                        continue
+                                                if bin_build_time != inst_build_time:
+                                                        # 2) Remote binary package is valid, and local package
+                                                        #    is not up to date. Force reinstall.
+                                                        reinstall = True
+                if reinstall:
+                        self.reinstall_list.add(root_slot)
+                return reinstall
+
+        def trigger_rebuilds(self):
+                """
+                Trigger rebuilds where necessary. If pkgA has been updated, and pkgB
+                depends on pkgA at both build-time and run-time, pkgB needs to be
+                rebuilt.
+                """
+                need_restart = False
+                graph = self._graph
+                build_deps = {}
+                runtime_deps = {}
+                leaf_nodes = deque(graph.leaf_nodes())
+
+                # Trigger rebuilds bottom-up (starting with the leaves) so that parents
+                # will always know which children are being rebuilt.
+                while leaf_nodes:
+                        node = leaf_nodes.popleft()
+                        slot_atom = node.slot_atom
+
+                        # Remove our leaf node from the graph, keeping track of deps.
+                        parents = graph.nodes[node][1].items()
+                        graph.remove(node)
+                        for parent, priorities in parents:
+                                for priority in priorities:
+                                        if priority.buildtime:
+                                                build_deps.setdefault(parent, {})[slot_atom] = node
+                                        if priority.runtime:
+                                                runtime_deps.setdefault(parent, {})[slot_atom] = node
+                                if not graph.child_nodes(parent):
+                                        leaf_nodes.append(parent)
+
+                        # Trigger rebuilds for our leaf node. Because all of our children
+                        # have been processed, build_deps and runtime_deps will be
+                        # completely filled in, and self.rebuild_list / self.reinstall_list
+                        # will tell us whether any of our children need to be rebuilt or
+                        # reinstalled.
+                        node_build_deps = build_deps.get(node, {})
+                        node_runtime_deps = runtime_deps.get(node, {})
+                        if self._trigger_rebuild(node, node_build_deps, node_runtime_deps):
+                                need_restart = True
+
+                return need_restart
+
+
 class _dynamic_depgraph_config(object):
 
         def __init__(self, depgraph, myparams, allow_backtracking, backtrack_parameters):
                 self.myparams = myparams.copy()
                 self._vdb_loaded = False
                 self._allow_backtracking = allow_backtracking
                 # Maps slot atom to package for each Package added to the graph.
                 self._slot_pkg_map = {}
                 # Maps nodes to the reasons they were selected for reinstallation.
                 self._reinstall_nodes = {}
@@ skipping 147 matching lines @@
         _dep_keys = ["DEPEND", "RDEPEND", "PDEPEND"]
         
         def __init__(self, settings, trees, myopts, myparams, spinner,
                 frozen_config=None, backtrack_parameters=BacktrackParameter(), allow_backtracking=False):
                 if frozen_config is None:
                         frozen_config = _frozen_depgraph_config(settings, trees,
                         myopts, spinner)
                 self._frozen_config = frozen_config
                 self._dynamic_config = _dynamic_depgraph_config(self, myparams,
                         allow_backtracking, backtrack_parameters)
+                self._rebuild = _rebuild_config(frozen_config, backtrack_parameters)
 
                 self._select_atoms = self._select_atoms_highest_available
                 self._select_package = self._select_pkg_highest_available
 
         def _load_vdb(self):
                 """
                 Load installed package metadata if appropriate. This used to be called
                 from the constructor, but that wasn't very nice since this procedure
                 is slow and it generates spinner output. So, now it's called on-demand
                 by various methods when necessary.
@@ skipping 345 matching lines @@
 
         def _add_dep(self, dep, allow_unsatisfied=False):
                 debug = "--debug" in self._frozen_config.myopts
                 buildpkgonly = "--buildpkgonly" in self._frozen_config.myopts
                 nodeps = "--nodeps" in self._frozen_config.myopts
                 deep = self._dynamic_config.myparams.get("deep", 0)
                 recurse = deep is True or dep.depth <= deep
                 if dep.blocker:
                         if not buildpkgonly and \
                                 not nodeps and \
+                                not dep.collapsed_priority.ignored and \
+                                not dep.collapsed_priority.optional and \
                                 dep.parent not in self._dynamic_config._slot_collision_nodes:
                                 if dep.parent.onlydeps:
                                         # It's safe to ignore blockers if the
                                         # parent is an --onlydeps node.
                                         return 1
                                 # The blocker applies to the root where
                                 # the parent is or will be installed.
                                 blocker = Blocker(atom=dep.atom,
                                         eapi=dep.parent.metadata["EAPI"],
                                         priority=dep.priority, root=dep.parent.root)
                                 self._dynamic_config._blocker_parents.add(blocker, dep.parent)
                         return 1
 
                 if dep.child is None:
                         dep_pkg, existing_node = self._select_package(dep.root, dep.atom,
                                 onlydeps=dep.onlydeps)
                 else:
                         # The caller has selected a specific package
                         # via self._minimize_packages().
                         dep_pkg = dep.child
                         existing_node = self._dynamic_config._slot_pkg_map[
                                 dep.root].get(dep_pkg.slot_atom)
 
                 if not dep_pkg:
-                        if dep.priority.optional:
-                                # This could be an unnecessary build-time dep
-                                # pulled in by --with-bdeps=y.
+                        if (dep.collapsed_priority.optional or
+                                dep.collapsed_priority.ignored):
+                                # This is an unnecessary build-time dep.
                                 return 1
                         if allow_unsatisfied:
                                 self._dynamic_config._unsatisfied_deps.append(dep)
                                 return 1
                         self._dynamic_config._unsatisfied_deps_for_display.append(
                                 ((dep.root, dep.atom), {"myparent":dep.parent}))
 
                         # The parent node should not already be in
                         # runtime_pkg_mask, since that would trigger an
                         # infinite backtracking loop.
@@ skipping 22 matching lines @@
                                                         msg.append("    parent: %s" % dep.parent)
                                                         msg.append("  priority: %s" % dep.priority)
                                                         msg.append("      root: %s" % dep.root)
                                                         msg.append("      atom: %s" % dep.atom)
                                                         msg.append("")
                                                         writemsg_level("".join("%s\n" % l for l in msg),
                                                                 noiselevel=-1, level=logging.DEBUG)
 
                         return 0
 
-                if not self._add_pkg(dep_pkg, dep):
+                self._rebuild.add(dep_pkg, dep)
+
+                if (not dep.collapsed_priority.ignored and
+                        not self._add_pkg(dep_pkg, dep)):
                         return 0
                 return 1
 
         def _check_slot_conflict(self, pkg, atom):
                 existing_node = self._dynamic_config._slot_pkg_map[pkg.root].get(pkg.slot_atom)
                 matches = None
                 if existing_node:
                         matches = pkg.cpv == existing_node.cpv
                         if pkg != existing_node and \
                                 atom is not None:
@@ skipping 349 matching lines @@
                 depkeys = ["DEPEND","RDEPEND","PDEPEND"]
                 for k in depkeys:
                         edepend[k] = metadata[k]
 
                 if not pkg.built and \
                         "--buildpkgonly" in self._frozen_config.myopts and \
                         "deep" not in self._dynamic_config.myparams:
                         edepend["RDEPEND"] = ""
                         edepend["PDEPEND"] = ""
 
+                ignore_build_time_deps = False
                 if pkg.built and not removal_action:
                         if self._frozen_config.myopts.get("--with-bdeps", "n") == "y":
                                 # Pull in build time deps as requested, but marked them as
                                 # "optional" since they are not strictly required. This allows
                                 # more freedom in the merge order calculation for solving
                                 # circular dependencies. Don't convert to PDEPEND since that
                                 # could make --with-bdeps=y less effective if it is used to
                                 # adjust merge order to prevent built_with_use() calls from
                                 # failing.
                                 pass
                         else:
-                                # built packages do not have build time dependencies.
-                                edepend["DEPEND"] = ""
+                                ignore_build_time_deps = True
 
                 if removal_action and self._frozen_config.myopts.get("--with-bdeps", "y") == "n":
-                        edepend["DEPEND"] = ""
+                        ignore_build_time_deps = True
 
                 if removal_action:
                         depend_root = myroot
                 else:
                         depend_root = "/"
                         root_deps = self._frozen_config.myopts.get("--root-deps")
                         if root_deps is not None:
                                 if root_deps is True:
                                         depend_root = myroot
                                 elif root_deps == "rdeps":
-                                        edepend["DEPEND"] = ""
+                                        ignore_build_time_deps = True
 
                 deps = (
                         (depend_root, edepend["DEPEND"],
-                                self._priority(buildtime=(not pkg.built),
-                                optional=pkg.built),
-                                pkg.built),
+                                self._priority(buildtime=True,
+                                optional=pkg.built,
+                                ignored=ignore_build_time_deps),
+                                pkg.built or ignore_build_time_deps),
                         (myroot, edepend["RDEPEND"],
                                 self._priority(runtime=True),
                                 False),
                         (myroot, edepend["PDEPEND"],
                                 self._priority(runtime_post=True),
                                 False)
                 )
 
                 debug = "--debug" in self._frozen_config.myopts
                 strict = mytype != "installed"
@@ skipping 103 matching lines @@
                         is_virt = hasattr(atom, '_orig_atom')
                         atom = getattr(atom, '_orig_atom', atom)
 
                         if ignore_blockers and atom.blocker:
                                 # For --with-bdeps, ignore build-time only blockers
                                 # that originate from built packages.
                                 continue
 
                         mypriority = dep_priority.copy()
                         if not atom.blocker:
+                                root_slot = (pkg.root, pkg.slot_atom)
                                 inst_pkgs = [inst_pkg for inst_pkg in vardb.match_pkgs(atom)
                                         if not reinstall_atoms.findAtomForPackage(inst_pkg,
                                                         modified_use=self._pkg_use_enabled(inst_pkg))]
                                 if inst_pkgs:
                                         for inst_pkg in inst_pkgs:
                                                 if self._pkg_visibility_check(inst_pkg):
                                                         # highest visible
                                                         mypriority.satisfied = inst_pkg
                                                         break
                                         if not mypriority.satisfied:
@@ skipping 89 matching lines @@
                                                                 mypriority.satisfied = inst_pkg
                                                                 break
                                                 if not mypriority.satisfied:
                                                         # none visible, so use highest
                                                         mypriority.satisfied = inst_pkgs[0]
 
                                 # Dependencies of virtuals are considered to have the
                                 # same depth as the virtual itself.
                                 dep = Dependency(atom=atom,
                                         blocker=atom.blocker, child=child, depth=virt_dep.depth,
-                                        parent=virt_pkg, priority=mypriority, root=dep_root)
+                                        parent=virt_pkg, priority=mypriority, root=dep_root,
+                                        collapsed_parent=pkg, collapsed_priority=dep_priority)
 
                                 ignored = False
                                 if not atom.blocker and \
                                         not recurse_satisfied and \
                                         mypriority.satisfied and \
                                         mypriority.satisfied.visible and \
                                         dep.child is not None and \
                                         not dep.child.installed and \
                                         self._dynamic_config._slot_pkg_map[dep.child.root].get(
                                         dep.child.slot_atom) is None:
@@ skipping 535 matching lines @@
         def _resolve(self, myfavorites):
                 """Given self._dynamic_config._initial_arg_list, pull in the root nodes, 
                 call self._creategraph to process theier deps and return 
                 a favorite list."""
                 debug = "--debug" in self._frozen_config.myopts
                 onlydeps = "--onlydeps" in self._frozen_config.myopts
                 myroot = self._frozen_config.target_root
                 pkgsettings = self._frozen_config.pkgsettings[myroot]
                 pprovideddict = pkgsettings.pprovideddict
                 virtuals = pkgsettings.getvirtuals()
-                for arg in self._expand_set_args(
-                        self._dynamic_config._initial_arg_list,
-                        add_to_digraph=True):
+                args = self._dynamic_config._initial_arg_list[:]
+                for root, atom in chain(self._rebuild.rebuild_list,
+                        self._rebuild.reinstall_list):
+                        args.append(AtomArg(arg=atom, atom=atom,
+                                root_config=self._frozen_config.roots[root]))
+                for arg in self._expand_set_args(args, add_to_digraph=True):
                         for atom in arg.pset.getAtoms():
                                 self._spinner_update()
                                 dep = Dependency(atom=atom, onlydeps=onlydeps,
                                         root=myroot, parent=arg)
                                 try:
                                         pprovided = pprovideddict.get(atom.cp)
                                         if pprovided and portage.match_from_list(atom, pprovided):
                                                 # A provided package has been specified on the command line.
                                                 self._dynamic_config._pprovided_args.append((arg, atom))
                                                 continue
@@ skipping 95 matching lines @@
                 if set(self._dynamic_config.digraph).intersection( \
                         self._dynamic_config._needed_unstable_keywords) or \
                         set(self._dynamic_config.digraph).intersection( \
                         self._dynamic_config._needed_use_config_changes) or \
                         set(self._dynamic_config.digraph).intersection( \
                         self._dynamic_config._needed_license_changes) :
                         #We failed if the user needs to change the configuration
                         self._dynamic_config._success_without_autounmask = True
                         return False, myfavorites
 
+                if self._rebuild.trigger_rebuilds():
+                        backtrack_infos = self._dynamic_config._backtrack_infos
+                        config = backtrack_infos.setdefault("config", {})
+                        config["rebuild_list"] = self._rebuild.rebuild_list
+                        config["reinstall_list"] = self._rebuild.reinstall_list
+                        self._dynamic_config._need_restart = True
+                        return False, myfavorites
+
                 # We're true here unless we are missing binaries.
                 return (True, myfavorites)
 
         def _set_args(self, args):
                 """
                 Create the "__non_set_args__" package set from atoms and packages given as
                 arguments. This method can be called multiple times if necessary.
                 The package selection cache is automatically invalidated, since
                 arguments influence package selections.
                 """
@@ skipping 469 matching lines @@
                                                 if not mreasons and \
                                                         not pkg.built and \
                                                         pkg.metadata["REQUIRED_USE"] and \
                                                         eapi_has_required_use(pkg.metadata["EAPI"]):
                                                         if not check_required_use(
                                                                 pkg.metadata["REQUIRED_USE"],
                                                                 self._pkg_use_enabled(pkg),
                                                                 pkg.iuse.is_valid_flag):
                                                                 required_use_unsatisfied.append(pkg)
                                                                 continue
-                                                if pkg.built and not mreasons:
+                                                root_slot = (pkg.root, pkg.slot_atom)
+                                                if pkg.built and root_slot in self._rebuild.rebuild_list:
+                                                        mreasons = ["need to rebuild from source"]
+                                                elif pkg.installed and root_slot in self._rebuild.reinstall_list:
+                                                        mreasons = ["need to rebuild from source"]
+                                                elif pkg.built and not mreasons:
                                                         mreasons = ["use flag configuration mismatch"]
                                         masked_packages.append(
                                                 (root_config, pkgsettings, cpv, repo, metadata, mreasons))
 
                 if check_backtrack:
                         if backtrack_mask:
                                 raise self._backtrack_mask()
                         else:
                                 return
 
@@ skipping 657 matching lines @@
                                                 continue
 
                                 # Ignore USE deps for the initial match since we want to
                                 # ensure that updates aren't missed solely due to the user's
                                 # USE configuration.
                                 for pkg in self._iter_match_pkgs(root_config, pkg_type, atom.without_use, 
                                         onlydeps=onlydeps):
                                         if pkg in self._dynamic_config._runtime_pkg_mask:
                                                 # The package has been masked by the backtracking logic
                                                 continue
+                                        root_slot = (pkg.root, pkg.slot_atom)
+                                        if pkg.built and root_slot in self._rebuild.rebuild_list:
+                                                continue
+                                        if (pkg.installed and
+                                                root_slot in self._rebuild.reinstall_list):
+                                                continue
 
                                         if not pkg.installed and \
                                                 self._frozen_config.excluded_pkgs.findAtomForPackage(pkg, \
                                                         modified_use=self._pkg_use_enabled(pkg)):
                                                 continue
 
                                         if built and not installed and nousepkg_atoms.findAtomForPackage(pkg, \
                                                 modified_use=self._pkg_use_enabled(pkg)):
                                                 break
 
@@ skipping 2953 matching lines @@
                 for msg_type, msgs in pkg.invalid.items():
                         for msg in msgs:
                                 mreasons.append(
                                         _MaskReason("invalid", "invalid: %s" % (msg,)))
 
         if not pkg.metadata["SLOT"]:
                 mreasons.append(
                         _MaskReason("invalid", "SLOT: undefined"))
 
         return mreasons