Update parallel_emerge to rebuild packages when use flags change.

parallel_emerge

 #!/usr/bin/python2.6
 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Program to run emerge in parallel, for significant speedup.
 
 Usage:
  ./parallel_emerge [--board=BOARD] [--workon=PKGS] [--no-workon-deps]
                    [emerge args] package"
@@ skipping 388 matching lines @@
     adjust_configs(opts, trees)
 
     # Save our configuration so far in the emerge object
     emerge = self.emerge
     emerge.action, emerge.opts = action, opts
     emerge.settings, emerge.trees, emerge.mtimedb = settings, trees, mtimedb
     emerge.cmdline_packages = cmdline_packages
     root = settings["ROOT"]
     emerge.root_config = trees[root]["root_config"]
 
+  def CheckUseFlags(self, pkgsettings, cur_pkg, new_pkg):
+    """Are the use flags in cur_pkg up to date?
+
+    Return True if use flags are up to date; return false otherwise."""
+
+    # cur_use:  The set of flags that were enabled when the package was
+    #           first installed.
+    # cur_iuse: The set of flags that affected the specified package
+    #           when it was first installed.
+    #
+    # The intersection of cur_use and cur_iuse provides the set of
+    # flags that were enabled and affected the specified package.
+    cur_use = cur_pkg.use.enabled
+    cur_iuse = cur_pkg.iuse.all
+
+    # Check whether this package is already installed with the right use
+    # flags.
+    #
+    # now_use: The set of flags (special and non-special) that are now
+    #          enabled for the specified package.
+    # now_iuse: The set of non-special flags that affect the specified
+    #           package.
+    now_use = new_pkg.use.enabled
+    now_iuse = new_pkg.iuse.all
+
+    # Tell portage we want to lookup the flags for the specified package
+    # in package.use.{mask,force}
+    pkgsettings.setcpv(new_pkg.cpv)
+
+    # Grab the set of flags that are requested for the given package.
+    # This includes flags that don't affect the package, and includes
+    # all sources of flags (e.g. USE environment variable, make.conf,
+    # make.defaults, package.use.{mask,force}, etc.).
+    #
+    # This is used by portage in the _reinstall_for_flags function below.
+    forced_flags = set(pkgsettings.useforce).union(pkgsettings.usemask)
+
+    depgraph = self.emerge.depgraph
+
+    flags = depgraph._reinstall_for_flags(forced_flags, cur_use,
+                                          cur_iuse, now_use, now_iuse)
+    return not flags
+
   def GenDependencyTree(self):
     """Get dependency tree info from emerge.
 
     TODO(): Update cros_extract_deps to also use this code.
     Returns:
       Dependency tree
     """
     start = time.time()
 
     # Setup emerge options.
@@ skipping 83 matching lines @@
         deps_tree[str(node.cpv)] = dict(action=str(node.operation),
                                         deps=deps)
 
     emptytree = "--emptytree" in emerge.opts
 
     # Ask portage for its install plan, so that we can only throw out
     # dependencies that portage throws out. Also, keep track of the old
     # versions of packages that we're either upgrading or replacing.
     #
     # The "vardb" is the database of installed packages.
-    vardb = emerge.trees[emerge.settings["ROOT"]]["vartree"].dbapi
+    root = emerge.settings["ROOT"]
+    frozen_config = depgraph._frozen_config
+    vardb = frozen_config.trees[root]["vartree"].dbapi
+    pkgsettings = frozen_config.pkgsettings[root]
     deps_info = {}
     for pkg in depgraph.altlist():
       if isinstance(pkg, Package):
         # If we're not in emptytree mode, and we're going to replace a package
         # that is already installed, then this operation is possibly optional.
         # ("--selective" mode is handled later, in RemoveInstalledPackages())
         optional = False
-        if not emptytree and vardb.cpv_exists(pkg.cpv):
-          optional = True
+        if not emptytree:
+          for vardb_pkg in vardb.match_pkgs(pkg.cpv):
+            if self.CheckUseFlags(pkgsettings, vardb_pkg, pkg):
+              optional = True
+              break
 
         # Add the package to our database.
         self.package_db[str(pkg.cpv)] = pkg
 
         # Save off info about the package
         deps_info[str(pkg.cpv)] = {"idx": len(deps_info),
                                    "optional": optional}
 
     # Delete the --tree option, because we don't really want to display a
     # tree. We just wanted to get emerge to leave uninstall instructions on
     # the graph. Later, when we display the graph, we'll want standard-looking
     # output, so removing the --tree option is important.
-    depgraph._frozen_config.myopts.pop("--tree", None)
+    frozen_config.myopts.pop("--tree", None)
 
     seconds = time.time() - start
     if "--quiet" not in emerge.opts:
       print "Deps calculated in %dm%.1fs" % (seconds / 60, seconds % 60)
 
     return deps_tree, deps_info
 
   def PrintTree(self, deps, depth=""):
     """Print the deps we have seen in the emerge output.
 
@@ skipping 23 matching lines @@
     # system at some point. Packages in final_db are either already
     # installed, or will be installed by the time we're done.
     final_db = emerge.depgraph._dynamic_config.mydbapi[root]
 
     # final_pkgs is a set of the packages we found in the final_db. These
     # packages are either already installed, or will be installed by the time
     # we're done. It's populated in BuildFinalPackageSet()
     final_pkgs = set()
 
     # These packages take a really long time to build, so, for expediency, we
-    # are blacklisting them from automatic rebuilds. Instead, these packages
-    # will only be rebuilt when they are explicitly rev'd.
+    # are blacklisting them from automatic rebuilds because one of their
+    # dependencies needs to be recompiled.
     rebuild_blacklist = set()
     for pkg in ("media-plugins/o3d", "dev-java/icedtea"):
       for match in final_db.match_pkgs(pkg):
         rebuild_blacklist.add(str(match.cpv))
 
     # deps_map is the actual dependency graph.
     #
     # Each package specifies a "needs" list and a "provides" list. The "needs"
     # list indicates which packages we depend on. The "provides" list
     # indicates the reverse dependencies -- what packages need us.
@@ skipping 99 matching lines @@
       # line are generally mandatory.
       #
       # There are a few exceptions to this rule:
       #  1. If the package isn't getting installed because it's in
       #     package.provided, it's not mandatory.
       #  2. If the package isn't getting installed because we're in --onlydeps
       #     mode, it's not mandatory either.
       if "--selective" in emerge.opts:
         selective = emerge.opts["--selective"] != "n"
       else:
-        selective = "--noreplace" in emerge.opts or "--update" in emerge.opts
+        selective = ("--noreplace" in emerge.opts or
+                     "--update" in emerge.opts or
+                     "--newuse" in emerge.opts or
+                     "--reinstall" in emerge.opts)
       onlydeps = "--onlydeps" in emerge.opts
       if not selective:
         for pkg in emerge.cmdline_packages:
           # If the package specified on the command-line is in our install
           # list, mark it as non-optional.
           found = False
           for db_pkg in final_db.match_pkgs(pkg):
             this_pkg = deps_info.get(db_pkg.cpv)
             if this_pkg:
               found = True
@@ skipping 94 matching lines @@
           if dep.find(needed) != -1:
             needed_pkg = dep
         if bad_pkg and needed_pkg:
           deps_map[needed_pkg]["provides"].add(bad_pkg)
           deps_map[bad_pkg]["needs"][needed_pkg] = "secret"
 
     def MergeChildren(pkg, merge_type):
       """Merge this package and all packages it provides."""
 
       this_pkg = deps_map[pkg]
-      if (this_pkg[merge_type] or pkg not in final_pkgs or
-          pkg in rebuild_blacklist):
+      if (this_pkg[merge_type] or pkg not in final_pkgs):
         return
 
       # Mark this package as non-optional
       deps_info[pkg]["optional"] = False
       this_pkg[merge_type] = True
-      for w in this_pkg["provides"]:
+      for w in this_pkg["provides"].difference(rebuild_blacklist):
         MergeChildren(w, merge_type)
 
       if this_pkg["action"] == "nomerge":
         this_pkg["action"] = "merge"
 
     def RemotePackageDatabase(binhost_url):
       """Grab the latest binary package database from the prebuilt server.
 
       We need to know the modification times of the prebuilt packages so that we
       know when it is OK to use these packages and when we should rebuild them
@@ skipping 119 matching lines @@
       # or source), make sure we also update the children. If a package is
       # built from source, all children must also be built from source.
       local_ready_cache, remote_ready_cache = {}, {}
       local_mtime_cache, remote_mtime_cache = {}, {}
       for pkg in final_pkgs:
         # If all the necessary local packages are ready, and their
         # modification times are in sync, we don't need to do anything here.
         local_mtime = LastModifiedWithDeps(pkg, local_pkgs, local_mtime_cache)
         local_ready = PrebuiltsReady(pkg, local_pkgs, local_ready_cache)
         if (not local_ready or local_pkgs.get(pkg, 0) < local_mtime and
-            pkg not in cycles):
+            pkg not in cycles and pkg not in rebuild_blacklist):
           # OK, at least one package is missing from the local cache or is
           # outdated. This means we're going to have to install the package
           # and all dependencies.
           #
           # If all the necessary remote packages are ready, and they're at
           # least as new as our local packages, we can install them.
           # Otherwise, we need to build from source.
           remote_mtime = LastModifiedWithDeps(pkg, remote_pkgs,
                                               remote_mtime_cache)
           remote_ready = PrebuiltsReady(pkg, remote_pkgs, remote_ready_cache)
           if remote_ready and (local_mtime <= remote_mtime or pkg in cycles):
             MergeChildren(pkg, "mandatory")
           else:
             MergeChildren(pkg, "mandatory_source")
 
     def UsePrebuiltPackages():
       """Update packages that can use prebuilts to do so."""
       start = time.time()
 
       # The bintree is the database of binary packages. By default, it's
       # empty.
       bintree = emerge.trees[root]["bintree"]
       bindb = bintree.dbapi
       root_config = emerge.root_config
+      pkgsettings = emerge.depgraph._frozen_config.pkgsettings[root]
       prebuilt_pkgs = {}
 
       # Populate the DB with packages
       bintree.populate("--getbinpkg" in emerge.opts,
                        "--getbinpkgonly" in emerge.opts)
 
-      # Update packages that can use prebuilts to do so.
+      # Build list of prebuilt packages
       for pkg, info in deps_map.iteritems():
         if info and not info["mandatory_source"] and info["action"] == "merge":
           db_keys = list(bindb._aux_cache_keys)
           try:
             db_vals = bindb.aux_get(pkg, db_keys + ["MTIME"])
           except KeyError:
             # No binary package
             continue
           mtime = int(db_vals.pop() or 0)
           metadata = zip(db_keys, db_vals)
           db_pkg = Package(built=True, cpv=pkg, installed=False,
                            metadata=metadata, onlydeps=False, mtime=mtime,
                            operation="merge", root_config=root_config,
                            type_name="binary")
-          self.package_db[pkg] = db_pkg
+          prebuilt_pkgs[pkg] = db_pkg
+
+      # Calculate what packages need to be rebuilt due to changes in use flags.
+      for pkg, db_pkg in prebuilt_pkgs.iteritems():
+        db_pkg_src = self.package_db[pkg]
+        if not self.CheckUseFlags(pkgsettings, db_pkg, db_pkg_src):
+          MergeChildren(pkg, "mandatory_source")
+
+      # Convert eligible packages to binaries.
+      for pkg, info in deps_map.iteritems():
+        if (info and not info["mandatory_source"] and
+            info["action"] == "merge" and pkg in prebuilt_pkgs):
+          self.package_db[pkg] = prebuilt_pkgs[pkg]
 
       seconds = time.time() - start
       if "--quiet" not in emerge.opts:
         print "Prebuilt DB populated in %dm%.1fs" % (seconds / 60, seconds % 60)
 
       return prebuilt_pkgs
 
     def AddRemainingPackages():
       """Fill in packages that don't have entries in the package db.
 
@@ skipping 565 matching lines @@
       world_set.update(new_world_pkgs)
 
   # Update environment (library cache, symlinks, etc.)
   if deps.board and "--pretend" not in emerge.opts:
     portage.env_update()
 
   print "Done"
 
 if __name__ == "__main__":
   main()