Update parallel_emerge to experimentally support Portage 2.1.9.

parallel_emerge

 #!/usr/bin/python2.6
 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Program to run emerge in parallel, for significant speedup.
 
 Usage:
  ./parallel_emerge [--board=BOARD] [--workon=PKGS] [--no-workon-deps]
                    [--force-remote-binary=PKGS] [emerge args] package
@@ skipping 64 matching lines @@
 # TODO(davidjames): Update Portage to expose public APIs for these features.
 from _emerge.actions import adjust_configs
 from _emerge.actions import load_emerge_config
 from _emerge.create_depgraph_params import create_depgraph_params
 from _emerge.depgraph import depgraph as emerge_depgraph
 from _emerge.depgraph import _frozen_depgraph_config
 from _emerge.main import emerge_main
 from _emerge.main import parse_opts
 from _emerge.Package import Package
 from _emerge.Scheduler import Scheduler
+from _emerge.SetArg import SetArg
 from _emerge.stdout_spinner import stdout_spinner
 import portage
 import portage.debug
 import portage.versions
 
+new_portage = not portage.VERSION.startswith("2.1.7.")
+if new_portage:
+  from portage._global_updates import _global_updates
+else:
+  from portage import _global_updates
 
 def Usage():
   """Print usage."""
   print "Usage:"
   print " ./parallel_emerge [--board=BOARD] [--workon=PKGS] [--no-workon-deps]"
   print "                   [--rebuild] [emerge args] package"
   print
   print "Packages specified as workon packages are always built from source."
   print "Unless --no-workon-deps is specified, packages that depend on these"
   print "packages are also built from source."
@@ skipping 283 matching lines @@
     settings, trees, mtimedb = load_emerge_config()
 
     # Check whether our portage tree is out of date. Typically, this happens
     # when you're setting up a new portage tree, such as in setup_board and
     # make_chroot. In that case, portage applies a bunch of global updates
     # here. Once the updates are finished, we need to commit any changes
     # that the global update made to our mtimedb, and reload the config.
     #
     # Portage normally handles this logic in emerge_main, but again, we can't
     # use that function here.
-    if portage._global_updates(trees, mtimedb["updates"]):
+    if _global_updates(trees, mtimedb["updates"]):
       mtimedb.commit()
       settings, trees, mtimedb = load_emerge_config(trees=trees)
 
     # Setup implied options. Portage normally handles this logic in
     # emerge_main.
     if "--buildpkgonly" in opts or "buildpkg" in settings.features:
       opts.setdefault("--buildpkg", True)
     if "--getbinpkgonly" in opts:
       opts.setdefault("--usepkgonly", True)
       opts.setdefault("--getbinpkg", True)
@@ skipping 27 matching lines @@
     adjust_configs(opts, trees)
 
     # Save our configuration so far in the emerge object
     emerge = self.emerge
     emerge.action, emerge.opts = action, opts
     emerge.settings, emerge.trees, emerge.mtimedb = settings, trees, mtimedb
     emerge.cmdline_packages = cmdline_packages
     root = settings["ROOT"]
     emerge.root_config = trees[root]["root_config"]
 
+    if new_portage and "--usepkg" in opts:
+      emerge.trees[root]["bintree"].populate("--getbinpkg" in opts)
+
   def CheckUseFlags(self, pkgsettings, cur_pkg, new_pkg):
     """Are the use flags in cur_pkg up to date?
 
     Return True if use flags are up to date; return false otherwise."""
 
     # cur_use:  The set of flags that were enabled when the package was
     #           first installed.
     # cur_iuse: The set of flags that affected the specified package
     #           when it was first installed.
     #
@@ skipping 40 matching lines @@
     # specifying --tree as well, we tell emerge that it's not safe to remove
     # uninstall instructions because we're planning on analyzing the output.
     emerge_opts["--tree"] = True
     emerge_opts["--emptytree"] = True
 
     # Set up parameters.
     params = create_depgraph_params(emerge_opts, emerge.action)
     frozen_config = _frozen_depgraph_config(emerge.settings, emerge.trees,
                                             emerge_opts, emerge.spinner)
     backtrack_max = emerge_opts.get('--backtrack', 5)
-    runtime_pkg_mask = None
+    backtrack_parameters = {}
     allow_backtracking = backtrack_max > 0
 
     # Try up to backtrack_max times to create a working depgraph. Each time we
     # run into a conflict, mask the offending package and try again.
     # TODO(davidjames): When Portage supports --force-remote-binary directly,
     #                   switch back to using the backtrack_depgraph function.
-    for i in range(backtrack_max + 1):
-      if i == backtrack_max:
-        # Looks like we hit the backtracking limit. Run the dependency
-        # calculation one more time (from scratch) to show the original error
-        # message.
-        runtime_pkg_mask = None
-        allow_backtracking = False
-
+    for i in range(backtrack_max + 2):
       # Create a depgraph object.
       depgraph = emerge_depgraph(emerge.settings, emerge.trees, emerge_opts,
           params, emerge.spinner, frozen_config=frozen_config,
           allow_backtracking=allow_backtracking,
-          runtime_pkg_mask=runtime_pkg_mask)
+          **backtrack_parameters)
 
       if i == 0:
         for cpv in self.forced_remote_binary_packages:
           # If --force-remote-binary was specified, we want to use this package
           # regardless of its use flags. Unfortunately, Portage doesn't support
           # ignoring use flags for just one package. To convince Portage to
           # install the package, we trick Portage into thinking the package has
           # the right use flags.
           # TODO(davidjames): Update Portage to support --force-remote-binary
           #                   directly, so that this hack isn't necessary.
           pkg = depgraph._pkg(cpv, "binary", emerge.root_config)
           pkgsettings = frozen_config.pkgsettings[pkg.root]
           pkgsettings.setcpv(pkg)
           pkg.use.enabled = pkgsettings["PORTAGE_USE"].split()
 
       # Select the packages we want.
       success, favorites = depgraph.select_files(packages)
       if success:
         break
-      elif depgraph.need_restart():
+      elif depgraph.need_restart() and i < backtrack_max:
         # Looks like we found some packages that can't be installed due to
         # conflicts. Try again, masking out the conflicting packages.
-        runtime_pkg_mask = depgraph.get_runtime_pkg_mask()
+        if new_portage:
+          backtrack_parameters = depgraph.get_backtrack_parameters()
+        else:
+          backtrack_parameters = {
+            'runtime_pkg_mask': depgraph.get_runtime_pkg_mask()
+          }
       elif allow_backtracking and i > 0:
-        # Looks like we tried all the possible combinations, and we still can't
-        # solve the graph. Stop backtracking, so that we can report an error
-        # message.
-        runtime_pkg_mask = None
+        # Looks like we can't solve the graph. Stop backtracking and report an
+        # error message.
+        backtrack_parameters.pop('runtime_pkg_mask', None)
         allow_backtracking = False
       else:
         break
 
     # Delete the --tree option, because we don't really want to display a
     # tree. We just wanted to get emerge to leave uninstall instructions on
     # the graph. Later, when we display the graph, we'll want standard-looking
     # output, so removing the --tree option is important.
     frozen_config.myopts.pop("--tree", None)
 
@@ skipping 75 matching lines @@
       #
       # Here's an example CPV: chromeos-base/power_manager-0.0.1-r1
       # Split up, this CPV would be:
       #   C -- Component: chromeos-base
       #   P -- Path:      power_manager
       #   V -- Version:   0.0.1-r1
       #
       # We just refer to CPVs as packages here because it's easier.
       deps = {}
       for child, priorities in node_deps[0].items():
+        if isinstance(child, SetArg): continue
         deps[str(child.cpv)] = dict(action=str(child.operation),
                                     deptype=str(priorities[-1]),
                                     deps={})
 
       # We've built our list of deps, so we can add our package to the tree.
       if isinstance(node, Package):
         deps_tree[str(node.cpv)] = dict(action=str(node.operation),
                                         deps=deps)
 
     emptytree = "--emptytree" in emerge.opts
@@ skipping 531 matching lines @@
           remote_ready = PrebuiltsReady(pkg, remote_pkgs, remote_ready_cache)
           if remote_ready and (local_mtime <= remote_mtime or pkg in cycles):
             MergeChildren(pkg, "mandatory")
           else:
             MergeChildren(pkg, "mandatory_source")
 
     def UsePrebuiltPackages(remote_pkgs):
       """Update packages that can use prebuilts to do so."""
       start = time.time()
 
-      # Build list of prebuilt packages
+      # Build list of prebuilt packages.
       prebuilt_pkgs = {}
       for pkg, info in deps_map.iteritems():
         if info and info["action"] == "merge":
           if (not info["force_remote_binary"] and info["mandatory_source"] or
               "--usepkgonly" not in emerge.opts and pkg not in remote_pkgs):
             continue
 
           db_pkg = emerge.depgraph._pkg(pkg, "binary", emerge.root_config)
           if info["force_remote_binary"]:
             # Undo our earlier hacks to the use flags so that the use flags
@@ skipping 181 matching lines @@
   It expects package identifiers to be passed to it via task_queue. When
   a task is started, it pushes the (target, filename) to the started_queue.
   The output is stored in filename. When a merge starts or finishes, we push
   EmergeJobState objects to the job_queue.
   """
 
   SetupWorkerSignals()
   settings, trees, mtimedb = emerge.settings, emerge.trees, emerge.mtimedb
   opts, spinner = emerge.opts, emerge.spinner
   opts["--nodeps"] = True
+  if new_portage:
+    # When Portage launches new processes, it goes on a rampage and closes all
+    # open file descriptors. Ask Portage not to do that, as it breaks us.
+    portage.process.get_open_fds = lambda: []
   while True:
     # Wait for a new item to show up on the queue. This is a blocking wait,
     # so if there's nothing to do, we just sit here.
     target = task_queue.get()
     if not target:
       # If target is None, this means that the main thread wants us to quit.
       # The other workers need to exit too, so we'll push the message back on
       # to the queue so they'll get it too.
       task_queue.put(target)
       return
     db_pkg = package_db[target]
     db_pkg.root_config = emerge.root_config
     install_list = [db_pkg]
     pkgname = db_pkg.pf
     output = tempfile.NamedTemporaryFile(prefix=pkgname + "-", delete=False)
     start_timestamp = time.time()
     job = EmergeJobState(target, pkgname, False, output.name, start_timestamp)
     job_queue.put(job)
     if "--pretend" in opts:
       retcode = 0
     else:
       save_stdout = sys.stdout
       save_stderr = sys.stderr
       try:
         sys.stdout = output
         sys.stderr = output
-        scheduler = Scheduler(settings, trees, mtimedb, opts, spinner,
-                              install_list, [], emerge.scheduler_graph)
+        if new_portage:
+          emerge.scheduler_graph.mergelist = install_list
+          scheduler = Scheduler(settings, trees, mtimedb, opts, spinner,
+              favorites=[], graph_config=emerge.scheduler_graph)
+        else:
+          scheduler = Scheduler(settings, trees, mtimedb, opts, spinner,
+              install_list, [], emerge.scheduler_graph)
         retcode = scheduler.merge()
       except Exception:
         traceback.print_exc(file=output)
         retcode = 1
       finally:
         sys.stdout = save_stdout
         sys.stderr = save_stderr
         output.close()
       if retcode is None:
         retcode = 0
@@ skipping 441 matching lines @@
   # need to upgrade the rest of the packages. So we'll go ahead and do that.
   if portage_upgrade:
     args = sys.argv[1:] + ["--nomerge=sys-apps/portage"]
     os.execvp(os.path.realpath(sys.argv[0]), args)
 
   print "Done"
   sys.exit(0)
 
 if __name__ == "__main__":
   main()