Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(524)

Unified Diff: parallel_emerge

Issue 6246058: Update links to point at new location of parallel_emerge. (Closed) Base URL: http://git.chromium.org/git/crosutils.git@master
Patch Set: Delete parallel_emerge Created 9 years, 11 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « mod_image_for_test.sh ('k') | no next file » | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: parallel_emerge
diff --git a/parallel_emerge b/parallel_emerge
deleted file mode 100755
index c6012489b6c92319027b534230c42ef7fe312784..0000000000000000000000000000000000000000
--- a/parallel_emerge
+++ /dev/null
@@ -1,1903 +0,0 @@
-#!/usr/bin/python2.6
-# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Program to run emerge in parallel, for significant speedup.
-
-Usage:
- ./parallel_emerge [--board=BOARD] [--workon=PKGS] [--no-workon-deps]
- [--force-remote-binary=PKGS] [emerge args] package
-
-Basic operation:
- Runs 'emerge -p --debug' to display dependencies, and stores a
- dependency graph. All non-blocked packages are launched in parallel,
- as 'emerge --nodeps package' with any blocked packages being emerged
- immediately upon deps being met.
-
- For this to work effectively, /usr/lib/portage/pym/portage/locks.py
- must be stubbed out, preventing portage from slowing itself with
- unneccesary locking, as this script ensures that emerge is run in such
- a way that common resources are never in conflict. This is controlled
- by an environment variable PORTAGE_LOCKS set in parallel emerge
- subprocesses.
-
- Parallel Emerge unlocks two things during operation, here's what you
- must do to keep this safe:
- * Storage dir containing binary packages. - Don't emerge new
- packages while installing the existing ones.
- * Portage database - You must not examine deps while modifying the
- database. Therefore you may only parallelize "-p" read only access,
- or "--nodeps" write only access.
- Caveats:
- * Some ebuild packages have incorrectly specified deps, and running
- them in parallel is more likely to bring out these failures.
- * Some ebuilds (especially the build part) have complex dependencies
- that are not captured well by this script (it may be necessary to
- install an old package to build, but then install a newer version
- of the same package for a runtime dep).
-"""
-
-import codecs
-import copy
-import errno
-import multiprocessing
-import os
-import Queue
-import shlex
-import signal
-import sys
-import tempfile
-import time
-import traceback
-import urllib2
-
-# If PORTAGE_USERNAME isn't specified, scrape it from the $HOME variable. On
-# Chromium OS, the default "portage" user doesn't have the necessary
-# permissions. It'd be easier if we could default to $USERNAME, but $USERNAME
-# is "root" here because we get called through sudo.
-#
-# We need to set this before importing any portage modules, because portage
-# looks up "PORTAGE_USERNAME" at import time.
-#
-# NOTE: .bashrc sets PORTAGE_USERNAME = $USERNAME, so most people won't
-# encounter this case unless they have an old chroot or blow away the
-# environment by running sudo without the -E specifier.
-if "PORTAGE_USERNAME" not in os.environ:
- homedir = os.environ.get("HOME")
- if homedir:
- os.environ["PORTAGE_USERNAME"] = os.path.basename(homedir)
-
-# Portage doesn't expose dependency trees in its public API, so we have to
-# make use of some private APIs here. These modules are found under
-# /usr/lib/portage/pym/.
-#
-# TODO(davidjames): Update Portage to expose public APIs for these features.
-from _emerge.actions import adjust_configs
-from _emerge.actions import load_emerge_config
-from _emerge.create_depgraph_params import create_depgraph_params
-from _emerge.depgraph import depgraph as emerge_depgraph
-from _emerge.depgraph import _frozen_depgraph_config
-from _emerge.main import emerge_main
-from _emerge.main import parse_opts
-from _emerge.Package import Package
-from _emerge.Scheduler import Scheduler
-from _emerge.SetArg import SetArg
-from _emerge.stdout_spinner import stdout_spinner
-import portage
-import portage.debug
-import portage.versions
-
-new_portage = not portage.VERSION.startswith("2.1.7.")
-if new_portage:
- from portage._global_updates import _global_updates
-else:
- from portage import _global_updates
-
-def Usage():
- """Print usage."""
- print "Usage:"
- print " ./parallel_emerge [--board=BOARD] [--workon=PKGS] [--no-workon-deps]"
- print " [--rebuild] [emerge args] package"
- print
- print "Packages specified as workon packages are always built from source."
- print "Unless --no-workon-deps is specified, packages that depend on these"
- print "packages are also built from source."
- print
- print "The --workon argument is mainly useful when you want to build and"
- print "install packages that you are working on unconditionally, but do not"
- print "to have to rev the package to indicate you want to build it from"
- print "source. The build_packages script will automatically supply the"
- print "workon argument to emerge, ensuring that packages selected using"
- print "cros-workon are rebuilt."
- print
- print "The --rebuild option rebuilds packages whenever their dependencies"
- print "are changed. This ensures that your build is correct."
- sys.exit(1)
-
-
-# These are dependencies that are not specified in the package,
-# but will prevent the package from installing.
-secret_deps = {}
-
-# Global start time
-GLOBAL_START = time.time()
-
-
-class EmergeData(object):
- """This simple struct holds various emerge variables.
-
- This struct helps us easily pass emerge variables around as a unit.
- These variables are used for calculating dependencies and installing
- packages.
- """
-
- __slots__ = ["action", "cmdline_packages", "depgraph", "mtimedb", "opts",
- "root_config", "scheduler_graph", "settings", "spinner",
- "trees"]
-
- def __init__(self):
- # The action the user requested. If the user is installing packages, this
- # is None. If the user is doing anything other than installing packages,
- # this will contain the action name, which will map exactly to the
- # long-form name of the associated emerge option.
- #
- # Example: If you call parallel_emerge --unmerge package, the action name
- # will be "unmerge"
- self.action = None
-
- # The list of packages the user passed on the command-line.
- self.cmdline_packages = None
-
- # The emerge dependency graph. It'll contain all the packages involved in
- # this merge, along with their versions.
- self.depgraph = None
-
- # A dict of the options passed to emerge. This dict has been cleaned up
- # a bit by parse_opts, so that it's a bit easier for the emerge code to
- # look at the options.
- #
- # Emerge takes a few shortcuts in its cleanup process to make parsing of
- # the options dict easier. For example, if you pass in "--usepkg=n", the
- # "--usepkg" flag is just left out of the dictionary altogether. Because
- # --usepkg=n is the default, this makes parsing easier, because emerge
- # can just assume that if "--usepkg" is in the dictionary, it's enabled.
- #
- # These cleanup processes aren't applied to all options. For example, the
- # --with-bdeps flag is passed in as-is. For a full list of the cleanups
- # applied by emerge, see the parse_opts function in the _emerge.main
- # package.
- self.opts = None
-
- # A dictionary used by portage to maintain global state. This state is
- # loaded from disk when portage starts up, and saved to disk whenever we
- # call mtimedb.commit().
- #
- # This database contains information about global updates (i.e., what
- # version of portage we have) and what we're currently doing. Portage
- # saves what it is currently doing in this database so that it can be
- # resumed when you call it with the --resume option.
- #
- # parallel_emerge does not save what it is currently doing in the mtimedb,
- # so we do not support the --resume option.
- self.mtimedb = None
-
- # The portage configuration for our current root. This contains the portage
- # settings (see below) and the three portage trees for our current root.
- # (The three portage trees are explained below, in the documentation for
- # the "trees" member.)
- self.root_config = None
-
- # The scheduler graph is used by emerge to calculate what packages to
- # install. We don't actually install any deps, so this isn't really used,
- # but we pass it in to the Scheduler object anyway.
- self.scheduler_graph = None
-
- # Portage settings for our current session. Most of these settings are set
- # in make.conf inside our current install root.
- self.settings = None
-
- # The spinner, which spews stuff to stdout to indicate that portage is
- # doing something. We maintain our own spinner, so we set the portage
- # spinner to "silent" mode.
- self.spinner = None
-
- # The portage trees. There are separate portage trees for each root. To get
- # the portage tree for the current root, you can look in self.trees[root],
- # where root = self.settings["ROOT"].
- #
- # In each root, there are three trees: vartree, porttree, and bintree.
- # - vartree: A database of the currently-installed packages.
- # - porttree: A database of ebuilds, that can be used to build packages.
- # - bintree: A database of binary packages.
- self.trees = None
-
-
-class DepGraphGenerator(object):
- """Grab dependency information about packages from portage.
-
- Typical usage:
- deps = DepGraphGenerator()
- deps.Initialize(sys.argv[1:])
- deps_tree, deps_info = deps.GenDependencyTree()
- deps_graph = deps.GenDependencyGraph(deps_tree, deps_info)
- deps.PrintTree(deps_tree)
- PrintDepsMap(deps_graph)
- """
-
- __slots__ = ["board", "emerge", "mandatory_source", "no_workon_deps",
- "nomerge", "package_db", "rebuild", "show_output",
- "force_remote_binary", "forced_remote_binary_packages"]
-
- def __init__(self):
- self.board = None
- self.emerge = EmergeData()
- self.mandatory_source = set()
- self.no_workon_deps = False
- self.nomerge = set()
- self.package_db = {}
- self.rebuild = False
- self.show_output = False
- self.force_remote_binary = set()
- self.forced_remote_binary_packages = set()
-
- def ParseParallelEmergeArgs(self, argv):
- """Read the parallel emerge arguments from the command-line.
-
- We need to be compatible with emerge arg format. We scrape arguments that
- are specific to parallel_emerge, and pass through the rest directly to
- emerge.
- Args:
- argv: arguments list
- Returns:
- Arguments that don't belong to parallel_emerge
- """
- emerge_args = []
- for arg in argv:
- # Specifically match arguments that are specific to parallel_emerge, and
- # pass through the rest.
- if arg.startswith("--board="):
- self.board = arg.replace("--board=", "")
- elif arg.startswith("--workon="):
- workon_str = arg.replace("--workon=", "")
- package_list = shlex.split(" ".join(shlex.split(workon_str)))
- self.mandatory_source.update(package_list)
- elif arg.startswith("--force-remote-binary="):
- force_remote_binary = arg.replace("--force-remote-binary=", "")
- force_remote_binary = \
- shlex.split(" ".join(shlex.split(force_remote_binary)))
- self.force_remote_binary.update(force_remote_binary)
- elif arg.startswith("--nomerge="):
- nomerge_str = arg.replace("--nomerge=", "")
- package_list = shlex.split(" ".join(shlex.split(nomerge_str)))
- self.nomerge.update(package_list)
- elif arg == "--no-workon-deps":
- self.no_workon_deps = True
- elif arg == "--rebuild":
- self.rebuild = True
- elif arg == "--show-output":
- self.show_output = True
- else:
- # Not one of our options, so pass through to emerge.
- emerge_args.append(arg)
-
- if self.rebuild:
- if self.no_workon_deps:
- print "--rebuild is not compatible with --no-workon-deps"
- sys.exit(1)
-
- return emerge_args
-
- def Initialize(self, args):
- """Initializer. Parses arguments and sets up portage state."""
-
- # Parse and strip out args that are just intended for parallel_emerge.
- emerge_args = self.ParseParallelEmergeArgs(args)
-
- # Setup various environment variables based on our current board. These
- # variables are normally setup inside emerge-${BOARD}, but since we don't
- # call that script, we have to set it up here. These variables serve to
- # point our tools at /build/BOARD and to setup cross compiles to the
- # appropriate board as configured in toolchain.conf.
- if self.board:
- os.environ["PORTAGE_CONFIGROOT"] = "/build/" + self.board
- os.environ["PORTAGE_SYSROOT"] = "/build/" + self.board
- os.environ["SYSROOT"] = "/build/" + self.board
- scripts_dir = os.path.dirname(os.path.realpath(__file__))
- # Strip the variant out of the board name to look for the toolchain. This
- # is similar to what setup_board does.
- board_no_variant = self.board.split('_')[0]
- public_toolchain_path = ("%s/../overlays/overlay-%s/toolchain.conf" %
- (scripts_dir, board_no_variant))
- private_toolchain_path = (
- "%s/../private-overlays/overlay-%s-private/toolchain.conf" %
- (scripts_dir, board_no_variant))
- if os.path.isfile(public_toolchain_path):
- toolchain_path = public_toolchain_path
- elif os.path.isfile(private_toolchain_path):
- toolchain_path = private_toolchain_path
- else:
- print "Not able to locate toolchain.conf in board overlays"
- sys.exit(1)
-
- f = open(toolchain_path)
- os.environ["CHOST"] = f.readline().strip()
- f.close()
-
- # Although CHROMEOS_ROOT isn't specific to boards, it's normally setup
- # inside emerge-${BOARD}, so we set it up here for compatibility. It
- # will be going away soon as we migrate to CROS_WORKON_SRCROOT.
- os.environ.setdefault("CHROMEOS_ROOT", os.environ["HOME"] + "/trunk")
-
- # Turn off interactive delays
- os.environ["EBEEP_IGNORE"] = "1"
- os.environ["EPAUSE_IGNORE"] = "1"
- os.environ["UNMERGE_DELAY"] = "0"
-
- # Parse the emerge options.
- action, opts, cmdline_packages = parse_opts(emerge_args)
-
- # If we're installing to the board, we want the --root-deps option so that
- # portage will install the build dependencies to that location as well.
- if self.board:
- opts.setdefault("--root-deps", True)
-
- # Set environment variables based on options. Portage normally sets these
- # environment variables in emerge_main, but we can't use that function,
- # because it also does a bunch of other stuff that we don't want.
- # TODO(davidjames): Patch portage to move this logic into a function we can
- # reuse here.
- if "--debug" in opts:
- os.environ["PORTAGE_DEBUG"] = "1"
- if "--config-root" in opts:
- os.environ["PORTAGE_CONFIGROOT"] = opts["--config-root"]
- if "--root" in opts:
- os.environ["ROOT"] = opts["--root"]
- if "--accept-properties" in opts:
- os.environ["ACCEPT_PROPERTIES"] = opts["--accept-properties"]
-
- # Portage has two flags for doing collision protection: collision-protect
- # and protect-owned. The protect-owned feature is enabled by default and
- # is quite useful: it checks to make sure that we don't have multiple
- # packages that own the same file. The collision-protect feature is more
- # strict, and less useful: it fails if it finds a conflicting file, even
- # if that file was created by an earlier ebuild that failed to install.
- #
- # We want to disable collision-protect here because we don't handle
- # failures during the merge step very well. Sometimes we leave old files
- # lying around and they cause problems, so for now we disable the flag.
- # TODO(davidjames): Look for a better solution.
- features = os.environ.get("FEATURES", "") + " -collision-protect"
-
- # If we're installing packages to the board, and we're not using the
- # official flag, we can enable the following optimizations:
- # 1) Don't lock during install step. This allows multiple packages to be
- # installed at once. This is safe because our board packages do not
- # muck with each other during the post-install step.
- # 2) Don't update the environment until the end of the build. This is
- # safe because board packages don't need to run during the build --
- # they're cross-compiled, so our CPU architecture doesn't support them
- # anyway.
- if self.board and os.environ.get("CHROMEOS_OFFICIAL") != "1":
- os.environ.setdefault("PORTAGE_LOCKS", "false")
- features = features + " no-env-update"
-
- os.environ["FEATURES"] = features
-
- # Now that we've setup the necessary environment variables, we can load the
- # emerge config from disk.
- settings, trees, mtimedb = load_emerge_config()
-
- # Check whether our portage tree is out of date. Typically, this happens
- # when you're setting up a new portage tree, such as in setup_board and
- # make_chroot. In that case, portage applies a bunch of global updates
- # here. Once the updates are finished, we need to commit any changes
- # that the global update made to our mtimedb, and reload the config.
- #
- # Portage normally handles this logic in emerge_main, but again, we can't
- # use that function here.
- if _global_updates(trees, mtimedb["updates"]):
- mtimedb.commit()
- settings, trees, mtimedb = load_emerge_config(trees=trees)
-
- # Setup implied options. Portage normally handles this logic in
- # emerge_main.
- if "--buildpkgonly" in opts or "buildpkg" in settings.features:
- opts.setdefault("--buildpkg", True)
- if "--getbinpkgonly" in opts:
- opts.setdefault("--usepkgonly", True)
- opts.setdefault("--getbinpkg", True)
- if "getbinpkg" in settings.features:
- # Per emerge_main, FEATURES=getbinpkg overrides --getbinpkg=n
- opts["--getbinpkg"] = True
- if "--getbinpkg" in opts or "--usepkgonly" in opts:
- opts.setdefault("--usepkg", True)
- if "--fetch-all-uri" in opts:
- opts.setdefault("--fetchonly", True)
- if "--skipfirst" in opts:
- opts.setdefault("--resume", True)
- if "--buildpkgonly" in opts:
- # --buildpkgonly will not merge anything, so it overrides all binary
- # package options.
- for opt in ("--getbinpkg", "--getbinpkgonly",
- "--usepkg", "--usepkgonly"):
- opts.pop(opt, None)
- if (settings.get("PORTAGE_DEBUG", "") == "1" and
- "python-trace" in settings.features):
- portage.debug.set_trace(True)
-
- # Complain about unsupported options
- for opt in ("--ask", "--ask-enter-invalid", "--complete-graph",
- "--resume", "--skipfirst"):
- if opt in opts:
- print "%s is not supported by parallel_emerge" % opt
- sys.exit(1)
-
- # Make emerge specific adjustments to the config (e.g. colors!)
- adjust_configs(opts, trees)
-
- # Save our configuration so far in the emerge object
- emerge = self.emerge
- emerge.action, emerge.opts = action, opts
- emerge.settings, emerge.trees, emerge.mtimedb = settings, trees, mtimedb
- emerge.cmdline_packages = cmdline_packages
- root = settings["ROOT"]
- emerge.root_config = trees[root]["root_config"]
-
- if new_portage and "--usepkg" in opts:
- emerge.trees[root]["bintree"].populate("--getbinpkg" in opts)
-
- def CheckUseFlags(self, pkgsettings, cur_pkg, new_pkg):
- """Are the use flags in cur_pkg up to date?
-
- Return True if use flags are up to date; return false otherwise."""
-
- # cur_use: The set of flags that were enabled when the package was
- # first installed.
- # cur_iuse: The set of flags that affected the specified package
- # when it was first installed.
- #
- # The intersection of cur_use and cur_iuse provides the set of
- # flags that were enabled and affected the specified package.
- cur_use = cur_pkg.use.enabled
- cur_iuse = cur_pkg.iuse.all
-
- # Check whether this package is already installed with the right use
- # flags.
- #
- # now_use: The set of flags (special and non-special) that are now
- # enabled for the specified package.
- # now_iuse: The set of non-special flags that affect the specified
- # package.
- now_use = new_pkg.use.enabled
- now_iuse = new_pkg.iuse.all
-
- # Tell portage we want to lookup the flags for the specified package
- # in package.use.{mask,force}
- pkgsettings.setcpv(new_pkg.cpv)
-
- # Grab the set of flags that are requested for the given package.
- # This includes flags that don't affect the package, and includes
- # all sources of flags (e.g. USE environment variable, make.conf,
- # make.defaults, package.use.{mask,force}, etc.).
- #
- # This is used by portage in the _reinstall_for_flags function below.
- forced_flags = set(pkgsettings.useforce).union(pkgsettings.usemask)
-
- depgraph = self.emerge.depgraph
- flags = depgraph._reinstall_for_flags(forced_flags, cur_use,
- cur_iuse, now_use, now_iuse)
- return not flags
-
- def CreateDepgraph(self, emerge, packages):
- """Create an emerge depgraph object."""
- # Setup emerge options.
- emerge_opts = emerge.opts.copy()
-
- # Enable --emptytree so that we get the full tree, which we need for
- # dependency analysis. By default, with this option, emerge optimizes
- # the graph by removing uninstall instructions from the graph. By
- # specifying --tree as well, we tell emerge that it's not safe to remove
- # uninstall instructions because we're planning on analyzing the output.
- emerge_opts["--tree"] = True
- emerge_opts["--emptytree"] = True
-
- # Set up parameters.
- params = create_depgraph_params(emerge_opts, emerge.action)
- frozen_config = _frozen_depgraph_config(emerge.settings, emerge.trees,
- emerge_opts, emerge.spinner)
- backtrack_max = emerge_opts.get('--backtrack', 5)
- backtrack_parameters = {}
- allow_backtracking = backtrack_max > 0
-
- # Try up to backtrack_max times to create a working depgraph. Each time we
- # run into a conflict, mask the offending package and try again.
- # TODO(davidjames): When Portage supports --force-remote-binary directly,
- # switch back to using the backtrack_depgraph function.
- for i in range(backtrack_max + 2):
- # Create a depgraph object.
- depgraph = emerge_depgraph(emerge.settings, emerge.trees, emerge_opts,
- params, emerge.spinner, frozen_config=frozen_config,
- allow_backtracking=allow_backtracking,
- **backtrack_parameters)
-
- if i == 0:
- for cpv in self.forced_remote_binary_packages:
- # If --force-remote-binary was specified, we want to use this package
- # regardless of its use flags. Unfortunately, Portage doesn't support
- # ignoring use flags for just one package. To convince Portage to
- # install the package, we trick Portage into thinking the package has
- # the right use flags.
- # TODO(davidjames): Update Portage to support --force-remote-binary
- # directly, so that this hack isn't necessary.
- pkg = depgraph._pkg(cpv, "binary", emerge.root_config)
- pkgsettings = frozen_config.pkgsettings[pkg.root]
- pkgsettings.setcpv(pkg)
- pkg.use.enabled = pkgsettings["PORTAGE_USE"].split()
-
- # Select the packages we want.
- success, favorites = depgraph.select_files(packages)
- if success:
- break
- elif depgraph.need_restart() and i < backtrack_max:
- # Looks like we found some packages that can't be installed due to
- # conflicts. Try again, masking out the conflicting packages.
- if new_portage:
- backtrack_parameters = depgraph.get_backtrack_parameters()
- else:
- backtrack_parameters = {
- 'runtime_pkg_mask': depgraph.get_runtime_pkg_mask()
- }
- elif allow_backtracking and i > 0:
- # Looks like we can't solve the graph. Stop backtracking and report an
- # error message.
- backtrack_parameters.pop('runtime_pkg_mask', None)
- allow_backtracking = False
- else:
- break
-
- # Delete the --tree option, because we don't really want to display a
- # tree. We just wanted to get emerge to leave uninstall instructions on
- # the graph. Later, when we display the graph, we'll want standard-looking
- # output, so removing the --tree option is important.
- frozen_config.myopts.pop("--tree", None)
-
- emerge.depgraph = depgraph
-
- # Is it impossible to honor the user's request? Bail!
- if not success:
- depgraph.display_problems()
- sys.exit(1)
-
- def GenDependencyTree(self, remote_pkgs):
- """Get dependency tree info from emerge.
-
- TODO(): Update cros_extract_deps to also use this code.
- Returns:
- Dependency tree
- """
- start = time.time()
-
- emerge = self.emerge
-
- # Create a list of packages to merge
- packages = set(emerge.cmdline_packages[:])
- if self.mandatory_source:
- packages.update(self.mandatory_source)
- if self.force_remote_binary:
- forced_pkgs = {}
- for pkg in remote_pkgs:
- category, pkgname, _, _ = portage.catpkgsplit(pkg)
- full_pkgname = "%s/%s" % (category, pkgname)
- if (pkgname in self.force_remote_binary or
- full_pkgname in self.force_remote_binary):
- forced_pkgs.setdefault(full_pkgname, []).append(pkg)
-
- # Add forced binary packages to the dependency list. This is necessary
- # to ensure that the install plan contains the right package.
- #
- # Putting the forced binary package at the beginning of the list is an
- # optimization that helps avoid unnecessary backtracking (e.g., if
- # Portage first selects the wrong version, and then backtracks later, it
- # takes a bit longer and uses up an unnecessary backtrack iteration.)
- packages = list(packages)
- for pkgs in forced_pkgs.values():
- forced_package = portage.versions.best(pkgs)
- packages.insert(0, "=%s" % forced_package)
- self.forced_remote_binary_packages.add(forced_package)
-
- # Tell emerge to be quiet. We print plenty of info ourselves so we don't
- # need any extra output from portage.
- portage.util.noiselimit = -1
-
- # My favorite feature: The silent spinner. It doesn't spin. Ever.
- # I'd disable the colors by default too, but they look kind of cool.
- emerge.spinner = stdout_spinner()
- emerge.spinner.update = emerge.spinner.update_quiet
-
- if "--quiet" not in emerge.opts:
- print "Calculating deps..."
-
- self.CreateDepgraph(emerge, packages)
- depgraph = emerge.depgraph
-
- # Build our own tree from the emerge digraph.
- deps_tree = {}
- digraph = depgraph._dynamic_config.digraph
- for node, node_deps in digraph.nodes.items():
- # Calculate dependency packages that need to be installed first. Each
- # child on the digraph is a dependency. The "operation" field specifies
- # what we're doing (e.g. merge, uninstall, etc.). The "priorities" array
- # contains the type of dependency (e.g. build, runtime, runtime_post,
- # etc.)
- #
- # Emerge itself actually treats some dependencies as "soft" dependencies
- # and sometimes ignores them. We don't do that -- we honor all
- # dependencies unless we're forced to prune them because they're cyclic.
- #
- # Portage refers to the identifiers for packages as a CPV. This acronym
- # stands for Component/Path/Version.
- #
- # Here's an example CPV: chromeos-base/power_manager-0.0.1-r1
- # Split up, this CPV would be:
- # C -- Component: chromeos-base
- # P -- Path: power_manager
- # V -- Version: 0.0.1-r1
- #
- # We just refer to CPVs as packages here because it's easier.
- deps = {}
- for child, priorities in node_deps[0].items():
- if isinstance(child, SetArg): continue
- deps[str(child.cpv)] = dict(action=str(child.operation),
- deptype=str(priorities[-1]),
- deps={})
-
- # We've built our list of deps, so we can add our package to the tree.
- if isinstance(node, Package):
- deps_tree[str(node.cpv)] = dict(action=str(node.operation),
- deps=deps)
-
- emptytree = "--emptytree" in emerge.opts
-
- # Ask portage for its install plan, so that we can only throw out
- # dependencies that portage throws out. Also, keep track of the old
- # versions of packages that we're either upgrading or replacing.
- #
- # The "vardb" is the database of installed packages.
- root = emerge.settings["ROOT"]
- frozen_config = depgraph._frozen_config
- vardb = frozen_config.trees[root]["vartree"].dbapi
- pkgsettings = frozen_config.pkgsettings[root]
-
- deps_info = {}
- for pkg in depgraph.altlist():
- if isinstance(pkg, Package):
- # If we're not using --force-remote-binary, check what flags are being
- # used by the real package.
- if "--usepkgonly" not in emerge.opts:
- try:
- pkg = emerge.depgraph._pkg(pkg.cpv, "ebuild", emerge.root_config)
- except portage.exception.PackageNotFound:
- # This is a --force-remote-binary package.
- pass
- self.package_db[pkg.cpv] = pkg
-
- # If we're not in emptytree mode, and we're going to replace a package
- # that is already installed, then this operation is possibly optional.
- # ("--selective" mode is handled later, in RemoveInstalledPackages())
- optional = False
- if not emptytree:
- for vardb_pkg in vardb.match_pkgs(pkg.cpv):
- if self.CheckUseFlags(pkgsettings, vardb_pkg, pkg):
- optional = True
- break
-
- # Save off info about the package
- deps_info[str(pkg.cpv)] = {"idx": len(deps_info),
- "optional": optional}
-
- seconds = time.time() - start
- if "--quiet" not in emerge.opts:
- print "Deps calculated in %dm%.1fs" % (seconds / 60, seconds % 60)
-
- return deps_tree, deps_info
-
- def PrintTree(self, deps, depth=""):
- """Print the deps we have seen in the emerge output.
-
- Args:
- deps: Dependency tree structure.
- depth: Allows printing the tree recursively, with indentation.
- """
- for entry in sorted(deps):
- action = deps[entry]["action"]
- print "%s %s (%s)" % (depth, entry, action)
- self.PrintTree(deps[entry]["deps"], depth=depth + " ")
-
- def RemotePackageDatabase(self, binhost_url):
- """Grab the latest binary package database from the prebuilt server.
-
- We need to know the modification times of the prebuilt packages so that we
- know when it is OK to use these packages and when we should rebuild them
- instead.
-
- Args:
- binhost_url: Base URL of remote packages (PORTAGE_BINHOST).
-
- Returns:
- A dict mapping package identifiers to modification times.
- """
-
- if not binhost_url:
- return {}
-
- def retry_urlopen(url, tries=3):
- """Open the specified url, retrying if we run into temporary errors.
-
- We retry for both network errors and 5xx Server Errors. We do not retry
- for HTTP errors with a non-5xx code.
-
- Args:
- url: The specified url.
- tries: The number of times to try.
-
- Returns:
- The result of urllib2.urlopen(url).
- """
- for i in range(tries):
- try:
- return urllib2.urlopen(url)
- except urllib2.HTTPError as e:
- print "Cannot GET %s: %s" % (url, str(e))
- if i + 1 >= tries or e.code < 500:
- raise
- except urllib2.URLError as e:
- print "Cannot GET %s: %s" % (url, str(e))
- if i + 1 >= tries:
- raise
- print "Sleeping for 10 seconds before retrying..."
- time.sleep(10)
-
- url = os.path.join(binhost_url, "Packages")
- try:
- f = retry_urlopen(url)
- except urllib2.HTTPError as e:
- if e.code == 404:
- return {}
- else:
- raise
- prebuilt_pkgs = {}
- for line in f:
- if line.startswith("CPV: "):
- pkg = line.replace("CPV: ", "").rstrip()
- elif line.startswith("MTIME: "):
- prebuilt_pkgs[pkg] = int(line[:-1].replace("MTIME: ", ""))
- f.close()
-
- return prebuilt_pkgs
-
- def GenDependencyGraph(self, deps_tree, deps_info, remote_pkgs):
- """Generate a doubly linked dependency graph.
-
- Args:
- deps_tree: Dependency tree structure.
- deps_info: More details on the dependencies.
- Returns:
- Deps graph in the form of a dict of packages, with each package
- specifying a "needs" list and "provides" list.
- """
- emerge = self.emerge
- root = emerge.settings["ROOT"]
-
- # It's useful to know what packages will actually end up on the
- # system at some point. Packages in final_db are either already
- # installed, or will be installed by the time we're done.
- final_db = emerge.depgraph._dynamic_config.mydbapi[root]
-
- # final_pkgs is a set of the packages we found in the final_db. These
- # packages are either already installed, or will be installed by the time
- # we're done. It's populated in BuildFinalPackageSet()
- final_pkgs = set()
-
- # These packages take a really long time to build, so, for expediency, we
- # are blacklisting them from automatic rebuilds because one of their
- # dependencies needs to be recompiled.
- rebuild_blacklist = set()
- for pkg in ("chromeos-base/chromeos-chrome", "media-plugins/o3d",
- "dev-java/icedtea"):
- for match in final_db.match_pkgs(pkg):
- rebuild_blacklist.add(str(match.cpv))
-
- # deps_map is the actual dependency graph.
- #
- # Each package specifies a "needs" list and a "provides" list. The "needs"
- # list indicates which packages we depend on. The "provides" list
- # indicates the reverse dependencies -- what packages need us.
- #
- # We also provide some other information in the dependency graph:
- # - action: What we're planning on doing with this package. Generally,
- # "merge", "nomerge", or "uninstall"
- # - mandatory_source:
- # If true, indicates that this package must be compiled from source.
- # We set this for "workon" packages, and for packages where the
- # binaries are known to be out of date.
- # - mandatory:
- # If true, indicates that this package must be installed. We don't care
- # whether it's binary or source, unless the mandatory_source flag is
- # also set.
- # - force_remote_binary:
- # If true, indicates that we want to update to the latest remote prebuilt
- # of this package. Packages that depend on this package should be built
- # from source.
- #
- deps_map = {}
-
- def ReverseTree(packages):
- """Convert tree to digraph.
-
- Take the tree of package -> requirements and reverse it to a digraph of
- buildable packages -> packages they unblock.
- Args:
- packages: Tree(s) of dependencies.
- Returns:
- Unsanitized digraph.
- """
- for pkg in packages:
-
- # Create an entry for the package
- action = packages[pkg]["action"]
- default_pkg = {"needs": {}, "provides": set(), "action": action,
- "mandatory_source": False, "mandatory": False,
- "force_remote_binary": False}
- this_pkg = deps_map.setdefault(pkg, default_pkg)
-
- # Create entries for dependencies of this package first.
- ReverseTree(packages[pkg]["deps"])
-
- # Add dependencies to this package.
- for dep, dep_item in packages[pkg]["deps"].iteritems():
- dep_pkg = deps_map[dep]
- dep_type = dep_item["deptype"]
- if dep_type != "runtime_post":
- dep_pkg["provides"].add(pkg)
- this_pkg["needs"][dep] = dep_type
-
- def BuildFinalPackageSet():
- # If this package is installed, or will get installed, add it to
- # final_pkgs
- for pkg in deps_map:
- for match in final_db.match_pkgs(pkg):
- final_pkgs.add(str(match.cpv))
-
- def FindCycles():
- """Find cycles in the dependency tree.
-
- Returns:
- A dict mapping cyclic packages to a dict of the deps that cause
- cycles. For each dep that causes cycles, it returns an example
- traversal of the graph that shows the cycle.
- """
-
- def FindCyclesAtNode(pkg, cycles, unresolved, resolved):
- """Find cycles in cyclic dependencies starting at specified package.
-
- Args:
- pkg: Package identifier.
- cycles: A dict mapping cyclic packages to a dict of the deps that
- cause cycles. For each dep that causes cycles, it returns an
- example traversal of the graph that shows the cycle.
- unresolved: Nodes that have been visited but are not fully processed.
- resolved: Nodes that have been visited and are fully processed.
- """
- pkg_cycles = cycles.get(pkg)
- if pkg in resolved and not pkg_cycles:
- # If we already looked at this package, and found no cyclic
- # dependencies, we can stop now.
- return
- unresolved.append(pkg)
- for dep in deps_map[pkg]["needs"]:
- if dep in unresolved:
- idx = unresolved.index(dep)
- mycycle = unresolved[idx:] + [dep]
- for i in range(len(mycycle) - 1):
- pkg1, pkg2 = mycycle[i], mycycle[i+1]
- cycles.setdefault(pkg1, {}).setdefault(pkg2, mycycle)
- elif not pkg_cycles or dep not in pkg_cycles:
- # Looks like we haven't seen this edge before.
- FindCyclesAtNode(dep, cycles, unresolved, resolved)
- unresolved.pop()
- resolved.add(pkg)
-
- cycles, unresolved, resolved = {}, [], set()
- for pkg in deps_map:
- FindCyclesAtNode(pkg, cycles, unresolved, resolved)
- return cycles
-
- def RemoveInstalledPackages():
- """Remove installed packages, propagating dependencies."""
-
- # If we're in non-selective mode, the packages specified on the command
- # line are generally mandatory.
- #
- # There are a few exceptions to this rule:
- # 1. If the package isn't getting installed because it's in
- # package.provided, it's not mandatory.
- # 2. If the package isn't getting installed because we're in --onlydeps
- # mode, it's not mandatory either.
- if "--selective" in emerge.opts:
- selective = emerge.opts["--selective"] != "n"
- else:
- selective = ("--noreplace" in emerge.opts or
- "--update" in emerge.opts or
- "--newuse" in emerge.opts or
- "--reinstall" in emerge.opts)
- onlydeps = "--onlydeps" in emerge.opts
- if not selective:
- for pkg in emerge.cmdline_packages:
- # If the package specified on the command-line is in our install
- # list, mark it as non-optional.
- found = False
- for db_pkg in final_db.match_pkgs(pkg):
- this_pkg = deps_info.get(db_pkg.cpv)
- if this_pkg:
- found = True
- this_pkg["optional"] = False
-
- # We didn't find the package in our final db. If we're not in
- # --onlydeps mode, this likely means that the package was specified
- # in package.provided.
- if not found and not onlydeps and "--verbose" in emerge.opts:
- print "Skipping %s (is it in package.provided?)" % pkg
-
- # Schedule packages that aren't on the install list for removal
- rm_pkgs = set(deps_map.keys()) - set(deps_info.keys())
-
- # Schedule optional packages for removal
- for pkg, info in deps_info.items():
- if info["optional"]:
- rm_pkgs.add(pkg)
-
- # Schedule nomerge packages for removal
- for pkg in self.nomerge:
- for db_pkg in final_db.match_pkgs(pkg):
- if db_pkg.cpv in deps_map:
- rm_pkgs.add(str(db_pkg.cpv))
-
- # Remove the packages we don't want, simplifying the graph and making
- # it easier for us to crack cycles.
- for pkg in sorted(rm_pkgs):
- this_pkg = deps_map[pkg]
- needs = this_pkg["needs"]
- provides = this_pkg["provides"]
- for dep in needs:
- dep_provides = deps_map[dep]["provides"]
- dep_provides.update(provides)
- dep_provides.discard(pkg)
- dep_provides.discard(dep)
- for target in provides:
- target_needs = deps_map[target]["needs"]
- target_needs.update(needs)
- target_needs.pop(pkg, None)
- target_needs.pop(target, None)
- del deps_map[pkg]
-
- def PrintCycleBreak(basedep, dep, mycycle):
- """Print details about a cycle that we are planning on breaking.
-
- We are breaking a cycle where dep needs basedep. mycycle is an
- example cycle which contains dep -> basedep."""
-
- # If it's an optional dependency, there's no need to spam the user with
- # warning messages.
- needs = deps_map[dep]["needs"]
- depinfo = needs.get(basedep, "deleted")
- if depinfo == "optional":
- return
-
- # Notify the user that we're breaking a cycle.
- print "Breaking %s -> %s (%s)" % (dep, basedep, depinfo)
-
- # Show cycle.
- for i in range(len(mycycle) - 1):
- pkg1, pkg2 = mycycle[i], mycycle[i+1]
- needs = deps_map[pkg1]["needs"]
- depinfo = needs.get(pkg2, "deleted")
- if pkg1 == dep and pkg2 == basedep:
- depinfo = depinfo + ", deleting"
- print " %s -> %s (%s)" % (pkg1, pkg2, depinfo)
-
- def SanitizeTree():
- """Remove circular dependencies.
-
- We prune all dependencies involved in cycles that go against the emerge
- ordering. This has a nice property: we're guaranteed to merge
- dependencies in the same order that portage does.
-
- Because we don't treat any dependencies as "soft" unless they're killed
- by a cycle, we pay attention to a larger number of dependencies when
- merging. This hurts performance a bit, but helps reliability.
- """
- start = time.time()
- cycles = FindCycles()
- while cycles:
- for dep, mycycles in cycles.iteritems():
- for basedep, mycycle in mycycles.iteritems():
- if deps_info[basedep]["idx"] >= deps_info[dep]["idx"]:
- PrintCycleBreak(basedep, dep, mycycle)
- del deps_map[dep]["needs"][basedep]
- deps_map[basedep]["provides"].remove(dep)
- cycles = FindCycles()
- seconds = time.time() - start
- if "--quiet" not in emerge.opts and seconds >= 0.1:
- print "Tree sanitized in %dm%.1fs" % (seconds / 60, seconds % 60)
-
- def AddSecretDeps():
- """Find these tagged packages and add extra dependencies.
-
- For debugging dependency problems.
- """
- for bad in secret_deps:
- needed = secret_deps[bad]
- bad_pkg = None
- needed_pkg = None
- for dep in deps_map:
- if dep.find(bad) != -1:
- bad_pkg = dep
- if dep.find(needed) != -1:
- needed_pkg = dep
- if bad_pkg and needed_pkg:
- deps_map[needed_pkg]["provides"].add(bad_pkg)
- deps_map[bad_pkg]["needs"][needed_pkg] = "secret"
-
- def MergeChildren(pkg, merge_type):
- """Merge this package and all packages it provides."""
-
- this_pkg = deps_map[pkg]
- if (this_pkg[merge_type] or pkg not in final_pkgs):
- return
-
- if pkg not in deps_info:
- emerge_cmd = "emerge"
- if self.board:
- emerge_cmd = "emerge-%s" % self.board
- emerge_cmd += " -pe =%s %s" % (pkg, " ".join(emerge.cmdline_packages))
- use_str = os.environ.get("USE")
- if use_str:
- emerge_cmd = 'USE="%s" %s' % (use_str, emerge_cmd)
- print "ERROR: emerge has refused to update %s" % pkg
- print "Are there impossible-to-satisfy constraints in the dependency"
- print "graph? To debug the issue, try the following command:"
- print " %s" % emerge_cmd
- sys.exit(1)
-
- # Mark this package as non-optional
- deps_info[pkg]["optional"] = False
- this_pkg[merge_type] = True
- for w in this_pkg["provides"].difference(rebuild_blacklist):
- MergeChildren(w, merge_type)
-
- if this_pkg["action"] == "nomerge":
- this_pkg["action"] = "merge"
-
- def LocalPackageDatabase():
- """Get the modification times of the packages in the local database.
-
- We need to know the modification times of the local packages so that we
- know when they need to be rebuilt.
-
- Returns:
- A dict mapping package identifiers to modification times.
- """
- if self.board:
- path = "/build/%s/packages/Packages" % self.board
- else:
- path = "/var/lib/portage/pkgs/Packages"
- local_pkgs = {}
- for line in file(path):
- if line.startswith("CPV: "):
- pkg = line.replace("CPV: ", "").rstrip()
- elif line.startswith("MTIME: "):
- local_pkgs[pkg] = int(line[:-1].replace("MTIME: ", ""))
-
- return local_pkgs
-
- def AutoRebuildDeps(local_pkgs, remote_pkgs, cycles):
- """Recursively rebuild packages when necessary using modification times.
-
- If you've modified a package, it's a good idea to rebuild all the packages
- that depend on it from source. This function looks for any packages which
- depend on packages that have been modified and ensures that they get
- rebuilt.
-
- Args:
- local_pkgs: Modification times from the local database.
- remote_pkgs: Modification times from the prebuilt server.
- cycles: Dictionary returned from FindCycles()
-
- Returns:
- The set of packages we marked as needing to be merged.
- """
-
- def PrebuiltsReady(pkg, pkg_db, cache):
- """Check whether the prebuilts are ready for pkg and all deps.
-
- Args:
- pkg: The specified package.
- pkg_db: The package DB to use.
- cache: A dict, where the results are stored.
-
- Returns:
- True iff the prebuilts are ready for pkg and all deps.
- """
- if pkg in cache:
- return cache[pkg]
- if pkg not in pkg_db and pkg not in self.forced_remote_binary_packages:
- cache[pkg] = False
- else:
- cache[pkg] = True
- for dep in deps_map[pkg]["needs"]:
- if not PrebuiltsReady(dep, pkg_db, cache):
- cache[pkg] = False
- break
- return cache[pkg]
-
- def LastModifiedWithDeps(pkg, pkg_db, cache):
- """Calculate the last modified time of a package and its dependencies.
-
- This function looks at all the packages needed by the specified package
- and checks the most recent modification time of all of those packages.
- If the dependencies of a package were modified more recently than the
- package itself, then we know the package needs to be rebuilt.
-
- Args:
- pkg: The specified package.
- pkg_db: The package DB to use.
- cache: A dict, where the last modified times are stored.
-
- Returns:
- The last modified time of the specified package and its dependencies.
- """
- if pkg in cache:
- return cache[pkg]
-
- cache[pkg] = pkg_db.get(pkg, 0)
- for dep in deps_map[pkg]["needs"]:
- t = LastModifiedWithDeps(dep, pkg_db, cache)
- cache[pkg] = max(cache[pkg], t)
- return cache[pkg]
-
- # For every package that's getting updated in our local cache (binary
- # or source), make sure we also update the children. If a package is
- # built from source, all children must also be built from source.
- local_ready_cache, remote_ready_cache = {}, {}
- local_mtime_cache, remote_mtime_cache = {}, {}
- for pkg in final_pkgs.difference(rebuild_blacklist):
- # If all the necessary local packages are ready, and their
- # modification times are in sync, we don't need to do anything here.
- local_mtime = LastModifiedWithDeps(pkg, local_pkgs, local_mtime_cache)
- local_ready = PrebuiltsReady(pkg, local_pkgs, local_ready_cache)
- if (not local_ready or local_pkgs.get(pkg, 0) < local_mtime and
- pkg not in cycles):
- # OK, at least one package is missing from the local cache or is
- # outdated. This means we're going to have to install the package
- # and all dependencies.
- #
- # If all the necessary remote packages are ready, and they're at
- # least as new as our local packages, we can install them.
- # Otherwise, we need to build from source.
- remote_mtime = LastModifiedWithDeps(pkg, remote_pkgs,
- remote_mtime_cache)
- remote_ready = PrebuiltsReady(pkg, remote_pkgs, remote_ready_cache)
- if remote_ready and (local_mtime <= remote_mtime or pkg in cycles):
- MergeChildren(pkg, "mandatory")
- else:
- MergeChildren(pkg, "mandatory_source")
-
- def UsePrebuiltPackages(remote_pkgs):
- """Update packages that can use prebuilts to do so."""
- start = time.time()
-
- # Build list of prebuilt packages.
- prebuilt_pkgs = {}
- for pkg, info in deps_map.iteritems():
- if info and info["action"] == "merge":
- if (not info["force_remote_binary"] and info["mandatory_source"] or
- "--usepkgonly" not in emerge.opts and pkg not in remote_pkgs):
- continue
-
- db_pkg = emerge.depgraph._pkg(pkg, "binary", emerge.root_config)
- if info["force_remote_binary"]:
- # Undo our earlier hacks to the use flags so that the use flags
- # display correctly.
- db_pkg.use.enabled = db_pkg.metadata["USE"].split()
- prebuilt_pkgs[pkg] = db_pkg
-
- # Calculate what packages need to be rebuilt due to changes in use flags.
- pkgsettings = emerge.depgraph._frozen_config.pkgsettings[root]
- for pkg, db_pkg in prebuilt_pkgs.iteritems():
- if not self.CheckUseFlags(pkgsettings, db_pkg, self.package_db[pkg]):
- MergeChildren(pkg, "mandatory_source")
-
- # Convert eligible packages to binaries.
- for pkg, info in deps_map.iteritems():
- if info and info["action"] == "merge" and pkg in prebuilt_pkgs:
- if not info["mandatory_source"] or info["force_remote_binary"]:
- self.package_db[pkg] = prebuilt_pkgs[pkg]
-
- seconds = time.time() - start
- if "--quiet" not in emerge.opts:
- print "Prebuilt DB populated in %dm%.1fs" % (seconds / 60, seconds % 60)
-
- return prebuilt_pkgs
-
- ReverseTree(deps_tree)
- BuildFinalPackageSet()
- AddSecretDeps()
-
- # Mark that we want to use remote binaries only for a particular package.
- vardb = emerge.depgraph._frozen_config.trees[root]["vartree"].dbapi
- for pkg in self.force_remote_binary:
- for db_pkg in final_db.match_pkgs(pkg):
- match = deps_map.get(str(db_pkg.cpv))
- if match:
- match["force_remote_binary"] = True
-
- rebuild_blacklist.add(str(db_pkg.cpv))
- if not vardb.match_pkgs(db_pkg.cpv):
- MergeChildren(str(db_pkg.cpv), "mandatory")
-
- if self.no_workon_deps:
- for pkg in self.mandatory_source.copy():
- for db_pkg in final_db.match_pkgs(pkg):
- deps_map[str(db_pkg.cpv)]["mandatory_source"] = True
- else:
- for pkg in self.mandatory_source.copy():
- for db_pkg in final_db.match_pkgs(pkg):
- MergeChildren(str(db_pkg.cpv), "mandatory_source")
-
- cycles = FindCycles()
- if self.rebuild:
- local_pkgs = LocalPackageDatabase()
- AutoRebuildDeps(local_pkgs, remote_pkgs, cycles)
-
- # We need to remove installed packages so that we can use the dependency
- # ordering of the install process to show us what cycles to crack. Once
- # we've done that, we also need to recalculate our list of cycles so that
- # we don't include the installed packages in our cycles.
- RemoveInstalledPackages()
- SanitizeTree()
- if deps_map:
- if "--usepkg" in emerge.opts:
- UsePrebuiltPackages(remote_pkgs)
- return deps_map
-
- def PrintInstallPlan(self, deps_map):
- """Print an emerge-style install plan.
-
- The install plan lists what packages we're installing, in order.
- It's useful for understanding what parallel_emerge is doing.
-
- Args:
- deps_map: The dependency graph.
- """
-
- def InstallPlanAtNode(target, deps_map):
- nodes = []
- nodes.append(target)
- for dep in deps_map[target]["provides"]:
- del deps_map[dep]["needs"][target]
- if not deps_map[dep]["needs"]:
- nodes.extend(InstallPlanAtNode(dep, deps_map))
- return nodes
-
- deps_map = copy.deepcopy(deps_map)
- install_plan = []
- plan = set()
- for target, info in deps_map.iteritems():
- if not info["needs"] and target not in plan:
- for item in InstallPlanAtNode(target, deps_map):
- plan.add(item)
- install_plan.append(self.package_db[item])
-
- for pkg in plan:
- del deps_map[pkg]
-
- if deps_map:
- print "Cyclic dependencies:", " ".join(deps_map)
- PrintDepsMap(deps_map)
- sys.exit(1)
-
- self.emerge.depgraph.display(install_plan)
-
-
-def PrintDepsMap(deps_map):
- """Print dependency graph, for each package list it's prerequisites."""
- for i in sorted(deps_map):
- print "%s: (%s) needs" % (i, deps_map[i]["action"])
- needs = deps_map[i]["needs"]
- for j in sorted(needs):
- print " %s" % (j)
- if not needs:
- print " no dependencies"
-
-
-class EmergeJobState(object):
- __slots__ = ["done", "filename", "last_notify_timestamp", "last_output_seek",
- "last_output_timestamp", "pkgname", "retcode", "start_timestamp",
- "target"]
-
- def __init__(self, target, pkgname, done, filename, start_timestamp,
- retcode=None):
-
- # The full name of the target we're building (e.g.
- # chromeos-base/chromeos-0.0.1-r60)
- self.target = target
-
- # The short name of the target we're building (e.g. chromeos-0.0.1-r60)
- self.pkgname = pkgname
-
- # Whether the job is done. (True if the job is done; false otherwise.)
- self.done = done
-
- # The filename where output is currently stored.
- self.filename = filename
-
- # The timestamp of the last time we printed the name of the log file. We
- # print this at the beginning of the job, so this starts at
- # start_timestamp.
- self.last_notify_timestamp = start_timestamp
-
- # The location (in bytes) of the end of the last complete line we printed.
- # This starts off at zero. We use this to jump to the right place when we
- # print output from the same ebuild multiple times.
- self.last_output_seek = 0
-
- # The timestamp of the last time we printed output. Since we haven't
- # printed output yet, this starts at zero.
- self.last_output_timestamp = 0
-
- # The return code of our job, if the job is actually finished.
- self.retcode = retcode
-
- # The timestamp when our job started.
- self.start_timestamp = start_timestamp
-
-
-def SetupWorkerSignals():
- def ExitHandler(signum, frame):
- # Remove our signal handlers so we don't get called recursively.
- signal.signal(signal.SIGINT, signal.SIG_DFL)
- signal.signal(signal.SIGTERM, signal.SIG_DFL)
-
- # Try to exit cleanly
- sys.exit(1)
-
- # Ensure that we exit quietly and cleanly, if possible, when we receive
- # SIGTERM or SIGINT signals. By default, when the user hits CTRL-C, all
- # of the child processes will print details about KeyboardInterrupt
- # exceptions, which isn't very helpful.
- signal.signal(signal.SIGINT, ExitHandler)
- signal.signal(signal.SIGTERM, ExitHandler)
-
-
-def EmergeWorker(task_queue, job_queue, emerge, package_db):
- """This worker emerges any packages given to it on the task_queue.
-
- Args:
- task_queue: The queue of tasks for this worker to do.
- job_queue: The queue of results from the worker.
- emerge: An EmergeData() object.
- package_db: A dict, mapping package ids to portage Package objects.
-
- It expects package identifiers to be passed to it via task_queue. When
- a task is started, it pushes the (target, filename) to the started_queue.
- The output is stored in filename. When a merge starts or finishes, we push
- EmergeJobState objects to the job_queue.
- """
-
- SetupWorkerSignals()
- settings, trees, mtimedb = emerge.settings, emerge.trees, emerge.mtimedb
- opts, spinner = emerge.opts, emerge.spinner
- opts["--nodeps"] = True
- if new_portage:
- # When Portage launches new processes, it goes on a rampage and closes all
- # open file descriptors. Ask Portage not to do that, as it breaks us.
- portage.process.get_open_fds = lambda: []
- while True:
- # Wait for a new item to show up on the queue. This is a blocking wait,
- # so if there's nothing to do, we just sit here.
- target = task_queue.get()
- if not target:
- # If target is None, this means that the main thread wants us to quit.
- # The other workers need to exit too, so we'll push the message back on
- # to the queue so they'll get it too.
- task_queue.put(target)
- return
- db_pkg = package_db[target]
- db_pkg.root_config = emerge.root_config
- install_list = [db_pkg]
- pkgname = db_pkg.pf
- output = tempfile.NamedTemporaryFile(prefix=pkgname + "-", delete=False)
- start_timestamp = time.time()
- job = EmergeJobState(target, pkgname, False, output.name, start_timestamp)
- job_queue.put(job)
- if "--pretend" in opts:
- retcode = 0
- else:
- save_stdout = sys.stdout
- save_stderr = sys.stderr
- try:
- sys.stdout = output
- sys.stderr = output
- if new_portage:
- emerge.scheduler_graph.mergelist = install_list
- scheduler = Scheduler(settings, trees, mtimedb, opts, spinner,
- favorites=[], graph_config=emerge.scheduler_graph)
- else:
- scheduler = Scheduler(settings, trees, mtimedb, opts, spinner,
- install_list, [], emerge.scheduler_graph)
- retcode = scheduler.merge()
- except Exception:
- traceback.print_exc(file=output)
- retcode = 1
- finally:
- sys.stdout = save_stdout
- sys.stderr = save_stderr
- output.close()
- if retcode is None:
- retcode = 0
-
- job = EmergeJobState(target, pkgname, True, output.name, start_timestamp,
- retcode)
- job_queue.put(job)
-
-
-class LinePrinter(object):
- """Helper object to print a single line."""
-
- def __init__(self, line):
- self.line = line
-
- def Print(self, seek_locations):
- print self.line
-
-
-class JobPrinter(object):
- """Helper object to print output of a job."""
-
- def __init__(self, job, unlink=False):
- """Print output of job.
-
- If unlink is True, unlink the job output file when done."""
- self.current_time = time.time()
- self.job = job
- self.unlink = unlink
-
- def Print(self, seek_locations):
-
- job = self.job
-
- # Calculate how long the job has been running.
- seconds = self.current_time - job.start_timestamp
-
- # Note that we've printed out the job so far.
- job.last_output_timestamp = self.current_time
-
- # Note that we're starting the job
- info = "job %s (%dm%.1fs)" % (job.pkgname, seconds / 60, seconds % 60)
- last_output_seek = seek_locations.get(job.filename, 0)
- if last_output_seek:
- print "=== Continue output for %s ===" % info
- else:
- print "=== Start output for %s ===" % info
-
- # Print actual output from job
- f = codecs.open(job.filename, encoding='utf-8', errors='replace')
- f.seek(last_output_seek)
- prefix = job.pkgname + ":"
- for line in f:
-
- # Save off our position in the file
- if line and line[-1] == "\n":
- last_output_seek = f.tell()
- line = line[:-1]
-
- # Print our line
- print prefix, line.encode('utf-8', 'replace')
- f.close()
-
- # Save our last spot in the file so that we don't print out the same
- # location twice.
- seek_locations[job.filename] = last_output_seek
-
- # Note end of output section
- if job.done:
- print "=== Complete: %s ===" % info
- else:
- print "=== Still running: %s ===" % info
-
- if self.unlink:
- os.unlink(job.filename)
-
-
-def PrintWorker(queue):
- """A worker that prints stuff to the screen as requested."""
-
- def ExitHandler(signum, frame):
- # Switch to default signal handlers so that we'll die after two signals.
- signal.signal(signal.SIGINT, signal.SIG_DFL)
- signal.signal(signal.SIGTERM, signal.SIG_DFL)
-
- # Don't exit on the first SIGINT / SIGTERM, because the parent worker will
- # handle it and tell us when we need to exit.
- signal.signal(signal.SIGINT, ExitHandler)
- signal.signal(signal.SIGTERM, ExitHandler)
-
- # seek_locations is a map indicating the position we are at in each file.
- # It starts off empty, but is set by the various Print jobs as we go along
- # to indicate where we left off in each file.
- seek_locations = {}
- while True:
- try:
- job = queue.get()
- if job:
- job.Print(seek_locations)
- else:
- break
- except IOError as ex:
- if ex.errno == errno.EINTR:
- # Looks like we received a signal. Keep printing.
- continue
- raise
-
-
-class EmergeQueue(object):
- """Class to schedule emerge jobs according to a dependency graph."""
-
- def __init__(self, deps_map, emerge, package_db, show_output):
- # Store the dependency graph.
- self._deps_map = deps_map
- # Initialize the running queue to empty
- self._jobs = {}
- # List of total package installs represented in deps_map.
- install_jobs = [x for x in deps_map if deps_map[x]["action"] == "merge"]
- self._total_jobs = len(install_jobs)
- self._show_output = show_output
-
- if "--pretend" in emerge.opts:
- print "Skipping merge because of --pretend mode."
- sys.exit(0)
-
- # Setup scheduler graph object. This is used by the child processes
- # to help schedule jobs.
- emerge.scheduler_graph = emerge.depgraph.schedulerGraph()
-
- # Calculate how many jobs we can run in parallel. We don't want to pass
- # the --jobs flag over to emerge itself, because that'll tell emerge to
- # hide its output, and said output is quite useful for debugging hung
- # jobs.
- procs = min(self._total_jobs,
- emerge.opts.pop("--jobs", multiprocessing.cpu_count()))
- self._emerge_queue = multiprocessing.Queue()
- self._job_queue = multiprocessing.Queue()
- self._print_queue = multiprocessing.Queue()
- args = (self._emerge_queue, self._job_queue, emerge, package_db)
- self._pool = multiprocessing.Pool(procs, EmergeWorker, args)
- self._print_worker = multiprocessing.Process(target=PrintWorker,
- args=[self._print_queue])
- self._print_worker.start()
-
- # Initialize the failed queue to empty.
- self._retry_queue = []
- self._failed = set()
-
- # Print an update before we launch the merges.
- self._Status()
-
- # Setup an exit handler so that we print nice messages if we are
- # terminated.
- self._SetupExitHandler()
-
- # Schedule our jobs.
- for target, info in deps_map.items():
- if not info["needs"]:
- self._Schedule(target)
-
- def _SetupExitHandler(self):
-
- def ExitHandler(signum, frame):
-
- # Kill our signal handlers so we don't get called recursively
- signal.signal(signal.SIGINT, signal.SIG_DFL)
- signal.signal(signal.SIGTERM, signal.SIG_DFL)
-
- # Print our current job status
- for target, job in self._jobs.iteritems():
- if job:
- self._print_queue.put(JobPrinter(job, unlink=True))
-
- # Notify the user that we are exiting
- self._Print("Exiting on signal %s" % signum)
-
- # Kill child threads, then exit.
- self._Exit()
- sys.exit(1)
-
- # Print out job status when we are killed
- signal.signal(signal.SIGINT, ExitHandler)
- signal.signal(signal.SIGTERM, ExitHandler)
-
- def _Schedule(self, target):
- # We maintain a tree of all deps, if this doesn't need
- # to be installed just free up it's children and continue.
- # It is possible to reinstall deps of deps, without reinstalling
- # first level deps, like so:
- # chromeos (merge) -> eselect (nomerge) -> python (merge)
- if self._deps_map[target]["action"] == "nomerge":
- self._Finish(target)
- else:
- # Kick off the build if it's marked to be built.
- self._jobs[target] = None
- self._emerge_queue.put(target)
-
- def _LoadAvg(self):
- loads = open("/proc/loadavg", "r").readline().split()[:3]
- return " ".join(loads)
-
- def _Print(self, line):
- """Print a single line."""
- self._print_queue.put(LinePrinter(line))
-
- def _Status(self):
- """Print status."""
- current_time = time.time()
- no_output = True
-
- # Print interim output every minute if --show-output is used. Otherwise,
- # print notifications about running packages every 2 minutes, and print
- # full output for jobs that have been running for 60 minutes or more.
- if self._show_output:
- interval = 60
- notify_interval = 0
- else:
- interval = 60 * 60
- notify_interval = 60 * 2
- for target, job in self._jobs.iteritems():
- if job:
- last_timestamp = max(job.start_timestamp, job.last_output_timestamp)
- if last_timestamp + interval < current_time:
- self._print_queue.put(JobPrinter(job))
- job.last_output_timestamp = current_time
- no_output = False
- elif (notify_interval and
- job.last_notify_timestamp + notify_interval < current_time):
- job_seconds = current_time - job.start_timestamp
- args = (job.pkgname, job_seconds / 60, job_seconds % 60, job.filename)
- info = "Still building %s (%dm%.1fs). Logs in %s" % args
- job.last_notify_timestamp = current_time
- self._Print(info)
- no_output = False
-
- # If we haven't printed any messages yet, print a general status message
- # here.
- if no_output:
- seconds = current_time - GLOBAL_START
- line = ("Pending %s, Ready %s, Running %s, Retrying %s, Total %s "
- "[Time %dm%.1fs Load %s]")
- qsize = self._emerge_queue.qsize()
- self._Print(line % (len(self._deps_map), qsize, len(self._jobs) - qsize,
- len(self._retry_queue), self._total_jobs,
- seconds / 60, seconds % 60, self._LoadAvg()))
-
- def _Finish(self, target):
- """Mark a target as completed and unblock dependecies."""
- for dep in self._deps_map[target]["provides"]:
- del self._deps_map[dep]["needs"][target]
- if not self._deps_map[dep]["needs"]:
- self._Schedule(dep)
- self._deps_map.pop(target)
-
- def _Retry(self):
- if self._retry_queue:
- target = self._retry_queue.pop(0)
- self._Schedule(target)
- self._Print("Retrying emerge of %s." % target)
-
- def _Exit(self):
- # Tell emerge workers to exit. They all exit when 'None' is pushed
- # to the queue.
- self._emerge_queue.put(None)
- self._pool.close()
- self._pool.join()
-
- # Now that our workers are finished, we can kill the print queue.
- self._print_queue.put(None)
- self._print_worker.join()
-
- def Run(self):
- """Run through the scheduled ebuilds.
-
- Keep running so long as we have uninstalled packages in the
- dependency graph to merge.
- """
- while self._deps_map:
- # Check here that we are actually waiting for something.
- if (self._emerge_queue.empty() and
- self._job_queue.empty() and
- not self._jobs and
- self._deps_map):
- # If we have failed on a package, retry it now.
- if self._retry_queue:
- self._Retry()
- else:
- # Tell child threads to exit.
- self._Exit()
-
- # The dependency map is helpful for debugging failures.
- PrintDepsMap(self._deps_map)
-
- # Tell the user why we're exiting.
- if self._failed:
- print "Packages failed: %s" % " ,".join(self._failed)
- else:
- print "Deadlock! Circular dependencies!"
- sys.exit(1)
-
- try:
- job = self._job_queue.get(timeout=5)
- except Queue.Empty:
- # Print an update.
- self._Status()
- continue
-
- target = job.target
-
- if not job.done:
- self._jobs[target] = job
- self._Print("Started %s (logged in %s)" % (target, job.filename))
- continue
-
- # Print output of job
- if self._show_output or job.retcode != 0:
- self._print_queue.put(JobPrinter(job, unlink=True))
- else:
- os.unlink(job.filename)
- del self._jobs[target]
-
- seconds = time.time() - job.start_timestamp
- details = "%s (in %dm%.1fs)" % (target, seconds / 60, seconds % 60)
-
- # Complain if necessary.
- if job.retcode != 0:
- # Handle job failure.
- if target in self._failed:
- # If this job has failed previously, give up.
- self._Print("Failed %s. Your build has failed." % details)
- else:
- # Queue up this build to try again after a long while.
- self._retry_queue.append(target)
- self._failed.add(target)
- self._Print("Failed %s, retrying later." % details)
- else:
- if target in self._failed and self._retry_queue:
- # If we have successfully retried a failed package, and there
- # are more failed packages, try the next one. We will only have
- # one retrying package actively running at a time.
- self._Retry()
-
- self._Print("Completed %s" % details)
- # Mark as completed and unblock waiting ebuilds.
- self._Finish(target)
-
- # Print an update.
- self._Status()
-
- # Tell child threads to exit.
- self._Print("Merge complete")
- self._Exit()
-
-
-def main():
-
- deps = DepGraphGenerator()
- deps.Initialize(sys.argv[1:])
- emerge = deps.emerge
-
- if emerge.action is not None:
- sys.argv = deps.ParseParallelEmergeArgs(sys.argv)
- sys.exit(emerge_main())
- elif not emerge.cmdline_packages:
- Usage()
- sys.exit(1)
-
- # Unless we're in pretend mode, there's not much point running without
- # root access. We need to be able to install packages.
- #
- # NOTE: Even if you're running --pretend, it's a good idea to run
- # parallel_emerge with root access so that portage can write to the
- # dependency cache. This is important for performance.
- if "--pretend" not in emerge.opts and portage.secpass < 2:
- print "parallel_emerge: superuser access is required."
- sys.exit(1)
-
- if "--quiet" not in emerge.opts:
- cmdline_packages = " ".join(emerge.cmdline_packages)
- nomerge_packages = " ".join(deps.nomerge)
- print "Starting fast-emerge."
- print " Building package %s on %s" % (cmdline_packages,
- deps.board or "root")
- if nomerge_packages:
- print " Skipping package %s on %s" % (nomerge_packages,
- deps.board or "root")
-
- remote_pkgs = {}
- if "--getbinpkg" in emerge.opts:
- binhost = emerge.settings["PORTAGE_BINHOST"]
- try:
- remote_pkgs = deps.RemotePackageDatabase(binhost)
- except (urllib2.HTTPError, urllib2.URLError):
- print "Cannot resolve binhost. Building from source..."
- del emerge.opts["--getbinpkg"]
-
- deps_tree, deps_info = deps.GenDependencyTree(remote_pkgs)
-
- # You want me to be verbose? I'll give you two trees! Twice as much value.
- if "--tree" in emerge.opts and "--verbose" in emerge.opts:
- deps.PrintTree(deps_tree)
-
- deps_graph = deps.GenDependencyGraph(deps_tree, deps_info, remote_pkgs)
-
- # OK, time to print out our progress so far.
- deps.PrintInstallPlan(deps_graph)
- if "--tree" in emerge.opts:
- PrintDepsMap(deps_graph)
-
- # Are we upgrading portage? If so, and there are more packages to merge,
- # schedule a restart of parallel_emerge to merge the rest. This ensures that
- # we pick up all updates to portage settings before merging any more
- # packages.
- portage_upgrade = False
- root = emerge.settings["ROOT"]
- final_db = emerge.depgraph._dynamic_config.mydbapi[root]
- if root == "/":
- for db_pkg in final_db.match_pkgs("sys-apps/portage"):
- portage_pkg = deps_graph.get(db_pkg.cpv)
- if portage_pkg and len(deps_graph) > 1:
- portage_pkg["needs"].clear()
- portage_pkg["provides"].clear()
- deps_graph = { str(db_pkg.cpv): portage_pkg }
- portage_upgrade = True
- if "--quiet" not in emerge.opts:
- print "Upgrading portage first, then restarting..."
-
- # Run the queued emerges.
- scheduler = EmergeQueue(deps_graph, emerge, deps.package_db, deps.show_output)
- scheduler.Run()
-
- # Update world.
- if ("--oneshot" not in emerge.opts and
- "--pretend" not in emerge.opts):
- world_set = emerge.root_config.sets["selected"]
- new_world_pkgs = []
- for pkg in emerge.cmdline_packages:
- for db_pkg in final_db.match_pkgs(pkg):
- print "Adding %s to world" % db_pkg.cp
- new_world_pkgs.append(db_pkg.cp)
- if new_world_pkgs:
- world_set.update(new_world_pkgs)
-
- # Update environment (library cache, symlinks, etc.)
- if deps.board and "--pretend" not in emerge.opts:
- portage.env_update()
-
- # If we already upgraded portage, we don't need to do so again. But we do
- # need to upgrade the rest of the packages. So we'll go ahead and do that.
- if portage_upgrade:
- args = sys.argv[1:] + ["--nomerge=sys-apps/portage"]
- os.execvp(os.path.realpath(sys.argv[0]), args)
-
- print "Done"
- sys.exit(0)
-
-if __name__ == "__main__":
- main()
« no previous file with comments | « mod_image_for_test.sh ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698