Index: parallel_emerge |
diff --git a/parallel_emerge b/parallel_emerge |
index 66b2334da64a8fa55118e4ab9389237b481ff62e..c1667134c09bb43dc0f0e09f3cbb80afdc095a30 100755 |
--- a/parallel_emerge |
+++ b/parallel_emerge |
@@ -7,7 +7,7 @@ |
Usage: |
./parallel_emerge [--board=BOARD] [--workon=PKGS] [--no-workon-deps] |
- [emerge args] package" |
+ [--force-remote-binary=PKGS] [emerge args] package |
Basic operation: |
Runs 'emerge -p --debug' to display dependencies, and stores a |
@@ -84,6 +84,7 @@ from _emerge.Scheduler import Scheduler |
from _emerge.stdout_spinner import stdout_spinner |
import portage |
import portage.debug |
+import portage.versions |
def Usage(): |
@@ -218,7 +219,8 @@ class DepGraphGenerator(object): |
""" |
__slots__ = ["board", "emerge", "mandatory_source", "no_workon_deps", |
- "nomerge", "package_db", "rebuild", "show_output"] |
+ "nomerge", "package_db", "rebuild", "show_output", |
+ "force_remote_binary", "forced_remote_binary_packages"] |
def __init__(self): |
self.board = None |
@@ -229,6 +231,8 @@ class DepGraphGenerator(object): |
self.package_db = {} |
self.rebuild = False |
self.show_output = False |
+ self.force_remote_binary = set() |
+ self.forced_remote_binary_packages = set() |
def ParseParallelEmergeArgs(self, argv): |
"""Read the parallel emerge arguments from the command-line. |
@@ -251,6 +255,11 @@ class DepGraphGenerator(object): |
workon_str = arg.replace("--workon=", "") |
package_list = shlex.split(" ".join(shlex.split(workon_str))) |
self.mandatory_source.update(package_list) |
+ elif arg.startswith("--force-remote-binary="): |
+ force_remote_binary = arg.replace("--force-remote-binary=", "") |
+ force_remote_binary = \ |
+ shlex.split(" ".join(shlex.split(force_remote_binary))) |
+ self.force_remote_binary.update(force_remote_binary) |
elif arg.startswith("--nomerge="): |
nomerge_str = arg.replace("--nomerge=", "") |
package_list = shlex.split(" ".join(shlex.split(nomerge_str))) |
@@ -460,7 +469,7 @@ class DepGraphGenerator(object): |
cur_iuse, now_use, now_iuse) |
return not flags |
- def GenDependencyTree(self): |
+ def GenDependencyTree(self, remote_pkgs): |
"""Get dependency tree info from emerge. |
TODO(): Update cros_extract_deps to also use this code. |
@@ -479,10 +488,7 @@ class DepGraphGenerator(object): |
# --workon and the dependencies have changed. |
emerge = self.emerge |
emerge_opts = emerge.opts.copy() |
- emerge_opts.pop("--getbinpkg", None) |
- if "--usepkgonly" not in emerge_opts: |
- emerge_opts.pop("--usepkg", None) |
- if self.mandatory_source or self.rebuild: |
+ if self.mandatory_source or self.rebuild or self.force_remote_binary: |
# Enable --emptytree so that we get the full tree, which we need for |
# dependency analysis. By default, with this option, emerge optimizes |
# the graph by removing uninstall instructions from the graph. By |
@@ -491,10 +497,30 @@ class DepGraphGenerator(object): |
emerge_opts["--tree"] = True |
emerge_opts["--emptytree"] = True |
+ # Tell emerge not to worry about use flags yet. We handle those inside |
+ # parallel_emerge itself. Further, when we use the --force-remote-binary |
+ # flag, we don't emerge to reject a package just because it has different |
+ # use flags. |
+ emerge_opts.pop("--newuse", None) |
+ emerge_opts.pop("--reinstall", None) |
+ |
# Create a list of packages to merge |
packages = set(emerge.cmdline_packages[:]) |
if self.mandatory_source: |
packages.update(self.mandatory_source) |
+ if self.force_remote_binary: |
+ forced_pkgs = {} |
+ for pkg in remote_pkgs: |
+ category, pkgname, _, _ = portage.catpkgsplit(pkg) |
+ full_pkgname = "%s/%s" % (category, pkgname) |
+ if (pkgname in self.force_remote_binary or |
+ full_pkgname in self.force_remote_binary): |
+ forced_pkgs.setdefault(full_pkgname, []).append(pkg) |
+ |
+ for pkgs in forced_pkgs.values(): |
+ forced_package = portage.versions.best(pkgs) |
+ packages.add("=%s" % forced_package) |
+ self.forced_remote_binary_packages.add(forced_package) |
# Tell emerge to be quiet. We print plenty of info ourselves so we don't |
# need any extra output from portage. |
@@ -580,9 +606,6 @@ class DepGraphGenerator(object): |
optional = True |
break |
- # Add the package to our database. |
- self.package_db[str(pkg.cpv)] = pkg |
- |
# Save off info about the package |
deps_info[str(pkg.cpv)] = {"idx": len(deps_info), |
"optional": optional} |
@@ -611,7 +634,65 @@ class DepGraphGenerator(object): |
print "%s %s (%s)" % (depth, entry, action) |
self.PrintTree(deps[entry]["deps"], depth=depth + " ") |
- def GenDependencyGraph(self, deps_tree, deps_info): |
+ def RemotePackageDatabase(self, binhost_url): |
+ """Grab the latest binary package database from the prebuilt server. |
+ |
+ We need to know the modification times of the prebuilt packages so that we |
+ know when it is OK to use these packages and when we should rebuild them |
+ instead. |
+ |
+ Args: |
+ binhost_url: Base URL of remote packages (PORTAGE_BINHOST). |
+ |
+ Returns: |
+ A dict mapping package identifiers to modification times. |
+ """ |
+ |
+ if not binhost_url: |
+ return {} |
+ |
+ def retry_urlopen(url, tries=3): |
+ """Open the specified url, retrying if we run into network errors. |
+ |
+ We do not retry for HTTP errors. |
+ |
+ Args: |
+ url: The specified url. |
+ tries: The number of times to try. |
+ |
+ Returns: |
+ The result of urllib2.urlopen(url). |
+ """ |
+ for i in range(tries): |
+ try: |
+ return urllib2.urlopen(url) |
+ except urllib2.HTTPError as e: |
+ raise |
+ except urllib2.URLError as e: |
+ if i + 1 == tries: |
+ raise |
+ else: |
+ print "Cannot GET %s: %s" % (url, e) |
+ |
+ url = os.path.join(binhost_url, "Packages") |
+ try: |
+ f = retry_urlopen(url) |
+ except urllib2.HTTPError as e: |
+ if e.code == 404: |
+ return {} |
+ else: |
+ raise |
+ prebuilt_pkgs = {} |
+ for line in f: |
+ if line.startswith("CPV: "): |
+ pkg = line.replace("CPV: ", "").rstrip() |
+ elif line.startswith("MTIME: "): |
+ prebuilt_pkgs[pkg] = int(line[:-1].replace("MTIME: ", "")) |
+ f.close() |
+ |
+ return prebuilt_pkgs |
+ |
+ def GenDependencyGraph(self, deps_tree, deps_info, remote_pkgs): |
"""Generate a doubly linked dependency graph. |
Args: |
@@ -660,6 +741,10 @@ class DepGraphGenerator(object): |
# If true, indicates that this package must be installed. We don't care |
# whether it's binary or source, unless the mandatory_source flag is |
# also set. |
+ # - force_remote_binary: |
+ # If true, indicates that we want to update to the latest remote prebuilt |
+ # of this package. Packages that depend on this package should be built |
+ # from source. |
# |
deps_map = {} |
@@ -678,7 +763,8 @@ class DepGraphGenerator(object): |
# Create an entry for the package |
action = packages[pkg]["action"] |
default_pkg = {"needs": {}, "provides": set(), "action": action, |
- "mandatory_source": False, "mandatory": False} |
+ "mandatory_source": False, "mandatory": False, |
+ "force_remote_binary": False} |
this_pkg = deps_map.setdefault(pkg, default_pkg) |
# Create entries for dependencies of this package first. |
@@ -909,64 +995,6 @@ class DepGraphGenerator(object): |
if this_pkg["action"] == "nomerge": |
this_pkg["action"] = "merge" |
- def RemotePackageDatabase(binhost_url): |
- """Grab the latest binary package database from the prebuilt server. |
- |
- We need to know the modification times of the prebuilt packages so that we |
- know when it is OK to use these packages and when we should rebuild them |
- instead. |
- |
- Args: |
- binhost_url: Base URL of remote packages (PORTAGE_BINHOST). |
- |
- Returns: |
- A dict mapping package identifiers to modification times. |
- """ |
- |
- if not binhost_url: |
- return {} |
- |
- def retry_urlopen(url, tries=3): |
- """Open the specified url, retrying if we run into network errors. |
- |
- We do not retry for HTTP errors. |
- |
- Args: |
- url: The specified url. |
- tries: The number of times to try. |
- |
- Returns: |
- The result of urllib2.urlopen(url). |
- """ |
- for i in range(tries): |
- try: |
- return urllib2.urlopen(url) |
- except urllib2.HTTPError as e: |
- raise |
- except urllib2.URLError as e: |
- if i + 1 == tries: |
- raise |
- else: |
- print "Cannot GET %s: %s" % (url, e) |
- |
- url = binhost_url + "/Packages" |
- try: |
- f = retry_urlopen(url) |
- except urllib2.HTTPError as e: |
- if e.code == 404: |
- return {} |
- else: |
- raise |
- prebuilt_pkgs = {} |
- for line in f: |
- if line.startswith("CPV: "): |
- pkg = line.replace("CPV: ", "").rstrip() |
- elif line.startswith("MTIME: "): |
- prebuilt_pkgs[pkg] = int(line[:-1].replace("MTIME: ", "")) |
- f.close() |
- |
- return prebuilt_pkgs |
- |
def LocalPackageDatabase(): |
"""Get the modification times of the packages in the local database. |
@@ -1019,7 +1047,7 @@ class DepGraphGenerator(object): |
""" |
if pkg in cache: |
return cache[pkg] |
- if pkg not in pkg_db: |
+ if pkg not in pkg_db and pkg not in self.forced_remote_binary_packages: |
cache[pkg] = False |
else: |
cache[pkg] = True |
@@ -1081,7 +1109,7 @@ class DepGraphGenerator(object): |
else: |
MergeChildren(pkg, "mandatory_source") |
- def UsePrebuiltPackages(): |
+ def UsePrebuiltPackages(remote_pkgs): |
"""Update packages that can use prebuilts to do so.""" |
start = time.time() |
@@ -1099,13 +1127,18 @@ class DepGraphGenerator(object): |
# Build list of prebuilt packages |
for pkg, info in deps_map.iteritems(): |
- if info and not info["mandatory_source"] and info["action"] == "merge": |
+ if info and info["action"] == "merge": |
+ if (not info["force_remote_binary"] and info["mandatory_source"] or |
+ "--usepkgonly" not in emerge.opts and pkg not in remote_pkgs): |
+ continue |
+ |
db_keys = list(bindb._aux_cache_keys) |
try: |
db_vals = bindb.aux_get(pkg, db_keys + ["MTIME"]) |
except KeyError: |
# No binary package |
continue |
+ |
mtime = int(db_vals.pop() or 0) |
metadata = zip(db_keys, db_vals) |
db_pkg = Package(built=True, cpv=pkg, installed=False, |
@@ -1116,15 +1149,16 @@ class DepGraphGenerator(object): |
# Calculate what packages need to be rebuilt due to changes in use flags. |
for pkg, db_pkg in prebuilt_pkgs.iteritems(): |
- db_pkg_src = self.package_db[pkg] |
- if not self.CheckUseFlags(pkgsettings, db_pkg, db_pkg_src): |
+ db_pkg_src = self.package_db.get(pkg) |
+ if db_pkg_src and not self.CheckUseFlags(pkgsettings, db_pkg, |
+ db_pkg_src): |
MergeChildren(pkg, "mandatory_source") |
# Convert eligible packages to binaries. |
for pkg, info in deps_map.iteritems(): |
- if (info and not info["mandatory_source"] and |
- info["action"] == "merge" and pkg in prebuilt_pkgs): |
- self.package_db[pkg] = prebuilt_pkgs[pkg] |
+ if info and info["action"] == "merge" and pkg in prebuilt_pkgs: |
+ if not info["mandatory_source"] or info["force_remote_binary"]: |
+ self.package_db[pkg] = prebuilt_pkgs[pkg] |
seconds = time.time() - start |
if "--quiet" not in emerge.opts: |
@@ -1154,6 +1188,18 @@ class DepGraphGenerator(object): |
BuildFinalPackageSet() |
AddSecretDeps() |
+ # Mark that we want to use remote binaries only for a particular package. |
+ vardb = emerge.depgraph._frozen_config.trees[root]["vartree"].dbapi |
+ for pkg in self.force_remote_binary: |
+ for db_pkg in final_db.match_pkgs(pkg): |
+ match = deps_map.get(str(db_pkg.cpv)) |
+ if match: |
+ match["force_remote_binary"] = True |
+ |
+ rebuild_blacklist.add(str(db_pkg.cpv)) |
+ if not vardb.match_pkgs(db_pkg.cpv): |
+ MergeChildren(str(db_pkg.cpv), "mandatory") |
+ |
if self.no_workon_deps: |
for pkg in self.mandatory_source.copy(): |
for db_pkg in final_db.match_pkgs(pkg): |
@@ -1166,10 +1212,6 @@ class DepGraphGenerator(object): |
cycles = FindCycles() |
if self.rebuild: |
local_pkgs = LocalPackageDatabase() |
- remote_pkgs = {} |
- if "--getbinpkg" in emerge.opts: |
- binhost = emerge.settings["PORTAGE_BINHOST"] |
- remote_pkgs = RemotePackageDatabase(binhost) |
AutoRebuildDeps(local_pkgs, remote_pkgs, cycles) |
# We need to remove installed packages so that we can use the dependency |
@@ -1180,7 +1222,7 @@ class DepGraphGenerator(object): |
SanitizeTree() |
if deps_map: |
if "--usepkg" in emerge.opts: |
- UsePrebuiltPackages() |
+ UsePrebuiltPackages(remote_pkgs) |
AddRemainingPackages() |
return deps_map |
@@ -1734,13 +1776,18 @@ def main(): |
print " Skipping package %s on %s" % (nomerge_packages, |
deps.board or "root") |
- deps_tree, deps_info = deps.GenDependencyTree() |
+ remote_pkgs = {} |
+ if "--getbinpkg" in emerge.opts: |
+ binhost = emerge.settings["PORTAGE_BINHOST"] |
+ remote_pkgs = deps.RemotePackageDatabase(binhost) |
+ |
+ deps_tree, deps_info = deps.GenDependencyTree(remote_pkgs) |
# You want me to be verbose? I'll give you two trees! Twice as much value. |
if "--tree" in emerge.opts and "--verbose" in emerge.opts: |
deps.PrintTree(deps_tree) |
- deps_graph = deps.GenDependencyGraph(deps_tree, deps_info) |
+ deps_graph = deps.GenDependencyGraph(deps_tree, deps_info, remote_pkgs) |
# OK, time to print out our progress so far. |
deps.PrintInstallPlan(deps_graph) |