Update Portage to sync BlockerDB at init, rather than before every package.

pym/_emerge/Scheduler.py

 # Copyright 1999-2011 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import print_function
 
 import gc
 import gzip
 import logging
 import shutil
 import signal
@@ skipping 299 matching lines @@
                         if isinstance(merge, PackageMerge):
                                 self._running_tasks.remove(merge.merge.pkg)
                 for q in self._task_queues.values():
                         q.clear()
 
         def _init_graph(self, graph_config):
                 """
                 Initialization structures used for dependency calculations
                 involving currently installed packages.
                 """
-                # TODO: Replace the BlockerDB with a depgraph of installed packages
-                # that's updated incrementally with each upgrade/uninstall operation
-                # This will be useful for making quick and safe decisions with respect
-                # to aggressive parallelization discussed in bug #279623.
                 self._set_graph_config(graph_config)
                 self._blocker_db = {}
                 for root in self.trees:
                         if graph_config is None:
                                 fake_vartree = FakeVartree(self.trees[root]["root_config"],
                                         pkg_cache=self._pkg_cache)
                         else:
                                 fake_vartree = graph_config.trees[root]['vartree']
+                        fake_vartree.sync()

[diandersAtChromium, 2011/03/22 02:57:59]

How did you decide this was the place to put the sync()?

...this is not the only construction place for FakeVartree.  Do those other
places need sync() also?


To answer my own question:
* The only place that changed to not auto-call sync was findInstalledBlockers.
* The only caller of findInstalledBlockers is based on iterating over
self._blocker_db
* This is the only place that puts things in self._blocker_db.

...so I guess you're good?

[davidjames, 2011/03/23 20:19:11]

Exactly right.

                         self._blocker_db[root] = BlockerDB(fake_vartree)
 
         def _destroy_graph(self):
                 """
                 Use this to free memory at the beginning of _calc_resume_list().
                 After _calc_resume_list(), the _init_graph() method
                 must to be called in order to re-generate the structures that
                 this method destroys. 
                 """
                 self._blocker_db = None
@@ skipping 294 matching lines @@
 
         def _schedule_unpack(self, unpack_phase):
                 """
                 Schedule an unpack phase on the unpack queue, in order
                 to serialize $DISTDIR access for live ebuilds.
                 """
                 self._task_queues.unpack.add(unpack_phase)
 
         def _find_blockers(self, new_pkg):
                 """
-                Returns a callable which should be called only when
-                the vdb lock has been acquired.
+                Returns a callable.

[diandersAtChromium, 2011/03/22 02:57:59]

...that will find the blockers for the given package.

[davidjames, 2011/03/23 20:19:11]

Done.

                 """
                 def get_blockers():
-                        return self._find_blockers_with_lock(new_pkg, acquire_lock=0)
+                        return self._find_blockers_impl(new_pkg)
                 return get_blockers
 
-        def _find_blockers_with_lock(self, new_pkg, acquire_lock=0):
+        def _find_blockers_impl(self, new_pkg):
                 if self._opts_ignore_blockers.intersection(self.myopts):
                         return None
 

[diandersAtChromium, 2011/03/22 02:57:59]

Are you sure that the gc.collect() is no longer needed?  Seems like a different
CL should remove this, maybe (here and below).

[davidjames, 2011/03/23 20:19:11]

This gc.collect() call was only needed to workaround bugs in Python 2.5. Portage
has since dropped support for Python 2.5 so this is no longer applicable.

Not sure whether this needs to be split up into a separate CL.

-                # Call gc.collect() here to avoid heap overflow that
-                # triggers 'Cannot allocate memory' errors (reported
-                # with python-2.5).
-                gc.collect()
-
                 blocker_db = self._blocker_db[new_pkg.root]
 
                 blocker_dblinks = []
-                for blocking_pkg in blocker_db.findInstalledBlockers(
-                        new_pkg, acquire_lock=acquire_lock):
+                for blocking_pkg in blocker_db.findInstalledBlockers(new_pkg):
                         if new_pkg.slot_atom == blocking_pkg.slot_atom:
                                 continue
                         if new_pkg.cpv == blocking_pkg.cpv:
                                 continue
                         blocker_dblinks.append(portage.dblink(
                                 blocking_pkg.category, blocking_pkg.pf, blocking_pkg.root,
                                 self.pkgsettings[blocking_pkg.root], treetype="vartree",
                                 vartree=self.trees[blocking_pkg.root]["vartree"]))
 
-                gc.collect()
-
                 return blocker_dblinks
 
         def _dblink_pkg(self, pkg_dblink):
                 cpv = pkg_dblink.mycpv
                 type_name = RootConfig.tree_pkg_map[pkg_dblink.treetype]
                 root_config = self.trees[pkg_dblink.myroot]["root_config"]
                 installed = type_name == "installed"
                 return self._pkg(cpv, type_name, root_config, installed=installed)
 
         def _dblink_elog(self, pkg_dblink, phase, func, msgs):
@@ skipping 832 matching lines @@
                 self._status_display.running = self._jobs
                 self._schedule()
 
         def _extract_exit(self, build):
                 self._build_exit(build)
 
         def _task_complete(self, pkg):
                 self._completed_tasks.add(pkg)
                 self._unsatisfied_system_deps.discard(pkg)
                 self._choose_pkg_return_early = False
+                blocker_db = self._blocker_db[pkg.root]
+                blocker_db.discardBlocker(pkg)
 
         def _merge(self):
 
                 self._add_prefetchers()
                 self._add_packages()
                 pkg_queue = self._pkg_queue
                 failed_pkgs = self._failed_pkgs
                 portage.locks._quiet = self._background
                 portage.elog.add_listener(self._elog_listener)
                 rval = os.EX_OK
@@ skipping 537 matching lines @@
                 tree_type = depgraph.pkg_tree_map[type_name]
                 db = root_config.trees[tree_type].dbapi
                 db_keys = list(self.trees[root_config.root][
                         tree_type].dbapi._aux_cache_keys)
                 metadata = zip(db_keys, db.aux_get(cpv, db_keys))
                 pkg = Package(built=(type_name != "ebuild"),
                         cpv=cpv, installed=installed, metadata=metadata,
                         root_config=root_config, type_name=type_name)
                 self._pkg_cache[pkg] = pkg
                 return pkg