Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(565)

Side by Side Diff: parallel_emerge

Issue 3340010: Fix cycle cracking once and for all. (Closed) Base URL: http://git.chromium.org/git/crosutils.git
Patch Set: Confirm no changes Created 10 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/python2.6 1 #!/usr/bin/python2.6
2 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. 2 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Program to run emerge in parallel, for significant speedup. 6 """Program to run emerge in parallel, for significant speedup.
7 7
8 Usage: 8 Usage:
9 ./parallel_emerge [--board=BOARD] [--workon=PKGS] [--no-workon-deps] 9 ./parallel_emerge [--board=BOARD] [--workon=PKGS] [--no-workon-deps]
10 [emerge args] package" 10 [emerge args] package"
(...skipping 811 matching lines...) Expand 10 before | Expand all | Expand 10 after
822 822
823 # Show cycle. 823 # Show cycle.
824 for i in range(len(mycycle) - 1): 824 for i in range(len(mycycle) - 1):
825 pkg1, pkg2 = mycycle[i], mycycle[i+1] 825 pkg1, pkg2 = mycycle[i], mycycle[i+1]
826 needs = deps_map[pkg1]["needs"] 826 needs = deps_map[pkg1]["needs"]
827 depinfo = needs.get(pkg2, "deleted") 827 depinfo = needs.get(pkg2, "deleted")
828 if pkg1 == dep and pkg2 == basedep: 828 if pkg1 == dep and pkg2 == basedep:
829 depinfo = depinfo + ", deleting" 829 depinfo = depinfo + ", deleting"
830 print " %s -> %s (%s)" % (pkg1, pkg2, depinfo) 830 print " %s -> %s (%s)" % (pkg1, pkg2, depinfo)
831 831
832 def SanitizeTree(cycles): 832 def SanitizeTree():
833 """Remove circular dependencies. 833 """Remove circular dependencies.
834 834
835 We prune all dependencies involved in cycles that go against the emerge 835 We prune all dependencies involved in cycles that go against the emerge
836 ordering. This has a nice property: we're guaranteed to merge 836 ordering. This has a nice property: we're guaranteed to merge
837 dependencies in the same order that portage does. 837 dependencies in the same order that portage does.
838 838
839 Because we don't treat any dependencies as "soft" unless they're killed 839 Because we don't treat any dependencies as "soft" unless they're killed
840 by a cycle, we pay attention to a larger number of dependencies when 840 by a cycle, we pay attention to a larger number of dependencies when
841 merging. This hurts performance a bit, but helps reliability. 841 merging. This hurts performance a bit, but helps reliability.
842
843 Args:
844 cycles: Dict of packages involved in cyclic dependencies, mapping each
845 package to a list of the cycles the package is involved in. Produced
846 by FindCycles().
847 """ 842 """
848 for dep, mycycles in cycles.iteritems(): 843 start = time.time()
849 for basedep, mycycle in mycycles.iteritems(): 844 cycles = FindCycles()
850 if deps_info[basedep]["idx"] >= deps_info[dep]["idx"]: 845 while cycles:
851 PrintCycleBreak(basedep, dep, mycycle) 846 for dep, mycycles in cycles.iteritems():
852 del deps_map[dep]["needs"][basedep] 847 for basedep, mycycle in mycycles.iteritems():
853 deps_map[basedep]["provides"].remove(dep) 848 if deps_info[basedep]["idx"] >= deps_info[dep]["idx"]:
849 PrintCycleBreak(basedep, dep, mycycle)
850 del deps_map[dep]["needs"][basedep]
851 deps_map[basedep]["provides"].remove(dep)
852 cycles = FindCycles()
853 seconds = time.time() - start
854 if "--quiet" not in emerge.opts and seconds >= 0.1:
855 print "Tree sanitized in %dm%.1fs" % (seconds / 60, seconds % 60)
854 856
855 def AddSecretDeps(): 857 def AddSecretDeps():
856 """Find these tagged packages and add extra dependencies. 858 """Find these tagged packages and add extra dependencies.
857 859
858 For debugging dependency problems. 860 For debugging dependency problems.
859 """ 861 """
860 for bad in secret_deps: 862 for bad in secret_deps:
861 needed = secret_deps[bad] 863 needed = secret_deps[bad]
862 bad_pkg = None 864 bad_pkg = None
863 needed_pkg = None 865 needed_pkg = None
(...skipping 258 matching lines...) Expand 10 before | Expand all | Expand 10 after
1122 if self.rebuild: 1124 if self.rebuild:
1123 local_pkgs = LocalPackageDatabase() 1125 local_pkgs = LocalPackageDatabase()
1124 remote_pkgs = RemotePackageDatabase(emerge.settings["PORTAGE_BINHOST"]) 1126 remote_pkgs = RemotePackageDatabase(emerge.settings["PORTAGE_BINHOST"])
1125 AutoRebuildDeps(local_pkgs, remote_pkgs, cycles) 1127 AutoRebuildDeps(local_pkgs, remote_pkgs, cycles)
1126 1128
1127 # We need to remove installed packages so that we can use the dependency 1129 # We need to remove installed packages so that we can use the dependency
1128 # ordering of the install process to show us what cycles to crack. Once 1130 # ordering of the install process to show us what cycles to crack. Once
1129 # we've done that, we also need to recalculate our list of cycles so that 1131 # we've done that, we also need to recalculate our list of cycles so that
1130 # we don't include the installed packages in our cycles. 1132 # we don't include the installed packages in our cycles.
1131 RemoveInstalledPackages() 1133 RemoveInstalledPackages()
1132 cycles = FindCycles() 1134 SanitizeTree()
1133 SanitizeTree(cycles)
1134 if deps_map: 1135 if deps_map:
1135 if "--usepkg" in emerge.opts: 1136 if "--usepkg" in emerge.opts:
1136 UsePrebuiltPackages() 1137 UsePrebuiltPackages()
1137 AddRemainingPackages() 1138 AddRemainingPackages()
1138 return deps_map 1139 return deps_map
1139 1140
1140 def PrintInstallPlan(self, deps_map): 1141 def PrintInstallPlan(self, deps_map):
1141 """Print an emerge-style install plan. 1142 """Print an emerge-style install plan.
1142 1143
1143 The install plan lists what packages we're installing, in order. 1144 The install plan lists what packages we're installing, in order.
(...skipping 540 matching lines...) Expand 10 before | Expand all | Expand 10 after
1684 # If we already upgraded portage, we don't need to do so again. But we do 1685 # If we already upgraded portage, we don't need to do so again. But we do
1685 # need to upgrade the rest of the packages. So we'll go ahead and do that. 1686 # need to upgrade the rest of the packages. So we'll go ahead and do that.
1686 if portage_upgrade: 1687 if portage_upgrade:
1687 args = sys.argv[1:] + ["--nomerge=sys-apps/portage"] 1688 args = sys.argv[1:] + ["--nomerge=sys-apps/portage"]
1688 os.execvp(os.path.realpath(sys.argv[0]), args) 1689 os.execvp(os.path.realpath(sys.argv[0]), args)
1689 1690
1690 print "Done" 1691 print "Done"
1691 1692
1692 if __name__ == "__main__": 1693 if __name__ == "__main__":
1693 main() 1694 main()
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698