Update Portage to support multiple binhosts.

pym/portage/dbapi/bintree.py

 # Copyright 1998-2010 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 __all__ = ["bindbapi", "binarytree"]
 
 import portage
 portage.proxy.lazyimport.lazyimport(globals(),
         'portage.checksum:hashfunc_map,perform_multiple_checksums,verify_all',
         'portage.dbapi.dep_expand:dep_expand',
         'portage.dep:dep_getkey,isjustname,match_from_list',
@@ skipping 206 matching lines @@
                 if True:
                         self.root = root
                         #self.pkgdir=settings["PKGDIR"]
                         self.pkgdir = normalize_path(pkgdir)
                         self.dbapi = bindbapi(self, settings=settings)
                         self.update_ents = self.dbapi.update_ents
                         self.move_slot_ent = self.dbapi.move_slot_ent
                         self.populated = 0
                         self.tree = {}
                         self._remote_has_index = False
-                        self._remote_base_uri = None
                         self._remotepkgs = None # remote metadata indexed by cpv
                         self.__remotepkgs = {}  # indexed by tbz2 name (deprecated)
                         self.invalids = []
                         self.settings = settings
                         self._pkg_paths = {}
                         self._populating = False
                         self._all_directory = os.path.isdir(
                                 os.path.join(self.pkgdir, "All"))
                         self._pkgindex_version = 0
                         self._pkgindex_hashes = ["MD5","SHA1"]
                         self._pkgindex_file = os.path.join(self.pkgdir, "Packages")
                         self._pkgindex_keys = self.dbapi._aux_cache_keys.copy()
                         self._pkgindex_keys.update(["CPV", "MTIME", "SIZE"])
                         self._pkgindex_aux_keys = \
                                 ["BUILD_TIME", "CHOST", "DEPEND", "DESCRIPTION", "EAPI",
                                 "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES",
                                 "PROVIDE", "RDEPEND", "repository", "SLOT", "USE", "DEFINED_PHASES",
-                                "REQUIRED_USE"]
+                                "REQUIRED_USE", "BASE_URI"]
                         self._pkgindex_aux_keys = list(self._pkgindex_aux_keys)
                         self._pkgindex_use_evaluated_keys = \
                                 ("LICENSE", "RDEPEND", "DEPEND",
                                 "PDEPEND", "PROPERTIES", "PROVIDE")
                         self._pkgindex_header_keys = set([
                                 "ACCEPT_KEYWORDS", "ACCEPT_LICENSE",
                                 "ACCEPT_PROPERTIES", "CBUILD",
                                 "CHOST", "CONFIG_PROTECT", "CONFIG_PROTECT_MASK", "FEATURES",
                                 "GENTOO_MIRRORS", "INSTALL_MASK", "SYNC", "USE"])
                         self._pkgindex_default_pkg_data = {
@@ skipping 465 matching lines @@
                                 pkgindex.packages.extend(iter(metadata.values()))
                                 self._update_pkgindex_header(pkgindex.header)
                                 f = atomic_ofstream(self._pkgindex_file)
                                 pkgindex.write(f)
                                 f.close()
 
                 if getbinpkgs and not self.settings["PORTAGE_BINHOST"]:
                         writemsg(_("!!! PORTAGE_BINHOST unset, but use is requested.\n"),
                                 noiselevel=-1)
 
-                if getbinpkgs and 'PORTAGE_BINHOST' in self.settings:
-                        base_url = self.settings["PORTAGE_BINHOST"]
+                if not getbinpkgs or 'PORTAGE_BINHOST' not in self.settings:
+                        self.populated=1
+                        return
+                self._remotepkgs = {}
+                self.__remotepkgs = {}
+                for base_url in self.settings["PORTAGE_BINHOST"].split():
                         parsed_url = urlparse(base_url)
                         host = parsed_url.netloc
                         port = parsed_url.port
                         user = None
                         passwd = None
                         user_passwd = ""
                         if "@" in host:
                                 user, host = host.split("@", 1)
                                 user_passwd = user + "@"
                                 if ":" in user:
@@ skipping 107 matching lines @@
                                         f = atomic_ofstream(pkgindex_file)
                                         pkgindex.write(f)
                                         f.close()
                                 except (IOError, PortageException):
                                         if os.access(os.path.dirname(pkgindex_file), os.W_OK):
                                                 raise
                                         # The current user doesn't have permission to cache the
                                         # file, but that's alright.
                         if pkgindex:
                                 # Organize remote package list as a cpv -> metadata map.
-                                self._remotepkgs = _pkgindex_cpv_map_latest_build(pkgindex)
+                                remotepkgs = _pkgindex_cpv_map_latest_build(pkgindex)
+                                self._remotepkgs.update(remotepkgs)
                                 self._remote_has_index = True
-                                self._remote_base_uri = pkgindex.header.get("URI", base_url)
-                                self.__remotepkgs = {}
-                                for cpv in self._remotepkgs:
+                                remote_base_uri = pkgindex.header.get("URI", base_url)
+                                for cpv in remotepkgs:
                                         self.dbapi.cpv_inject(cpv)
-                                self.populated = 1
                                 if True:
                                         # Remote package instances override local package
                                         # if they are not identical.
                                         hash_names = ["SIZE"] + self._pkgindex_hashes
                                         for cpv, local_metadata in metadata.items():
                                                 remote_metadata = self._remotepkgs.get(cpv)
                                                 if remote_metadata is None:
                                                         continue
+                                                remote_metadata["BASE_URI"] = remote_base_uri
                                                 # Use digests to compare identity.
                                                 identical = True
                                                 for hash_name in hash_names:
                                                         local_value = local_metadata.get(hash_name)
                                                         if local_value is None:
                                                                 continue
                                                         remote_value = remote_metadata.get(hash_name)
                                                         if remote_value is None:
                                                                 continue
                                                         if local_value != remote_value:
                                                                 identical = False
                                                                 break
                                                 if identical:
                                                         del self._remotepkgs[cpv]
                                                 else:
                                                         # Override the local package in the aux_get cache.
                                                         self.dbapi._aux_cache[cpv] = remote_metadata
                                 else:
                                         # Local package instances override remote instances.
                                         for cpv in metadata:
                                                 self._remotepkgs.pop(cpv, None)
-                                return
-                        self._remotepkgs = {}
+                                continue
                         try:
                                 chunk_size = long(self.settings["PORTAGE_BINHOST_CHUNKSIZE"])
                                 if chunk_size < 8:
                                         chunk_size = 8
                         except (ValueError, KeyError):
                                 chunk_size = 3000
                         writemsg_stdout("\n")
                         writemsg_stdout(
                                 colorize("GOOD", _("Fetching bininfo from ")) + \
                                 re.sub(r'//(.+):.+@(.+)/', r'//\1:*password*@\2/', base_url) + "\n")
-                        self.__remotepkgs = portage.getbinpkg.dir_get_metadata(
-                                self.settings["PORTAGE_BINHOST"], chunk_size=chunk_size)
+                        remotepkgs = portage.getbinpkg.dir_get_metadata(
+                                base_url, chunk_size=chunk_size)
+                        self.__remotepkgs.update(remotepkgs)
                         #writemsg(green("  -- DONE!\n\n"))
 
-                        for mypkg in list(self.__remotepkgs):
+                        for mypkg in list(remotepkgs):
                                 if "CATEGORY" not in self.__remotepkgs[mypkg]:
                                         #old-style or corrupt package
                                         writemsg(_("!!! Invalid remote binary package: %s\n") % mypkg,
                                                 noiselevel=-1)
                                         del self.__remotepkgs[mypkg]
                                         continue
                                 mycat = self.__remotepkgs[mypkg]["CATEGORY"].strip()
                                 fullpkg = mycat+"/"+mypkg[:-5]
 
                                 if fullpkg in metadata:
@@ skipping 303 matching lines @@
                                 writemsg(_("Resuming download of this tbz2, but it is possible that it is corrupt.\n"),
                                         noiselevel=-1)
                 
                 mydest = os.path.dirname(self.getname(pkgname))
                 self._ensure_dir(mydest)
                 # urljoin doesn't work correctly with unrecognized protocols like sftp
                 if self._remote_has_index:
                         rel_url = self._remotepkgs[pkgname].get("PATH")
                         if not rel_url:
                                 rel_url = pkgname+".tbz2"
-                        url = self._remote_base_uri.rstrip("/") + "/" + rel_url.lstrip("/")
+                        remote_base_uri = self._remotepkgs[pkgname]["BASE_URI"]
+                        url = remote_base_uri.rstrip("/") + "/" + rel_url.lstrip("/")
                 else:
                         url = self.settings["PORTAGE_BINHOST"].rstrip("/") + "/" + tbz2name
                 protocol = urlparse(url)[0]
                 fcmd_prefix = "FETCHCOMMAND"
                 if resume:
                         fcmd_prefix = "RESUMECOMMAND"
                 fcmd = self.settings.get(fcmd_prefix + "_" + protocol.upper())
                 if not fcmd:
                         fcmd = self.settings.get(fcmd_prefix)
                 success = portage.getbinpkg.file_get(url, mydest, fcmd=fcmd)
@@ skipping 76 matching lines @@
         def getslot(self, mycatpkg):
                 "Get a slot for a catpkg; assume it exists."
                 myslot = ""
                 try:
                         myslot = self.dbapi.aux_get(mycatpkg,["SLOT"])[0]
                 except SystemExit as e:
                         raise
                 except Exception as e:
                         pass
                 return myslot