Remove chromite from crosutils.git. It's been moved to chromite.git.

chromite/lib/binpkg.py

-# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-#
-# Adapted from portage/getbinpkg.py -- Portage binary-package helper functions
-# Copyright 2003-2004 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-
-import operator
-import os
-import tempfile
-import time
-import urllib
-import urllib2
-
-class PackageIndex(object):
-  """A parser for the Portage Packages index file.
-
-  The Portage Packages index file serves to keep track of what packages are
-  included in a tree. It contains the following sections:
-    1) The header. The header tracks general key/value pairs that don't apply
-       to any specific package. E.g., it tracks the base URL of the packages
-       file, and the number of packages included in the file. The header is
-       terminated by a blank line.
-    2) The body. The body is a list of packages. Each package contains a list
-       of key/value pairs. Packages are either terminated by a blank line or
-       by the end of the file. Every package has a CPV entry, which serves as
-       a unique identifier for the package.
-   """
-
-  def __init__(self):
-    """Constructor."""
-
-    # The header tracks general key/value pairs that don't apply to any
-    # specific package. E.g., it tracks the base URL of the packages.
-    self.header = {}
-
-    # A list of packages (stored as a list of dictionaries).
-    self.packages = []
-
-    # Whether or not the PackageIndex has been modified since the last time it
-    # was written.
-    self.modified = False
-
-  def _PopulateDuplicateDB(self, db):
-    """Populate db with SHA1 -> URL mapping for packages.
-
-    Args:
-      db: Dictionary to populate with SHA1 -> URL mapping for packages.
-    """
-
-    uri = self.header['URI']
-    for pkg in self.packages:
-      cpv, sha1 = pkg['CPV'], pkg.get('SHA1')
-      if sha1:
-        path = pkg.get('PATH', urllib.quote(cpv + '.tbz2'))
-        db[sha1] = '%s/%s' % (uri.rstrip('/'), path)
-
-  def _ReadPkgIndex(self, pkgfile):
-    """Read a list of key/value pairs from the Packages file into a dictionary.
-
-    Both header entries and package entries are lists of key/value pairs, so
-    they can both be read by this function. Entries can be terminated by empty
-    lines or by the end of the file.
-
-    This function will read lines from the specified file until it encounters
-    the a blank line or the end of the file.
-
-    Keys and values in the Packages file are separated by a colon and a space.
-    Keys may contain capital letters, numbers, and underscores, but may not
-    contain colons. Values may contain any character except a newline. In
-    particular, it is normal for values to contain colons.
-
-    Lines that have content, and do not contain a valid key/value pair, are
-    ignored. This is for compatibility with the Portage package parser, and
-    to allow for future extensions to the Packages file format.
-
-    All entries must contain at least one key/value pair. If the end of the
-    fils is reached, an empty dictionary is returned.
-
-    Args:
-      pkgfile: A python file object.
-
-    Returns the dictionary of key-value pairs that was read from the file.
-    """
-    d = {}
-    for line in pkgfile:
-      line = line.rstrip('\n')
-      if not line:
-        assert d, 'Packages entry must contain at least one key/value pair'
-        break
-      line = line.split(': ', 1)
-      if len(line) == 2:
-        k, v = line
-        d[k] = v
-    return d
-
-  def _WritePkgIndex(self, pkgfile, entry):
-    """Write header entry or package entry to packages file.
-
-    The keys and values will be separated by a colon and a space. The entry
-    will be terminated by a blank line.
-
-    Args:
-      pkgfile: A python file object.
-      entry: A dictionary of the key/value pairs to write.
-    """
-    lines = ['%s: %s' % (k, v) for k, v in sorted(entry.items()) if v]
-    pkgfile.write('%s\n\n' % '\n'.join(lines))
-
-  def _ReadHeader(self, pkgfile):
-    """Read header of packages file.
-
-    Args:
-      pkgfile: A python file object.
-    """
-    assert not self.header, 'Should only read header once.'
-    self.header = self._ReadPkgIndex(pkgfile)
-
-  def _ReadBody(self, pkgfile):
-    """Read body of packages file.
-
-    Before calling this function, you must first read the header (using
-    _ReadHeader).
-
-    Args:
-      pkgfile: A python file object.
-    """
-    assert self.header, 'Should read header first.'
-    assert not self.packages, 'Should only read body once.'
-
-    # Read all of the sections in the body by looping until we reach the end
-    # of the file.
-    while True:
-      d = self._ReadPkgIndex(pkgfile)
-      if not d:
-        break
-      if 'CPV' in d:
-        self.packages.append(d)
-
-  def Read(self, pkgfile):
-    """Read the entire packages file.
-
-    Args:
-      pkgfile: A python file object.
-    """
-    self._ReadHeader(pkgfile)
-    self._ReadBody(pkgfile)
-
-  def RemoveFilteredPackages(self, filter_fn):
-    """Remove packages which match filter_fn.
-
-    Args:
-      filter_fn: A function which operates on packages. If it returns True,
-                 the package should be removed.
-    """
-
-    filtered = [p for p in self.packages if not filter_fn(p)]
-    if filtered != self.packages:
-      self.modified = True
-      self.packages = filtered
-
-  def ResolveDuplicateUploads(self, pkgindexes):
-    """Point packages at files that have already been uploaded.
-
-    For each package in our index, check if there is an existing package that
-    has already been uploaded to the same base URI. If so, point that package
-    at the existing file, so that we don't have to upload the file.
-
-    Args:
-      pkgindexes: A list of PackageIndex objects containing info about packages
-        that have already been uploaded.
-
-    Returns:
-      A list of the packages that still need to be uploaded.
-    """
-    db = {}
-    for pkgindex in pkgindexes:
-      pkgindex._PopulateDuplicateDB(db)
-
-    uploads = []
-    base_uri = self.header['URI']
-    for pkg in self.packages:
-      sha1 = pkg.get('SHA1')
-      uri = db.get(sha1)
-      if sha1 and uri and uri.startswith(base_uri):
-        pkg['PATH'] = uri[len(base_uri):].lstrip('/')
-      else:
-        uploads.append(pkg)
-    return uploads
-
-  def SetUploadLocation(self, base_uri, path_prefix):
-    """Set upload location to base_uri + path_prefix.
-
-    Args:
-      base_uri: Base URI for all packages in the file. We set
-        self.header['URI'] to this value, so all packages must live under
-        this directory.
-      path_prefix: Path prefix to use for all current packages in the file.
-        This will be added to the beginning of the path for every package.
-    """
-    self.header['URI'] = base_uri
-    for pkg in self.packages:
-      path = urllib.quote(pkg['CPV'] + '.tbz2')
-      pkg['PATH'] = '%s/%s' % (path_prefix.rstrip('/'), path)
-
-  def Write(self, pkgfile):
-    """Write a packages file to disk.
-
-    If 'modified' flag is set, the TIMESTAMP and PACKAGES fields in the header
-    will be updated before writing to disk.
-
-    Args:
-      pkgfile: A python file object.
-    """
-    if self.modified:
-      self.header['TIMESTAMP'] = str(long(time.time()))
-      self.header['PACKAGES'] = str(len(self.packages))
-      self.modified = False
-    self._WritePkgIndex(pkgfile, self.header)
-    for metadata in sorted(self.packages, key=operator.itemgetter('CPV')):
-      self._WritePkgIndex(pkgfile, metadata)
-
-  def WriteToNamedTemporaryFile(self):
-    """Write pkgindex to a temporary file.
-
-    Args:
-     pkgindex: The PackageIndex object.
-
-    Returns:
-      A temporary file containing the packages from pkgindex.
-    """
-    f = tempfile.NamedTemporaryFile()
-    self.Write(f)
-    f.flush()
-    f.seek(0)
-    return f
-
-
-def _RetryUrlOpen(url, tries=3):
-  """Open the specified url, retrying if we run into temporary errors.
-
-  We retry for both network errors and 5xx Server Errors. We do not retry
-  for HTTP errors with a non-5xx code.
-
-  Args:
-    url: The specified url.
-    tries: The number of times to try.
-
-  Returns:
-    The result of urllib2.urlopen(url).
-  """
-  for i in range(tries):
-    try:
-      return urllib2.urlopen(url)
-    except urllib2.HTTPError as e:
-      if i + 1 >= tries or e.code < 500:
-        raise
-      else:
-        print 'Cannot GET %s: %s' % (url, str(e))
-    except urllib2.URLError as e:
-      if i + 1 >= tries:
-        raise
-      else:
-        print 'Cannot GET %s: %s' % (url, str(e))
-    print 'Sleeping for 10 seconds before retrying...'
-    time.sleep(10)
-
-
-def GrabRemotePackageIndex(binhost_url):
-  """Grab the latest binary package database from the specified URL.
-
-  Args:
-    binhost_url: Base URL of remote packages (PORTAGE_BINHOST).
-
-  Returns:
-    A PackageIndex object, if the Packages file can be retrieved. If the
-    server returns status code 404, None is returned.
-  """
-
-  url = '%s/Packages' % binhost_url.rstrip('/')
-  try:
-    f = _RetryUrlOpen(url)
-  except urllib2.HTTPError as e:
-    if e.code == 404:
-      return None
-    raise
-
-  pkgindex = PackageIndex()
-  pkgindex.Read(f)
-  pkgindex.header.setdefault('URI', binhost_url)
-  f.close()
-  return pkgindex
-
-
-def GrabLocalPackageIndex(package_path):
-  """Read a local packages file from disk into a PackageIndex() object.
-
-  Args:
-    package_path: Directory containing Packages file.
-
-  Returns:
-    A PackageIndex object.
-  """
-  packages_file = file(os.path.join(package_path, 'Packages'))
-  pkgindex = PackageIndex()
-  pkgindex.Read(packages_file)
-  packages_file.close()
-  return pkgindex