Update cbuildbot.py to upload prebuilts from preflight buildbot.

chromite/lib/binpkg.py

+# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# Adapted from portage/getbinpkg.py -- Portage binary-package helper functions
+# Copyright 2003-2004 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import operator
+import os
+import time
+import urllib2
+
+class PackageIndex(object):
+
+  def __init__(self):
+    """Constructor."""
+    self.header = {}
+    self.packages = []
+    self.modified = False
+
+  def _ReadPkgIndex(self, pkgfile):
+    """Read entry from packages file.
+
+    Args:
+      pkgfile: A python file object.
+    """
+    d = {}
+    for line in pkgfile:
+      line = line.rstrip('\n')
+      if not line:
+        break
+      line = line.split(': ', 1)
+      if len(line) == 2:
+        k, v = line
+        d[k] = v
+    return d
+
+  def _WritePkgIndex(self, pkgfile, items):
+    """Write entry to packages file.
+
+    Args:
+      pkgfile: A python file object.
+      items: A list of tuples containing the entries to write.
+    """
+    for k, v in items:
+      pkgfile.write('%s: %s\n' % (k, v))
+    pkgfile.write('\n')
+
+  def Read(self, pkgfile):
+    """Read entire packages file.
+
+    Args:
+      pkgfile: A python file object.
+    """
+    self.ReadHeader(pkgfile)
+    self.ReadBody(pkgfile)
+
+  def ReadHeader(self, pkgfile):
+    """Read header of packages file.
+
+    Args:
+      pkgfile: A python file object.
+    """
+    self.header.update(self._ReadPkgIndex(pkgfile))
+
+  def ReadBody(self, pkgfile):
+    """Read body of packages file.
+
+    Args:
+      pkgfile: A python file object.
+    """
+    while True:
+      d = self._ReadPkgIndex(pkgfile)
+      if not d:
+        break
+      if d.get('CPV'):
+        self.packages.append(d)
+
+  def Write(self, pkgfile):
+    """Write a packages file to disk.
+
+    Args:
+      pkgfile: A python file object.
+    """
+    if self.modified:
+      self.header['TIMESTAMP'] = str(long(time.time()))
+      self.header['PACKAGES'] = str(len(self.packages))
+    keys = list(self.header)
+    keys.sort()
+    self._WritePkgIndex(pkgfile, [(k, self.header[k]) \
+      for k in keys if self.header[k]])
+    for metadata in sorted(self.packages, key=operator.itemgetter('CPV')):
+      cpv = metadata['CPV']
+      keys = list(metadata)
+      keys.sort()
+      self._WritePkgIndex(pkgfile,
+        [(k, metadata[k]) for k in keys if metadata[k]])
+
+
+def GrabRemotePackageIndex(binhost_url):
+  """Grab the latest binary package database from the specified URL.
+
+  Args:
+    binhost_url: Base URL of remote packages (PORTAGE_BINHOST).
+
+  Returns:
+    A PackageIndex object
+  """
+
+  def _RetryUrlOpen(url, tries=3):
+    """Open the specified url, retrying if we run into temporary errors.
+
+    We retry for both network errors and 5xx Server Errors. We do not retry
+    for HTTP errors with a non-5xx code.
+
+    Args:
+      url: The specified url.
+      tries: The number of times to try.
+
+    Returns:
+      The result of urllib2.urlopen(url).
+    """
+    for i in range(tries):
+      try:
+        return urllib2.urlopen(url)
+      except urllib2.HTTPError as e:
+        if i + 1 >= tries or e.code < 500:
+          raise
+        else:
+          print 'Cannot GET %s: %s' % (url, str(e))
+      except urllib2.URLError as e:
+        if i + 1 >= tries:
+          raise
+        else:
+          print 'Cannot GET %s: %s' % (url, str(e))
+      print "Sleeping for 10 seconds before retrying..."
+      time.sleep(10)
+
+  url = os.path.join(binhost_url, 'Packages')
+  try:
+    f = _RetryUrlOpen(url)
+  except urllib2.HTTPError as e:
+    if e.code == 404:
+      return None
+    raise
+
+  pkgindex = PackageIndex()
+  pkgindex.Read(f)
+  f.close()
+  return pkgindex
+
+