| Index: chromite/bin/cros_build_packages
|
| diff --git a/chromite/bin/cros_build_packages b/chromite/bin/cros_build_packages
|
| deleted file mode 100755
|
| index 0cf2f4f301882db0c8c25bd427c778e02286d97a..0000000000000000000000000000000000000000
|
| --- a/chromite/bin/cros_build_packages
|
| +++ /dev/null
|
| @@ -1,275 +0,0 @@
|
| -#!/usr/bin/python2.6
|
| -# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
|
| -# Use of this source code is governed by a BSD-style license that can be
|
| -# found in the LICENSE file.
|
| -
|
| -import optparse
|
| -import os
|
| -import multiprocessing
|
| -import sys
|
| -import tempfile
|
| -sys.path.insert(0, os.path.abspath(__file__ + "/../../lib"))
|
| -from cros_build_lib import Die
|
| -from cros_build_lib import Info
|
| -from cros_build_lib import RunCommand
|
| -from cros_build_lib import Warning
|
| -
|
| -
|
| -def BuildPackages():
|
| - """Build packages according to options specified on command-line."""
|
| -
|
| - if os.getuid() != 0:
|
| - Die("superuser access required")
|
| -
|
| - scripts_dir = os.path.abspath(__file__ + "/../../..")
|
| - builder = PackageBuilder(scripts_dir)
|
| - options, _ = builder.ParseArgs()
|
| -
|
| - # Calculate packages to install.
|
| - # TODO(davidjames): Grab these from a spec file.
|
| - packages = ["chromeos-base/chromeos"]
|
| - if options.withdev:
|
| - packages.append("chromeos-base/chromeos-dev")
|
| - if options.withfactory:
|
| - packages.append("chromeos-base/chromeos-factoryinstall")
|
| - if options.withtest:
|
| - packages.append("chromeos-base/chromeos-test")
|
| -
|
| - if options.usetarball:
|
| - builder.ExtractTarball(options, packages)
|
| - else:
|
| - builder.BuildTarball(options, packages)
|
| -
|
| -
|
| -def _Apply(args):
|
| - """Call the function specified in args[0], with arguments in args[1:]."""
|
| - return apply(args[0], args[1:])
|
| -
|
| -
|
| -def _GetLatestPrebuiltPrefix(board):
|
| - """Get the latest prebuilt prefix for the specified board.
|
| -
|
| - Args:
|
| - board: The board you want prebuilts for.
|
| - Returns:
|
| - Latest prebuilt prefix.
|
| - """
|
| - # TODO(davidjames): Also append profile names here.
|
| - prefix = "http://commondatastorage.googleapis.com/chromeos-prebuilt/board"
|
| - tmpfile = tempfile.NamedTemporaryFile()
|
| - _Run("curl '%s/%s-latest' -o %s" % (prefix, board, tmpfile.name), retries=3)
|
| - tmpfile.seek(0)
|
| - latest = tmpfile.read().strip()
|
| - tmpfile.close()
|
| - return "%s/%s" % (prefix, latest)
|
| -
|
| -
|
| -def _GetPrebuiltDownloadCommands(prefix):
|
| - """Return a list of commands for grabbing packages.
|
| -
|
| - There must be a file called "packages/Packages" that contains the list of
|
| - packages. The specified list of commands will fill the packages directory
|
| - with the bzipped packages from the specified prefix.
|
| -
|
| - Args:
|
| - prefix: Url prefix to download packages from.
|
| - Returns:
|
| - List of commands for grabbing packages.
|
| - """
|
| -
|
| - cmds = []
|
| - for line in file("packages/Packages"):
|
| - if line.startswith("CPV: "):
|
| - pkgpath, pkgname = line.replace("CPV: ", "").strip().split("/")
|
| - path = "%s/%s.tbz2" % (pkgpath, pkgname)
|
| - url = "%s/%s" % (prefix, path)
|
| - dirname = "packages/%s" % pkgpath
|
| - fullpath = "packages/%s" % path
|
| - if not os.path.exists(dirname):
|
| - os.makedirs(dirname)
|
| - if not os.path.exists(fullpath):
|
| - cmds.append("curl -s %s -o %s" % (url, fullpath))
|
| - return cmds
|
| -
|
| -
|
| -def _Run(cmd, retries=0):
|
| - """Run the specified command.
|
| -
|
| - If the command fails, and the retries have been exhausted, the program exits
|
| - with an appropriate error message.
|
| -
|
| - Args:
|
| - cmd: The command to run.
|
| - retries: If exit code is non-zero, retry this many times.
|
| - """
|
| - # TODO(davidjames): Move this to common library.
|
| - for _ in range(retries+1):
|
| - result = RunCommand(cmd, shell=True, exit_code=True, error_ok=True)
|
| - if result.returncode == 0:
|
| - Info("Command succeeded: %s" % cmd)
|
| - break
|
| - Warning("Command failed: %s" % cmd)
|
| - else:
|
| - Die("Command failed, exiting: %s" % cmd)
|
| -
|
| -
|
| -def _RunManyParallel(cmds, retries=0):
|
| - """Run list of provided commands in parallel.
|
| -
|
| - To work around a bug in the multiprocessing module, we use map_async instead
|
| - of the usual map function. See http://bugs.python.org/issue9205
|
| -
|
| - Args:
|
| - cmds: List of commands to run.
|
| - retries: Number of retries per command.
|
| - """
|
| - # TODO(davidjames): Move this to common library.
|
| - pool = multiprocessing.Pool()
|
| - args = []
|
| - for cmd in cmds:
|
| - args.append((_Run, cmd, retries))
|
| - result = pool.map_async(_Apply, args, chunksize=1)
|
| - while True:
|
| - try:
|
| - result.get(60*60)
|
| - break
|
| - except multiprocessing.TimeoutError:
|
| - pass
|
| -
|
| -
|
| -class PackageBuilder(object):
|
| - """A class for building and extracting tarballs of Chromium OS packages."""
|
| -
|
| - def __init__(self, scripts_dir):
|
| - self.scripts_dir = scripts_dir
|
| -
|
| - def BuildTarball(self, options, packages):
|
| - """Build a tarball with the specified packages.
|
| -
|
| - Args:
|
| - options: Options object, as output by ParseArgs.
|
| - packages: List of packages to build.
|
| - """
|
| -
|
| - board = options.board
|
| -
|
| - # Run setup_board. TODO(davidjames): Integrate the logic used in
|
| - # setup_board into chromite.
|
| - _Run("%s/setup_board --force --board=%s" % (self.scripts_dir, board))
|
| -
|
| - # Create complete build directory
|
| - _Run(self._EmergeBoardCmd(options, packages))
|
| -
|
| - # Archive build directory as tarballs
|
| - os.chdir("/build/%s" % board)
|
| - cmds = [
|
| - "tar -c --wildcards --exclude='usr/lib/debug/*' "
|
| - "--exclude='packages/*' * | pigz -c > packages/%s-build.tgz" % board,
|
| - "tar -c usr/lib/debug/* | pigz -c > packages/%s-debug.tgz" % board
|
| - ]
|
| -
|
| - # Run list of commands.
|
| - _RunManyParallel(cmds)
|
| -
|
| - def ExtractTarball(self, options, packages):
|
| - """Extract the latest build tarball, then update the specified packages.
|
| -
|
| - Args:
|
| - options: Options object, as output by ParseArgs.
|
| - packages: List of packages to update.
|
| - """
|
| -
|
| - board = options.board
|
| - prefix = _GetLatestPrebuiltPrefix(board)
|
| -
|
| - # If the user doesn't have emerge-${BOARD} setup yet, we need to run
|
| - # setup_board. TODO(davidjames): Integrate the logic used in setup_board
|
| - # into chromite.
|
| - if not os.path.exists("/usr/local/bin/emerge-%s" % board):
|
| - _Run("%s/setup_board --force --board=%s" % (self.scripts_dir, board))
|
| -
|
| - # Delete old build directory. This process might take a while, so do it in
|
| - # the background.
|
| - cmds = []
|
| - if os.path.exists("/build/%s" % board):
|
| - tempdir = tempfile.mkdtemp()
|
| - _Run("mv /build/%s %s" % (board, tempdir))
|
| - cmds.append("rm -rf %s" % tempdir)
|
| -
|
| - # Create empty build directory, and chdir into it.
|
| - os.makedirs("/build/%s/packages" % board)
|
| - os.chdir("/build/%s" % board)
|
| -
|
| - # Download and expand build tarball.
|
| - build_url = "%s/%s-build.tgz" % (prefix, board)
|
| - cmds.append("curl -s %s | tar -xz" % build_url)
|
| -
|
| - # Download and expand debug tarball (if requested).
|
| - if options.debug:
|
| - debug_url = "%s/%s-debug.tgz" % (prefix, board)
|
| - cmds.append("curl -s %s | tar -xz" % debug_url)
|
| -
|
| - # Download prebuilt packages.
|
| - _Run("curl '%s/Packages' -o packages/Packages" % prefix, retries=3)
|
| - cmds.extend(_GetPrebuiltDownloadCommands(prefix))
|
| -
|
| - # Run list of commands, with three retries per command, in case the network
|
| - # is flaky.
|
| - _RunManyParallel(cmds, retries=3)
|
| -
|
| - # Emerge remaining packages.
|
| - _Run(self._EmergeBoardCmd(options, packages))
|
| -
|
| - def ParseArgs(self):
|
| - """Parse arguments from the command line using optparse."""
|
| -
|
| - # TODO(davidjames): We should use spec files for this.
|
| - default_board = self._GetDefaultBoard()
|
| - parser = optparse.OptionParser()
|
| - parser.add_option("--board", dest="board", default=default_board,
|
| - help="The board to build packages for.")
|
| - parser.add_option("--debug", action="store_true", dest="debug",
|
| - default=False, help="Include debug symbols.")
|
| - parser.add_option("--nowithdev", action="store_false", dest="withdev",
|
| - default=True,
|
| - help="Don't build useful developer friendly utilities.")
|
| - parser.add_option("--nowithtest", action="store_false", dest="withtest",
|
| - default=True, help="Build packages required for testing.")
|
| - parser.add_option("--nowithfactory", action="store_false",
|
| - dest="withfactory", default=True,
|
| - help="Build factory installer")
|
| - parser.add_option("--nousepkg", action="store_false",
|
| - dest="usepkg", default=True,
|
| - help="Don't use binary packages.")
|
| - parser.add_option("--nousetarball", action="store_false",
|
| - dest="usetarball", default=True,
|
| - help="Don't use tarball.")
|
| - parser.add_option("--nofast", action="store_false", dest="fast",
|
| - default=True,
|
| - help="Don't merge packages in parallel.")
|
| -
|
| - return parser.parse_args()
|
| -
|
| - def _EmergeBoardCmd(self, options, packages):
|
| - """Calculate board emerge command."""
|
| - board = options.board
|
| - scripts_dir = self.scripts_dir
|
| - emerge_board = "emerge-%s" % board
|
| - if options.fast:
|
| - emerge_board = "%s/parallel_emerge --board=%s" % (scripts_dir, board)
|
| - usepkg = ""
|
| - if options.usepkg:
|
| - usepkg = "g"
|
| - return "%s -uDNv%s %s" % (emerge_board, usepkg, " ".join(packages))
|
| -
|
| - def _GetDefaultBoard(self):
|
| - """Get the default board configured by the user."""
|
| -
|
| - default_board_file = "%s/.default_board" % self.scripts_dir
|
| - default_board = None
|
| - if os.path.exists(default_board_file):
|
| - default_board = file(default_board_file).read().strip()
|
| - return default_board
|
| -
|
| -if __name__ == "__main__":
|
| - BuildPackages()
|
|
|