Index: chromite/bin/cros_build_packages |
diff --git a/chromite/bin/cros_build_packages b/chromite/bin/cros_build_packages |
deleted file mode 100755 |
index 0cf2f4f301882db0c8c25bd427c778e02286d97a..0000000000000000000000000000000000000000 |
--- a/chromite/bin/cros_build_packages |
+++ /dev/null |
@@ -1,275 +0,0 @@ |
-#!/usr/bin/python2.6 |
-# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. |
-# Use of this source code is governed by a BSD-style license that can be |
-# found in the LICENSE file. |
- |
-import optparse |
-import os |
-import multiprocessing |
-import sys |
-import tempfile |
-sys.path.insert(0, os.path.abspath(__file__ + "/../../lib")) |
-from cros_build_lib import Die |
-from cros_build_lib import Info |
-from cros_build_lib import RunCommand |
-from cros_build_lib import Warning |
- |
- |
-def BuildPackages(): |
- """Build packages according to options specified on command-line.""" |
- |
- if os.getuid() != 0: |
- Die("superuser access required") |
- |
- scripts_dir = os.path.abspath(__file__ + "/../../..") |
- builder = PackageBuilder(scripts_dir) |
- options, _ = builder.ParseArgs() |
- |
- # Calculate packages to install. |
- # TODO(davidjames): Grab these from a spec file. |
- packages = ["chromeos-base/chromeos"] |
- if options.withdev: |
- packages.append("chromeos-base/chromeos-dev") |
- if options.withfactory: |
- packages.append("chromeos-base/chromeos-factoryinstall") |
- if options.withtest: |
- packages.append("chromeos-base/chromeos-test") |
- |
- if options.usetarball: |
- builder.ExtractTarball(options, packages) |
- else: |
- builder.BuildTarball(options, packages) |
- |
- |
-def _Apply(args): |
- """Call the function specified in args[0], with arguments in args[1:].""" |
- return apply(args[0], args[1:]) |
- |
- |
-def _GetLatestPrebuiltPrefix(board): |
- """Get the latest prebuilt prefix for the specified board. |
- |
- Args: |
- board: The board you want prebuilts for. |
- Returns: |
- Latest prebuilt prefix. |
- """ |
- # TODO(davidjames): Also append profile names here. |
- prefix = "http://commondatastorage.googleapis.com/chromeos-prebuilt/board" |
- tmpfile = tempfile.NamedTemporaryFile() |
- _Run("curl '%s/%s-latest' -o %s" % (prefix, board, tmpfile.name), retries=3) |
- tmpfile.seek(0) |
- latest = tmpfile.read().strip() |
- tmpfile.close() |
- return "%s/%s" % (prefix, latest) |
- |
- |
-def _GetPrebuiltDownloadCommands(prefix): |
- """Return a list of commands for grabbing packages. |
- |
- There must be a file called "packages/Packages" that contains the list of |
- packages. The specified list of commands will fill the packages directory |
- with the bzipped packages from the specified prefix. |
- |
- Args: |
- prefix: Url prefix to download packages from. |
- Returns: |
- List of commands for grabbing packages. |
- """ |
- |
- cmds = [] |
- for line in file("packages/Packages"): |
- if line.startswith("CPV: "): |
- pkgpath, pkgname = line.replace("CPV: ", "").strip().split("/") |
- path = "%s/%s.tbz2" % (pkgpath, pkgname) |
- url = "%s/%s" % (prefix, path) |
- dirname = "packages/%s" % pkgpath |
- fullpath = "packages/%s" % path |
- if not os.path.exists(dirname): |
- os.makedirs(dirname) |
- if not os.path.exists(fullpath): |
- cmds.append("curl -s %s -o %s" % (url, fullpath)) |
- return cmds |
- |
- |
-def _Run(cmd, retries=0): |
- """Run the specified command. |
- |
- If the command fails, and the retries have been exhausted, the program exits |
- with an appropriate error message. |
- |
- Args: |
- cmd: The command to run. |
- retries: If exit code is non-zero, retry this many times. |
- """ |
- # TODO(davidjames): Move this to common library. |
- for _ in range(retries+1): |
- result = RunCommand(cmd, shell=True, exit_code=True, error_ok=True) |
- if result.returncode == 0: |
- Info("Command succeeded: %s" % cmd) |
- break |
- Warning("Command failed: %s" % cmd) |
- else: |
- Die("Command failed, exiting: %s" % cmd) |
- |
- |
-def _RunManyParallel(cmds, retries=0): |
- """Run list of provided commands in parallel. |
- |
- To work around a bug in the multiprocessing module, we use map_async instead |
- of the usual map function. See http://bugs.python.org/issue9205 |
- |
- Args: |
- cmds: List of commands to run. |
- retries: Number of retries per command. |
- """ |
- # TODO(davidjames): Move this to common library. |
- pool = multiprocessing.Pool() |
- args = [] |
- for cmd in cmds: |
- args.append((_Run, cmd, retries)) |
- result = pool.map_async(_Apply, args, chunksize=1) |
- while True: |
- try: |
- result.get(60*60) |
- break |
- except multiprocessing.TimeoutError: |
- pass |
- |
- |
-class PackageBuilder(object): |
- """A class for building and extracting tarballs of Chromium OS packages.""" |
- |
- def __init__(self, scripts_dir): |
- self.scripts_dir = scripts_dir |
- |
- def BuildTarball(self, options, packages): |
- """Build a tarball with the specified packages. |
- |
- Args: |
- options: Options object, as output by ParseArgs. |
- packages: List of packages to build. |
- """ |
- |
- board = options.board |
- |
- # Run setup_board. TODO(davidjames): Integrate the logic used in |
- # setup_board into chromite. |
- _Run("%s/setup_board --force --board=%s" % (self.scripts_dir, board)) |
- |
- # Create complete build directory |
- _Run(self._EmergeBoardCmd(options, packages)) |
- |
- # Archive build directory as tarballs |
- os.chdir("/build/%s" % board) |
- cmds = [ |
- "tar -c --wildcards --exclude='usr/lib/debug/*' " |
- "--exclude='packages/*' * | pigz -c > packages/%s-build.tgz" % board, |
- "tar -c usr/lib/debug/* | pigz -c > packages/%s-debug.tgz" % board |
- ] |
- |
- # Run list of commands. |
- _RunManyParallel(cmds) |
- |
- def ExtractTarball(self, options, packages): |
- """Extract the latest build tarball, then update the specified packages. |
- |
- Args: |
- options: Options object, as output by ParseArgs. |
- packages: List of packages to update. |
- """ |
- |
- board = options.board |
- prefix = _GetLatestPrebuiltPrefix(board) |
- |
- # If the user doesn't have emerge-${BOARD} setup yet, we need to run |
- # setup_board. TODO(davidjames): Integrate the logic used in setup_board |
- # into chromite. |
- if not os.path.exists("/usr/local/bin/emerge-%s" % board): |
- _Run("%s/setup_board --force --board=%s" % (self.scripts_dir, board)) |
- |
- # Delete old build directory. This process might take a while, so do it in |
- # the background. |
- cmds = [] |
- if os.path.exists("/build/%s" % board): |
- tempdir = tempfile.mkdtemp() |
- _Run("mv /build/%s %s" % (board, tempdir)) |
- cmds.append("rm -rf %s" % tempdir) |
- |
- # Create empty build directory, and chdir into it. |
- os.makedirs("/build/%s/packages" % board) |
- os.chdir("/build/%s" % board) |
- |
- # Download and expand build tarball. |
- build_url = "%s/%s-build.tgz" % (prefix, board) |
- cmds.append("curl -s %s | tar -xz" % build_url) |
- |
- # Download and expand debug tarball (if requested). |
- if options.debug: |
- debug_url = "%s/%s-debug.tgz" % (prefix, board) |
- cmds.append("curl -s %s | tar -xz" % debug_url) |
- |
- # Download prebuilt packages. |
- _Run("curl '%s/Packages' -o packages/Packages" % prefix, retries=3) |
- cmds.extend(_GetPrebuiltDownloadCommands(prefix)) |
- |
- # Run list of commands, with three retries per command, in case the network |
- # is flaky. |
- _RunManyParallel(cmds, retries=3) |
- |
- # Emerge remaining packages. |
- _Run(self._EmergeBoardCmd(options, packages)) |
- |
- def ParseArgs(self): |
- """Parse arguments from the command line using optparse.""" |
- |
- # TODO(davidjames): We should use spec files for this. |
- default_board = self._GetDefaultBoard() |
- parser = optparse.OptionParser() |
- parser.add_option("--board", dest="board", default=default_board, |
- help="The board to build packages for.") |
- parser.add_option("--debug", action="store_true", dest="debug", |
- default=False, help="Include debug symbols.") |
- parser.add_option("--nowithdev", action="store_false", dest="withdev", |
- default=True, |
- help="Don't build useful developer friendly utilities.") |
- parser.add_option("--nowithtest", action="store_false", dest="withtest", |
- default=True, help="Build packages required for testing.") |
- parser.add_option("--nowithfactory", action="store_false", |
- dest="withfactory", default=True, |
- help="Build factory installer") |
- parser.add_option("--nousepkg", action="store_false", |
- dest="usepkg", default=True, |
- help="Don't use binary packages.") |
- parser.add_option("--nousetarball", action="store_false", |
- dest="usetarball", default=True, |
- help="Don't use tarball.") |
- parser.add_option("--nofast", action="store_false", dest="fast", |
- default=True, |
- help="Don't merge packages in parallel.") |
- |
- return parser.parse_args() |
- |
- def _EmergeBoardCmd(self, options, packages): |
- """Calculate board emerge command.""" |
- board = options.board |
- scripts_dir = self.scripts_dir |
- emerge_board = "emerge-%s" % board |
- if options.fast: |
- emerge_board = "%s/parallel_emerge --board=%s" % (scripts_dir, board) |
- usepkg = "" |
- if options.usepkg: |
- usepkg = "g" |
- return "%s -uDNv%s %s" % (emerge_board, usepkg, " ".join(packages)) |
- |
- def _GetDefaultBoard(self): |
- """Get the default board configured by the user.""" |
- |
- default_board_file = "%s/.default_board" % self.scripts_dir |
- default_board = None |
- if os.path.exists(default_board_file): |
- default_board = file(default_board_file).read().strip() |
- return default_board |
- |
-if __name__ == "__main__": |
- BuildPackages() |