Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(180)

Unified Diff: chromite/bin/cros_build_packages

Issue 6371018: Remove chromite from crosutils.git. It's been moved to chromite.git. (Closed) Base URL: http://git.chromium.org/git/crosutils.git@master
Patch Set: Created 9 years, 11 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « chromite/__init__.py ('k') | chromite/bin/cros_changelog » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: chromite/bin/cros_build_packages
diff --git a/chromite/bin/cros_build_packages b/chromite/bin/cros_build_packages
deleted file mode 100755
index 0cf2f4f301882db0c8c25bd427c778e02286d97a..0000000000000000000000000000000000000000
--- a/chromite/bin/cros_build_packages
+++ /dev/null
@@ -1,275 +0,0 @@
-#!/usr/bin/python2.6
-# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import optparse
-import os
-import multiprocessing
-import sys
-import tempfile
-sys.path.insert(0, os.path.abspath(__file__ + "/../../lib"))
-from cros_build_lib import Die
-from cros_build_lib import Info
-from cros_build_lib import RunCommand
-from cros_build_lib import Warning
-
-
-def BuildPackages():
- """Build packages according to options specified on command-line."""
-
- if os.getuid() != 0:
- Die("superuser access required")
-
- scripts_dir = os.path.abspath(__file__ + "/../../..")
- builder = PackageBuilder(scripts_dir)
- options, _ = builder.ParseArgs()
-
- # Calculate packages to install.
- # TODO(davidjames): Grab these from a spec file.
- packages = ["chromeos-base/chromeos"]
- if options.withdev:
- packages.append("chromeos-base/chromeos-dev")
- if options.withfactory:
- packages.append("chromeos-base/chromeos-factoryinstall")
- if options.withtest:
- packages.append("chromeos-base/chromeos-test")
-
- if options.usetarball:
- builder.ExtractTarball(options, packages)
- else:
- builder.BuildTarball(options, packages)
-
-
-def _Apply(args):
- """Call the function specified in args[0], with arguments in args[1:]."""
- return apply(args[0], args[1:])
-
-
-def _GetLatestPrebuiltPrefix(board):
- """Get the latest prebuilt prefix for the specified board.
-
- Args:
- board: The board you want prebuilts for.
- Returns:
- Latest prebuilt prefix.
- """
- # TODO(davidjames): Also append profile names here.
- prefix = "http://commondatastorage.googleapis.com/chromeos-prebuilt/board"
- tmpfile = tempfile.NamedTemporaryFile()
- _Run("curl '%s/%s-latest' -o %s" % (prefix, board, tmpfile.name), retries=3)
- tmpfile.seek(0)
- latest = tmpfile.read().strip()
- tmpfile.close()
- return "%s/%s" % (prefix, latest)
-
-
-def _GetPrebuiltDownloadCommands(prefix):
- """Return a list of commands for grabbing packages.
-
- There must be a file called "packages/Packages" that contains the list of
- packages. The specified list of commands will fill the packages directory
- with the bzipped packages from the specified prefix.
-
- Args:
- prefix: Url prefix to download packages from.
- Returns:
- List of commands for grabbing packages.
- """
-
- cmds = []
- for line in file("packages/Packages"):
- if line.startswith("CPV: "):
- pkgpath, pkgname = line.replace("CPV: ", "").strip().split("/")
- path = "%s/%s.tbz2" % (pkgpath, pkgname)
- url = "%s/%s" % (prefix, path)
- dirname = "packages/%s" % pkgpath
- fullpath = "packages/%s" % path
- if not os.path.exists(dirname):
- os.makedirs(dirname)
- if not os.path.exists(fullpath):
- cmds.append("curl -s %s -o %s" % (url, fullpath))
- return cmds
-
-
-def _Run(cmd, retries=0):
- """Run the specified command.
-
- If the command fails, and the retries have been exhausted, the program exits
- with an appropriate error message.
-
- Args:
- cmd: The command to run.
- retries: If exit code is non-zero, retry this many times.
- """
- # TODO(davidjames): Move this to common library.
- for _ in range(retries+1):
- result = RunCommand(cmd, shell=True, exit_code=True, error_ok=True)
- if result.returncode == 0:
- Info("Command succeeded: %s" % cmd)
- break
- Warning("Command failed: %s" % cmd)
- else:
- Die("Command failed, exiting: %s" % cmd)
-
-
-def _RunManyParallel(cmds, retries=0):
- """Run list of provided commands in parallel.
-
- To work around a bug in the multiprocessing module, we use map_async instead
- of the usual map function. See http://bugs.python.org/issue9205
-
- Args:
- cmds: List of commands to run.
- retries: Number of retries per command.
- """
- # TODO(davidjames): Move this to common library.
- pool = multiprocessing.Pool()
- args = []
- for cmd in cmds:
- args.append((_Run, cmd, retries))
- result = pool.map_async(_Apply, args, chunksize=1)
- while True:
- try:
- result.get(60*60)
- break
- except multiprocessing.TimeoutError:
- pass
-
-
-class PackageBuilder(object):
- """A class for building and extracting tarballs of Chromium OS packages."""
-
- def __init__(self, scripts_dir):
- self.scripts_dir = scripts_dir
-
- def BuildTarball(self, options, packages):
- """Build a tarball with the specified packages.
-
- Args:
- options: Options object, as output by ParseArgs.
- packages: List of packages to build.
- """
-
- board = options.board
-
- # Run setup_board. TODO(davidjames): Integrate the logic used in
- # setup_board into chromite.
- _Run("%s/setup_board --force --board=%s" % (self.scripts_dir, board))
-
- # Create complete build directory
- _Run(self._EmergeBoardCmd(options, packages))
-
- # Archive build directory as tarballs
- os.chdir("/build/%s" % board)
- cmds = [
- "tar -c --wildcards --exclude='usr/lib/debug/*' "
- "--exclude='packages/*' * | pigz -c > packages/%s-build.tgz" % board,
- "tar -c usr/lib/debug/* | pigz -c > packages/%s-debug.tgz" % board
- ]
-
- # Run list of commands.
- _RunManyParallel(cmds)
-
- def ExtractTarball(self, options, packages):
- """Extract the latest build tarball, then update the specified packages.
-
- Args:
- options: Options object, as output by ParseArgs.
- packages: List of packages to update.
- """
-
- board = options.board
- prefix = _GetLatestPrebuiltPrefix(board)
-
- # If the user doesn't have emerge-${BOARD} setup yet, we need to run
- # setup_board. TODO(davidjames): Integrate the logic used in setup_board
- # into chromite.
- if not os.path.exists("/usr/local/bin/emerge-%s" % board):
- _Run("%s/setup_board --force --board=%s" % (self.scripts_dir, board))
-
- # Delete old build directory. This process might take a while, so do it in
- # the background.
- cmds = []
- if os.path.exists("/build/%s" % board):
- tempdir = tempfile.mkdtemp()
- _Run("mv /build/%s %s" % (board, tempdir))
- cmds.append("rm -rf %s" % tempdir)
-
- # Create empty build directory, and chdir into it.
- os.makedirs("/build/%s/packages" % board)
- os.chdir("/build/%s" % board)
-
- # Download and expand build tarball.
- build_url = "%s/%s-build.tgz" % (prefix, board)
- cmds.append("curl -s %s | tar -xz" % build_url)
-
- # Download and expand debug tarball (if requested).
- if options.debug:
- debug_url = "%s/%s-debug.tgz" % (prefix, board)
- cmds.append("curl -s %s | tar -xz" % debug_url)
-
- # Download prebuilt packages.
- _Run("curl '%s/Packages' -o packages/Packages" % prefix, retries=3)
- cmds.extend(_GetPrebuiltDownloadCommands(prefix))
-
- # Run list of commands, with three retries per command, in case the network
- # is flaky.
- _RunManyParallel(cmds, retries=3)
-
- # Emerge remaining packages.
- _Run(self._EmergeBoardCmd(options, packages))
-
- def ParseArgs(self):
- """Parse arguments from the command line using optparse."""
-
- # TODO(davidjames): We should use spec files for this.
- default_board = self._GetDefaultBoard()
- parser = optparse.OptionParser()
- parser.add_option("--board", dest="board", default=default_board,
- help="The board to build packages for.")
- parser.add_option("--debug", action="store_true", dest="debug",
- default=False, help="Include debug symbols.")
- parser.add_option("--nowithdev", action="store_false", dest="withdev",
- default=True,
- help="Don't build useful developer friendly utilities.")
- parser.add_option("--nowithtest", action="store_false", dest="withtest",
- default=True, help="Build packages required for testing.")
- parser.add_option("--nowithfactory", action="store_false",
- dest="withfactory", default=True,
- help="Build factory installer")
- parser.add_option("--nousepkg", action="store_false",
- dest="usepkg", default=True,
- help="Don't use binary packages.")
- parser.add_option("--nousetarball", action="store_false",
- dest="usetarball", default=True,
- help="Don't use tarball.")
- parser.add_option("--nofast", action="store_false", dest="fast",
- default=True,
- help="Don't merge packages in parallel.")
-
- return parser.parse_args()
-
- def _EmergeBoardCmd(self, options, packages):
- """Calculate board emerge command."""
- board = options.board
- scripts_dir = self.scripts_dir
- emerge_board = "emerge-%s" % board
- if options.fast:
- emerge_board = "%s/parallel_emerge --board=%s" % (scripts_dir, board)
- usepkg = ""
- if options.usepkg:
- usepkg = "g"
- return "%s -uDNv%s %s" % (emerge_board, usepkg, " ".join(packages))
-
- def _GetDefaultBoard(self):
- """Get the default board configured by the user."""
-
- default_board_file = "%s/.default_board" % self.scripts_dir
- default_board = None
- if os.path.exists(default_board_file):
- default_board = file(default_board_file).read().strip()
- return default_board
-
-if __name__ == "__main__":
- BuildPackages()
« no previous file with comments | « chromite/__init__.py ('k') | chromite/bin/cros_changelog » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698