OLD | NEW |
(Empty) | |
| 1 #!/usr/bin/python2.6 |
| 2 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. |
| 5 |
| 6 import optparse |
| 7 import os |
| 8 import multiprocessing |
| 9 import sys |
| 10 import tempfile |
| 11 sys.path.insert(0, os.path.abspath(__file__ + "/../../lib")) |
| 12 from cros_build_lib import Die |
| 13 from cros_build_lib import Info |
| 14 from cros_build_lib import RunCommand |
| 15 from cros_build_lib import Warning |
| 16 |
| 17 |
| 18 def BuildPackages(): |
| 19 """Build packages according to options specified on command-line.""" |
| 20 |
| 21 if os.getuid() != 0: |
| 22 Die("superuser access required") |
| 23 |
| 24 scripts_dir = os.path.abspath(__file__ + "/../../..") |
| 25 builder = PackageBuilder(scripts_dir) |
| 26 options, _ = builder.ParseArgs() |
| 27 |
| 28 # Calculate packages to install. |
| 29 # TODO(davidjames): Grab these from a spec file. |
| 30 packages = ["chromeos-base/chromeos"] |
| 31 if options.withdev: |
| 32 packages.append("chromeos-base/chromeos-dev") |
| 33 if options.withfactory: |
| 34 packages.append("chromeos-base/chromeos-factoryinstall") |
| 35 if options.withtest: |
| 36 packages.append("chromeos-base/chromeos-test") |
| 37 |
| 38 if options.usetarball: |
| 39 builder.ExtractTarball(options, packages) |
| 40 else: |
| 41 builder.BuildTarball(options, packages) |
| 42 |
| 43 |
| 44 def _Apply(args): |
| 45 """Call the function specified in args[0], with arguments in args[1:].""" |
| 46 return apply(args[0], args[1:]) |
| 47 |
| 48 |
| 49 def _GetLatestPrebuiltPrefix(board): |
| 50 """Get the latest prebuilt prefix for the specified board. |
| 51 |
| 52 Args: |
| 53 board: The board you want prebuilts for. |
| 54 Returns: |
| 55 Latest prebuilt prefix. |
| 56 """ |
| 57 # TODO(davidjames): Also append profile names here. |
| 58 prefix = "http://commondatastorage.googleapis.com/chromeos-prebuilt/board" |
| 59 tmpfile = tempfile.NamedTemporaryFile() |
| 60 _Run("curl '%s/%s-latest' -o %s" % (prefix, board, tmpfile.name), retries=3) |
| 61 tmpfile.seek(0) |
| 62 latest = tmpfile.read().strip() |
| 63 tmpfile.close() |
| 64 return "%s/%s" % (prefix, latest) |
| 65 |
| 66 |
| 67 def _GetPrebuiltDownloadCommands(prefix): |
| 68 """Return a list of commands for grabbing packages. |
| 69 |
| 70 There must be a file called "packages/Packages" that contains the list of |
| 71 packages. The specified list of commands will fill the packages directory |
| 72 with the bzipped packages from the specified prefix. |
| 73 |
| 74 Args: |
| 75 prefix: Url prefix to download packages from. |
| 76 Returns: |
| 77 List of commands for grabbing packages. |
| 78 """ |
| 79 |
| 80 cmds = [] |
| 81 for line in file("packages/Packages"): |
| 82 if line.startswith("CPV: "): |
| 83 pkgpath, pkgname = line.replace("CPV: ", "").strip().split("/") |
| 84 path = "%s/%s.tbz2" % (pkgpath, pkgname) |
| 85 url = "%s/%s" % (prefix, path) |
| 86 dirname = "packages/%s" % pkgpath |
| 87 fullpath = "packages/%s" % path |
| 88 if not os.path.exists(dirname): |
| 89 os.makedirs(dirname) |
| 90 if not os.path.exists(fullpath): |
| 91 cmds.append("curl -s %s -o %s" % (url, fullpath)) |
| 92 return cmds |
| 93 |
| 94 |
| 95 def _Run(cmd, retries=0): |
| 96 """Run the specified command. |
| 97 |
| 98 If the command fails, and the retries have been exhausted, the program exits |
| 99 with an appropriate error message. |
| 100 |
| 101 Args: |
| 102 cmd: The command to run. |
| 103 retries: If exit code is non-zero, retry this many times. |
| 104 """ |
| 105 # TODO(davidjames): Move this to common library. |
| 106 for _ in range(retries+1): |
| 107 result = RunCommand(cmd, shell=True, exit_code=True, error_ok=True) |
| 108 if result.returncode == 0: |
| 109 Info("Command succeeded: %s" % cmd) |
| 110 break |
| 111 Warning("Command failed: %s" % cmd) |
| 112 else: |
| 113 Die("Command failed, exiting: %s" % cmd) |
| 114 |
| 115 |
| 116 def _RunManyParallel(cmds, retries=0): |
| 117 """Run list of provided commands in parallel. |
| 118 |
| 119 To work around a bug in the multiprocessing module, we use map_async instead |
| 120 of the usual map function. See http://bugs.python.org/issue9205 |
| 121 |
| 122 Args: |
| 123 cmds: List of commands to run. |
| 124 retries: Number of retries per command. |
| 125 """ |
| 126 # TODO(davidjames): Move this to common library. |
| 127 pool = multiprocessing.Pool() |
| 128 args = [] |
| 129 for cmd in cmds: |
| 130 args.append((_Run, cmd, retries)) |
| 131 result = pool.map_async(_Apply, args, chunksize=1) |
| 132 while True: |
| 133 try: |
| 134 result.get(60*60) |
| 135 break |
| 136 except multiprocessing.TimeoutError: |
| 137 pass |
| 138 |
| 139 |
| 140 class PackageBuilder(object): |
| 141 """A class for building and extracting tarballs of Chromium OS packages.""" |
| 142 |
| 143 def __init__(self, scripts_dir): |
| 144 self.scripts_dir = scripts_dir |
| 145 |
| 146 def BuildTarball(self, options, packages): |
| 147 """Build a tarball with the specified packages. |
| 148 |
| 149 Args: |
| 150 options: Options object, as output by ParseArgs. |
| 151 packages: List of packages to build. |
| 152 """ |
| 153 |
| 154 board = options.board |
| 155 |
| 156 # Run setup_board. TODO(davidjames): Integrate the logic used in |
| 157 # setup_board into chromite. |
| 158 _Run("%s/setup_board --force --board=%s" % (self.scripts_dir, board)) |
| 159 |
| 160 # Create complete build directory |
| 161 _Run(self._EmergeBoardCmd(options, packages)) |
| 162 |
| 163 # Archive build directory as tarballs |
| 164 os.chdir("/build/%s" % board) |
| 165 cmds = [ |
| 166 "tar -c --wildcards --exclude='usr/lib/debug/*' " |
| 167 "--exclude='packages/*' * | pigz -c > packages/%s-build.tgz" % board, |
| 168 "tar -c usr/lib/debug/* | pigz -c > packages/%s-debug.tgz" % board |
| 169 ] |
| 170 |
| 171 # Run list of commands. |
| 172 _RunManyParallel(cmds) |
| 173 |
| 174 def ExtractTarball(self, options, packages): |
| 175 """Extract the latest build tarball, then update the specified packages. |
| 176 |
| 177 Args: |
| 178 options: Options object, as output by ParseArgs. |
| 179 packages: List of packages to update. |
| 180 """ |
| 181 |
| 182 board = options.board |
| 183 prefix = _GetLatestPrebuiltPrefix(board) |
| 184 |
| 185 # If the user doesn't have emerge-${BOARD} setup yet, we need to run |
| 186 # setup_board. TODO(davidjames): Integrate the logic used in setup_board |
| 187 # into chromite. |
| 188 if not os.path.exists("/usr/local/bin/emerge-%s" % board): |
| 189 _Run("%s/setup_board --force --board=%s" % (self.scripts_dir, board)) |
| 190 |
| 191 # Delete old build directory. This process might take a while, so do it in |
| 192 # the background. |
| 193 cmds = [] |
| 194 if os.path.exists("/build/%s" % board): |
| 195 tempdir = tempfile.mkdtemp() |
| 196 _Run("mv /build/%s %s" % (board, tempdir)) |
| 197 cmds.append("rm -rf %s" % tempdir) |
| 198 |
| 199 # Create empty build directory, and chdir into it. |
| 200 os.makedirs("/build/%s/packages" % board) |
| 201 os.chdir("/build/%s" % board) |
| 202 |
| 203 # Download and expand build tarball. |
| 204 build_url = "%s/%s-build.tgz" % (prefix, board) |
| 205 cmds.append("curl -s %s | tar -xz" % build_url) |
| 206 |
| 207 # Download and expand debug tarball (if requested). |
| 208 if options.debug: |
| 209 debug_url = "%s/%s-debug.tgz" % (prefix, board) |
| 210 cmds.append("curl -s %s | tar -xz" % debug_url) |
| 211 |
| 212 # Download prebuilt packages. |
| 213 _Run("curl '%s/Packages' -o packages/Packages" % prefix, retries=3) |
| 214 cmds.extend(_GetPrebuiltDownloadCommands(prefix)) |
| 215 |
| 216 # Run list of commands, with three retries per command, in case the network |
| 217 # is flaky. |
| 218 _RunManyParallel(cmds, retries=3) |
| 219 |
| 220 # Emerge remaining packages. |
| 221 _Run(self._EmergeBoardCmd(options, packages)) |
| 222 |
| 223 def ParseArgs(self): |
| 224 """Parse arguments from the command line using optparse.""" |
| 225 |
| 226 # TODO(davidjames): We should use spec files for this. |
| 227 default_board = self._GetDefaultBoard() |
| 228 parser = optparse.OptionParser() |
| 229 parser.add_option("--board", dest="board", default=default_board, |
| 230 help="The board to build packages for.") |
| 231 parser.add_option("--debug", action="store_true", dest="debug", |
| 232 default=False, help="Include debug symbols.") |
| 233 parser.add_option("--nowithdev", action="store_false", dest="withdev", |
| 234 default=True, |
| 235 help="Don't build useful developer friendly utilities.") |
| 236 parser.add_option("--nowithtest", action="store_false", dest="withtest", |
| 237 default=True, help="Build packages required for testing.") |
| 238 parser.add_option("--nowithfactory", action="store_false", |
| 239 dest="withfactory", default=True, |
| 240 help="Build factory installer") |
| 241 parser.add_option("--nousepkg", action="store_false", |
| 242 dest="usepkg", default=True, |
| 243 help="Don't use binary packages.") |
| 244 parser.add_option("--nousetarball", action="store_false", |
| 245 dest="usetarball", default=True, |
| 246 help="Don't use tarball.") |
| 247 parser.add_option("--nofast", action="store_false", dest="fast", |
| 248 default=True, |
| 249 help="Don't merge packages in parallel.") |
| 250 |
| 251 return parser.parse_args() |
| 252 |
| 253 def _EmergeBoardCmd(self, options, packages): |
| 254 """Calculate board emerge command.""" |
| 255 board = options.board |
| 256 scripts_dir = self.scripts_dir |
| 257 emerge_board = "emerge-%s" % board |
| 258 if options.fast: |
| 259 emerge_board = "%s/parallel_emerge --board=%s" % (scripts_dir, board) |
| 260 usepkg = "" |
| 261 if options.usepkg: |
| 262 usepkg = "g" |
| 263 return "%s -uDNv%s %s" % (emerge_board, usepkg, " ".join(packages)) |
| 264 |
| 265 def _GetDefaultBoard(self): |
| 266 """Get the default board configured by the user.""" |
| 267 |
| 268 default_board_file = "%s/.default_board" % self.scripts_dir |
| 269 default_board = None |
| 270 if os.path.exists(default_board_file): |
| 271 default_board = file(default_board_file).read().strip() |
| 272 return default_board |
| 273 |
| 274 if __name__ == "__main__": |
| 275 BuildPackages() |
OLD | NEW |