Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(266)

Side by Side Diff: chromite/bin/cros_build_packages

Issue 3315020: Prototype cros_build_packages, which builds and extracts build tarballs. (Closed) Base URL: http://git.chromium.org/git/crosutils.git
Patch Set: Add back blank line Created 10 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/python2.6
2 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
5
6 import optparse
7 import os
8 import multiprocessing
9 import sys
10 import tempfile
11 sys.path.insert(0, os.path.abspath(__file__ + "/../../lib"))
12 from cros_build_lib import Die
13 from cros_build_lib import Info
14 from cros_build_lib import RunCommand
15 from cros_build_lib import Warning
16
17
18 def BuildPackages():
19 """Build packages according to options specified on command-line."""
20
21 if os.getuid() != 0:
22 Die("superuser access required")
23
24 scripts_dir = os.path.abspath(__file__ + "/../../..")
25 builder = PackageBuilder(scripts_dir)
26 options, _ = builder.ParseArgs()
27
28 # Calculate packages to install.
29 # TODO(davidjames): Grab these from a spec file.
30 packages = ["chromeos-base/chromeos"]
31 if options.withdev:
32 packages.append("chromeos-base/chromeos-dev")
33 if options.withfactory:
34 packages.append("chromeos-base/chromeos-factoryinstall")
35 if options.withtest:
36 packages.append("chromeos-base/chromeos-test")
37
38 if options.usetarball:
39 builder.ExtractTarball(options, packages)
40 else:
41 builder.BuildTarball(options, packages)
42
43
44 def _Apply(args):
45 """Call the function specified in args[0], with arguments in args[1:]."""
46 return apply(args[0], args[1:])
47
48
49 def _GetLatestPrebuiltPrefix(board):
50 """Get the latest prebuilt prefix for the specified board.
51
52 Args:
53 board: The board you want prebuilts for.
54 Returns:
55 Latest prebuilt prefix.
56 """
57 # TODO(davidjames): Also append profile names here.
58 prefix = "http://commondatastorage.googleapis.com/chromeos-prebuilt/board"
59 tmpfile = tempfile.NamedTemporaryFile()
60 _Run("curl '%s/%s-latest' -o %s" % (prefix, board, tmpfile.name), retries=3)
61 tmpfile.seek(0)
62 latest = tmpfile.read().strip()
63 tmpfile.close()
64 return "%s/%s" % (prefix, latest)
65
66
67 def _GetPrebuiltDownloadCommands(prefix):
68 """Return a list of commands for grabbing packages.
69
70 There must be a file called "packages/Packages" that contains the list of
71 packages. The specified list of commands will fill the packages directory
72 with the bzipped packages from the specified prefix.
73
74 Args:
75 prefix: Url prefix to download packages from.
76 Returns:
77 List of commands for grabbing packages.
78 """
79
80 cmds = []
81 for line in file("packages/Packages"):
82 if line.startswith("CPV: "):
83 pkgpath, pkgname = line.replace("CPV: ", "").strip().split("/")
84 path = "%s/%s.tbz2" % (pkgpath, pkgname)
85 url = "%s/%s" % (prefix, path)
86 dirname = "packages/%s" % pkgpath
87 fullpath = "packages/%s" % path
88 if not os.path.exists(dirname):
89 os.makedirs(dirname)
90 if not os.path.exists(fullpath):
91 cmds.append("curl -s %s -o %s" % (url, fullpath))
92 return cmds
93
94
95 def _Run(cmd, retries=0):
96 """Run the specified command.
97
98 If the command fails, and the retries have been exhausted, the program exits
99 with an appropriate error message.
100
101 Args:
102 cmd: The command to run.
103 retries: If exit code is non-zero, retry this many times.
104 """
105 # TODO(davidjames): Move this to common library.
106 for _ in range(retries+1):
107 result = RunCommand(cmd, shell=True, exit_code=True, error_ok=True)
108 if result.returncode == 0:
109 Info("Command succeeded: %s" % cmd)
110 break
111 Warning("Command failed: %s" % cmd)
112 else:
113 Die("Command failed, exiting: %s" % cmd)
114
115
116 def _RunManyParallel(cmds, retries=0):
117 """Run list of provided commands in parallel.
118
119 To work around a bug in the multiprocessing module, we use map_async instead
120 of the usual map function. See http://bugs.python.org/issue9205
121
122 Args:
123 cmds: List of commands to run.
124 retries: Number of retries per command.
125 """
126 # TODO(davidjames): Move this to common library.
127 pool = multiprocessing.Pool()
128 args = []
129 for cmd in cmds:
130 args.append((_Run, cmd, retries))
131 result = pool.map_async(_Apply, args, chunksize=1)
132 while True:
133 try:
134 result.get(60*60)
135 break
136 except multiprocessing.TimeoutError:
137 pass
138
139
140 class PackageBuilder(object):
141 """A class for building and extracting tarballs of Chromium OS packages."""
142
143 def __init__(self, scripts_dir):
144 self.scripts_dir = scripts_dir
145
146 def BuildTarball(self, options, packages):
147 """Build a tarball with the specified packages.
148
149 Args:
150 options: Options object, as output by ParseArgs.
151 packages: List of packages to build.
152 """
153
154 board = options.board
155
156 # Run setup_board. TODO(davidjames): Integrate the logic used in
157 # setup_board into chromite.
158 _Run("%s/setup_board --force --board=%s" % (self.scripts_dir, board))
159
160 # Create complete build directory
161 _Run(self._EmergeBoardCmd(options, packages))
162
163 # Archive build directory as tarballs
164 os.chdir("/build/%s" % board)
165 cmds = [
166 "tar -c --wildcards --exclude='usr/lib/debug/*' "
167 "--exclude='packages/*' * | pigz -c > packages/%s-build.tgz" % board,
168 "tar -c usr/lib/debug/* | pigz -c > packages/%s-debug.tgz" % board
169 ]
170
171 # Run list of commands.
172 _RunManyParallel(cmds)
173
174 def ExtractTarball(self, options, packages):
175 """Extract the latest build tarball, then update the specified packages.
176
177 Args:
178 options: Options object, as output by ParseArgs.
179 packages: List of packages to update.
180 """
181
182 board = options.board
183 prefix = _GetLatestPrebuiltPrefix(board)
184
185 # If the user doesn't have emerge-${BOARD} setup yet, we need to run
186 # setup_board. TODO(davidjames): Integrate the logic used in setup_board
187 # into chromite.
188 if not os.path.exists("/usr/local/bin/emerge-%s" % board):
189 _Run("%s/setup_board --force --board=%s" % (self.scripts_dir, board))
190
191 # Delete old build directory. This process might take a while, so do it in
192 # the background.
193 cmds = []
194 if os.path.exists("/build/%s" % board):
195 tempdir = tempfile.mkdtemp()
196 _Run("mv /build/%s %s" % (board, tempdir))
197 cmds.append("rm -rf %s" % tempdir)
198
199 # Create empty build directory, and chdir into it.
200 os.makedirs("/build/%s/packages" % board)
201 os.chdir("/build/%s" % board)
202
203 # Download and expand build tarball.
204 build_url = "%s/%s-build.tgz" % (prefix, board)
205 cmds.append("curl -s %s | tar -xz" % build_url)
206
207 # Download and expand debug tarball (if requested).
208 if options.debug:
209 debug_url = "%s/%s-debug.tgz" % (prefix, board)
210 cmds.append("curl -s %s | tar -xz" % debug_url)
211
212 # Download prebuilt packages.
213 _Run("curl '%s/Packages' -o packages/Packages" % prefix, retries=3)
214 cmds.extend(_GetPrebuiltDownloadCommands(prefix))
215
216 # Run list of commands, with three retries per command, in case the network
217 # is flaky.
218 _RunManyParallel(cmds, retries=3)
219
220 # Emerge remaining packages.
221 _Run(self._EmergeBoardCmd(options, packages))
222
223 def ParseArgs(self):
224 """Parse arguments from the command line using optparse."""
225
226 # TODO(davidjames): We should use spec files for this.
227 default_board = self._GetDefaultBoard()
228 parser = optparse.OptionParser()
229 parser.add_option("--board", dest="board", default=default_board,
230 help="The board to build packages for.")
231 parser.add_option("--debug", action="store_true", dest="debug",
232 default=False, help="Include debug symbols.")
233 parser.add_option("--nowithdev", action="store_false", dest="withdev",
234 default=True,
235 help="Don't build useful developer friendly utilities.")
236 parser.add_option("--nowithtest", action="store_false", dest="withtest",
237 default=True, help="Build packages required for testing.")
238 parser.add_option("--nowithfactory", action="store_false",
239 dest="withfactory", default=True,
240 help="Build factory installer")
241 parser.add_option("--nousepkg", action="store_false",
242 dest="usepkg", default=True,
243 help="Don't use binary packages.")
244 parser.add_option("--nousetarball", action="store_false",
245 dest="usetarball", default=True,
246 help="Don't use tarball.")
247 parser.add_option("--nofast", action="store_false", dest="fast",
248 default=True,
249 help="Don't merge packages in parallel.")
250
251 return parser.parse_args()
252
253 def _EmergeBoardCmd(self, options, packages):
254 """Calculate board emerge command."""
255 board = options.board
256 scripts_dir = self.scripts_dir
257 emerge_board = "emerge-%s" % board
258 if options.fast:
259 emerge_board = "%s/parallel_emerge --board=%s" % (scripts_dir, board)
260 usepkg = ""
261 if options.usepkg:
262 usepkg = "g"
263 return "%s -uDNv%s %s" % (emerge_board, usepkg, " ".join(packages))
264
265 def _GetDefaultBoard(self):
266 """Get the default board configured by the user."""
267
268 default_board_file = "%s/.default_board" % self.scripts_dir
269 default_board = None
270 if os.path.exists(default_board_file):
271 default_board = file(default_board_file).read().strip()
272 return default_board
273
274 if __name__ == "__main__":
275 BuildPackages()
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698