OLD | NEW |
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 # Copyright 2013 The Chromium Authors. All rights reserved. | 2 # Copyright 2013 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Downloads, builds (with instrumentation) and installs shared libraries.""" | 6 """Downloads, builds (with instrumentation) and installs shared libraries.""" |
7 | 7 |
8 import argparse | 8 import argparse |
9 import ast | 9 import ast |
| 10 import fcntl |
10 import os | 11 import os |
11 import platform | 12 import platform |
12 import re | 13 import re |
13 import shlex | 14 import shlex |
14 import shutil | 15 import shutil |
15 import subprocess | 16 import subprocess |
16 import sys | 17 import sys |
17 | 18 |
18 SCRIPT_ABSOLUTE_PATH = os.path.dirname(os.path.abspath(__file__)) | 19 SCRIPT_ABSOLUTE_PATH = os.path.dirname(os.path.abspath(__file__)) |
19 | 20 |
(...skipping 22 matching lines...) Expand all Loading... |
42 return os.path.realpath(os.path.join(SCRIPT_ABSOLUTE_PATH, '..', | 43 return os.path.realpath(os.path.join(SCRIPT_ABSOLUTE_PATH, '..', |
43 path_relative_to_gyp)) | 44 path_relative_to_gyp)) |
44 | 45 |
45 | 46 |
46 class InstrumentedPackageBuilder(object): | 47 class InstrumentedPackageBuilder(object): |
47 """Checks out and builds a single instrumented package.""" | 48 """Checks out and builds a single instrumented package.""" |
48 def __init__(self, args, clobber): | 49 def __init__(self, args, clobber): |
49 self._cc = args.cc | 50 self._cc = args.cc |
50 self._cxx = args.cxx | 51 self._cxx = args.cxx |
51 self._extra_configure_flags = unescape_flags(args.extra_configure_flags) | 52 self._extra_configure_flags = unescape_flags(args.extra_configure_flags) |
52 self._jobs = args.jobs | |
53 self._libdir = args.libdir | 53 self._libdir = args.libdir |
54 self._package = args.package | 54 self._package = args.package |
55 self._patch = real_path(args.patch) if args.patch else None | 55 self._patch = real_path(args.patch) if args.patch else None |
56 self._pre_build = \ | 56 self._pre_build = \ |
57 real_path(args.pre_build) if args.pre_build else None | 57 real_path(args.pre_build) if args.pre_build else None |
58 self._sanitizer = args.sanitizer | 58 self._sanitizer = args.sanitizer |
59 self._verbose = args.verbose | 59 self._verbose = args.verbose |
60 self._clobber = clobber | 60 self._clobber = clobber |
61 self._working_dir = os.path.join( | 61 self._working_dir = os.path.join( |
62 real_path(args.intermediate_dir), self._package, '') | 62 real_path(args.intermediate_dir), self._package, '') |
(...skipping 28 matching lines...) Expand all Loading... |
91 self._build_env['LDFLAGS'] = self._ldflags | 91 self._build_env['LDFLAGS'] = self._ldflags |
92 | 92 |
93 if self._sanitizer == 'asan': | 93 if self._sanitizer == 'asan': |
94 # Do not report leaks during the build process. | 94 # Do not report leaks during the build process. |
95 self._build_env['ASAN_OPTIONS'] = \ | 95 self._build_env['ASAN_OPTIONS'] = \ |
96 '%s:detect_leaks=0' % self._build_env.get('ASAN_OPTIONS', '') | 96 '%s:detect_leaks=0' % self._build_env.get('ASAN_OPTIONS', '') |
97 | 97 |
98 # libappindicator1 needs this. | 98 # libappindicator1 needs this. |
99 self._build_env['CSC'] = '/usr/bin/mono-csc' | 99 self._build_env['CSC'] = '/usr/bin/mono-csc' |
100 | 100 |
101 def shell_call(self, command, env=None, cwd=None): | 101 def shell_call(self, command, env=None, cwd=None, ignore_ret_code=False): |
102 """Wrapper around subprocess.Popen(). | 102 """Wrapper around subprocess.Popen(). |
103 | 103 |
104 Calls command with specific environment and verbosity using | 104 Calls command with specific environment and verbosity using |
105 subprocess.Popen(). | 105 subprocess.Popen(). |
106 """ | 106 """ |
107 child = subprocess.Popen( | 107 child = subprocess.Popen( |
108 command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, | 108 command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, |
109 env=env, shell=True, cwd=cwd) | 109 env=env, shell=True, cwd=cwd) |
110 stdout, stderr = child.communicate() | 110 stdout, stderr = child.communicate() |
| 111 if ignore_ret_code: |
| 112 if self._verbose: |
| 113 print stdout |
| 114 return |
111 if self._verbose or child.returncode: | 115 if self._verbose or child.returncode: |
112 print stdout | 116 print stdout |
113 if child.returncode: | 117 if child.returncode: |
114 raise Exception('Failed to run: %s' % command) | 118 raise Exception('Failed to run: %s' % command) |
115 | 119 |
116 def maybe_download_source(self): | 120 def maybe_download_source(self): |
117 """Checks out the source code (if needed). | 121 """Checks out the source code (if needed). |
118 | 122 |
119 Checks out the source code for the package, if required (i.e. unless running | 123 Checks out the source code for the package, if required (i.e. unless running |
120 in no-clobber mode). Initializes self._source_dir and self._source_archives. | 124 in no-clobber mode). Initializes self._source_dir and self._source_archives. |
121 """ | 125 """ |
122 get_fresh_source = self._clobber or not os.path.exists(self._working_dir) | 126 get_fresh_source = self._clobber or not os.path.exists(self._working_dir) |
123 if get_fresh_source: | 127 if get_fresh_source: |
124 shutil.rmtree(self._working_dir, ignore_errors=True) | 128 shutil.rmtree(self._working_dir, ignore_errors=True) |
125 os.makedirs(self._working_dir) | 129 os.makedirs(self._working_dir) |
| 130 |
| 131 # Download one source package at a time, otherwise, there will |
| 132 # be connection errors in gnutls_handshake(). |
| 133 lock = open('apt-source-lock', 'w') |
| 134 fcntl.flock(lock, fcntl.LOCK_EX) |
126 self.shell_call('apt-get source %s' % self._package, | 135 self.shell_call('apt-get source %s' % self._package, |
127 cwd=self._working_dir) | 136 cwd=self._working_dir) |
| 137 fcntl.flock(lock, fcntl.LOCK_UN) |
128 | 138 |
129 (dirpath, dirnames, filenames) = os.walk(self._working_dir).next() | 139 (dirpath, dirnames, filenames) = os.walk(self._working_dir).next() |
130 | 140 |
131 if len(dirnames) != 1: | 141 if len(dirnames) != 1: |
132 raise Exception( | 142 raise Exception( |
133 '`apt-get source %s\' must create exactly one subdirectory.' | 143 '`apt-get source %s\' must create exactly one subdirectory.' |
134 % self._package) | 144 % self._package) |
135 self._source_dir = os.path.join(dirpath, dirnames[0], '') | 145 self._source_dir = os.path.join(dirpath, dirnames[0], '') |
136 | 146 |
137 if len(filenames) == 0: | 147 if len(filenames) == 0: |
(...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
208 | 218 |
209 def cleanup_after_install(self): | 219 def cleanup_after_install(self): |
210 """Removes unneeded files in self.temp_libdir().""" | 220 """Removes unneeded files in self.temp_libdir().""" |
211 # .la files are not needed, nuke them. | 221 # .la files are not needed, nuke them. |
212 # In case --no-static is not supported, nuke any static libraries we built. | 222 # In case --no-static is not supported, nuke any static libraries we built. |
213 self.shell_call( | 223 self.shell_call( |
214 'find %s -name *.la -or -name *.a | xargs rm -f' % self.temp_libdir()) | 224 'find %s -name *.la -or -name *.a | xargs rm -f' % self.temp_libdir()) |
215 # .pc files are not needed. | 225 # .pc files are not needed. |
216 self.shell_call('rm %s/pkgconfig -rf' % self.temp_libdir()) | 226 self.shell_call('rm %s/pkgconfig -rf' % self.temp_libdir()) |
217 | 227 |
218 def make(self, args, jobs=None, env=None, cwd=None): | 228 def make(self, args, env=None, cwd=None, ignore_ret_code=False): |
219 """Invokes `make'. | 229 """Invokes `make'. |
220 | 230 |
221 Invokes `make' with the specified args, using self._build_env and | 231 Invokes `make' with the specified args, using self._build_env and |
222 self._source_dir by default. | 232 self._source_dir by default. |
223 """ | 233 """ |
224 if jobs is None: | |
225 jobs = self._jobs | |
226 if cwd is None: | 234 if cwd is None: |
227 cwd = self._source_dir | 235 cwd = self._source_dir |
228 if env is None: | 236 if env is None: |
229 env = self._build_env | 237 env = self._build_env |
230 cmd = ['make', '-j%s' % jobs] + args | 238 cmd = ['make'] + args |
231 self.shell_call(' '.join(cmd), env=env, cwd=cwd) | 239 self.shell_call(' '.join(cmd), env=env, cwd=cwd, |
| 240 ignore_ret_code=ignore_ret_code) |
232 | 241 |
233 def make_install(self, args, **kwargs): | 242 def make_install(self, args, **kwargs): |
234 """Invokes `make install'.""" | 243 """Invokes `make install'.""" |
235 self.make(['install'] + args, **kwargs) | 244 self.make(['install'] + args, **kwargs) |
236 | 245 |
237 def build_and_install(self): | 246 def build_and_install(self): |
238 """Builds and installs the DSOs. | 247 """Builds and installs the DSOs. |
239 | 248 |
240 Builds the package with ./configure + make, installs it to a temporary | 249 Builds the package with ./configure + make, installs it to a temporary |
241 location, then moves the relevant files to their permanent location. | 250 location, then moves the relevant files to their permanent location. |
242 """ | 251 """ |
243 configure_cmd = './configure --libdir=/%s/ %s' % ( | 252 configure_cmd = './configure --libdir=/%s/ %s' % ( |
244 self._libdir, self._extra_configure_flags) | 253 self._libdir, self._extra_configure_flags) |
245 self.shell_call(configure_cmd, env=self._build_env, cwd=self._source_dir) | 254 self.shell_call(configure_cmd, env=self._build_env, cwd=self._source_dir) |
246 | 255 |
247 # Some makefiles use BUILDROOT or INSTALL_ROOT instead of DESTDIR. | 256 # Some makefiles use BUILDROOT or INSTALL_ROOT instead of DESTDIR. |
248 args = ['DESTDIR', 'BUILDROOT', 'INSTALL_ROOT'] | 257 args = ['DESTDIR', 'BUILDROOT', 'INSTALL_ROOT'] |
249 make_args = ['%s=%s' % (name, self.temp_dir()) for name in args] | 258 make_args = ['%s=%s' % (name, self.temp_dir()) for name in args] |
250 self.make(make_args) | 259 self.make(make_args) |
251 | 260 |
252 # Some packages don't support parallel install. Use -j1 always. | 261 self.make_install(make_args) |
253 self.make_install(make_args, jobs=1) | |
254 | 262 |
255 self.cleanup_after_install() | 263 self.cleanup_after_install() |
256 | 264 |
257 self.fix_rpaths(self.temp_libdir()) | 265 self.fix_rpaths(self.temp_libdir()) |
258 | 266 |
259 # Now move the contents of the temporary destdir to their final place. | 267 # Now move the contents of the temporary destdir to their final place. |
260 # We only care for the contents of LIBDIR. | 268 # We only care for the contents of LIBDIR. |
261 self.shell_call('cp %s/* %s/ -rdf' % (self.temp_libdir(), | 269 self.shell_call('cp %s/* %s/ -rdf' % (self.temp_libdir(), |
262 self.dest_libdir())) | 270 self.dest_libdir())) |
263 | 271 |
(...skipping 100 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
364 make_args.append('USE_64=1') | 372 make_args.append('USE_64=1') |
365 | 373 |
366 # Make sure we don't override the default flags in the makefile. | 374 # Make sure we don't override the default flags in the makefile. |
367 for variable in ['CFLAGS', 'CXXFLAGS', 'LDFLAGS']: | 375 for variable in ['CFLAGS', 'CXXFLAGS', 'LDFLAGS']: |
368 del self._build_env[variable] | 376 del self._build_env[variable] |
369 | 377 |
370 # Hardcoded paths. | 378 # Hardcoded paths. |
371 temp_dir = os.path.join(self._source_dir, 'nss') | 379 temp_dir = os.path.join(self._source_dir, 'nss') |
372 temp_libdir = os.path.join(temp_dir, 'lib') | 380 temp_libdir = os.path.join(temp_dir, 'lib') |
373 | 381 |
374 # Parallel build is not supported. Also, the build happens in | 382 # The build happens in <source_dir>/nss. Building fails after all |
375 # <source_dir>/nss. | 383 # the required DSOs have been built, so ignore the error. |
376 try: | 384 self.make(make_args, cwd=temp_dir, ignore_ret_code=True) |
377 self.make(make_args, jobs=1, cwd=temp_dir) | |
378 except Exception: | |
379 # Building fails after all the required DSOs have been built, so ignore | |
380 # the error. | |
381 pass | |
382 | 385 |
383 self.fix_rpaths(temp_libdir) | 386 self.fix_rpaths(temp_libdir) |
384 | 387 |
385 # 'make install' is not supported. Copy the DSOs manually. | 388 # 'make install' is not supported. Copy the DSOs manually. |
386 for (dirpath, dirnames, filenames) in os.walk(temp_libdir): | 389 for (dirpath, dirnames, filenames) in os.walk(temp_libdir): |
387 for filename in filenames: | 390 for filename in filenames: |
388 if filename.endswith('.so'): | 391 if filename.endswith('.so'): |
389 full_path = os.path.join(dirpath, filename) | 392 full_path = os.path.join(dirpath, filename) |
390 if self._verbose: | 393 if self._verbose: |
391 print 'download_build_install.py: installing %s' % full_path | 394 print 'download_build_install.py: installing %s' % full_path |
392 shutil.copy(full_path, self.dest_libdir()) | 395 shutil.copy(full_path, self.dest_libdir()) |
393 | 396 |
394 | 397 |
395 class StubBuilder(InstrumentedPackageBuilder): | 398 class StubBuilder(InstrumentedPackageBuilder): |
396 def download_build_install(self): | 399 def download_build_install(self): |
397 self._touch(os.path.join(self._destdir, '%s.txt' % self._package)) | 400 self._touch(os.path.join(self._destdir, '%s.txt' % self._package)) |
| 401 self.shell_call('mkdir -p %s' % self.dest_libdir()) |
398 self._touch(os.path.join(self.dest_libdir(), '%s.so.0' % self._package)) | 402 self._touch(os.path.join(self.dest_libdir(), '%s.so.0' % self._package)) |
399 | 403 |
400 def _touch(self, path): | 404 def _touch(self, path): |
401 with open(path, 'w'): | 405 with open(path, 'w'): |
402 pass | 406 pass |
403 | 407 |
404 | 408 |
405 def main(): | 409 def main(): |
406 parser = argparse.ArgumentParser( | 410 parser = argparse.ArgumentParser( |
407 description='Download, build and install an instrumented package.') | 411 description='Download, build and install an instrumented package.') |
408 | 412 |
409 parser.add_argument('-j', '--jobs', type=int, default=1) | |
410 parser.add_argument('-p', '--package', required=True) | 413 parser.add_argument('-p', '--package', required=True) |
411 parser.add_argument( | 414 parser.add_argument( |
412 '-i', '--product-dir', default='.', | 415 '-i', '--product-dir', default='.', |
413 help='Relative path to the directory with chrome binaries') | 416 help='Relative path to the directory with chrome binaries') |
414 parser.add_argument( | 417 parser.add_argument( |
415 '-m', '--intermediate-dir', default='.', | 418 '-m', '--intermediate-dir', default='.', |
416 help='Relative path to the directory for temporary build files') | 419 help='Relative path to the directory for temporary build files') |
417 parser.add_argument('--extra-configure-flags', default='') | 420 parser.add_argument('--extra-configure-flags', default='') |
418 parser.add_argument('--cflags', default='') | 421 parser.add_argument('--cflags', default='') |
419 parser.add_argument('--ldflags', default='') | 422 parser.add_argument('--ldflags', default='') |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
451 builder = Libpci3Builder(args, clobber) | 454 builder = Libpci3Builder(args, clobber) |
452 elif args.build_method == 'stub': | 455 elif args.build_method == 'stub': |
453 builder = StubBuilder(args, clobber) | 456 builder = StubBuilder(args, clobber) |
454 else: | 457 else: |
455 raise Exception('Unrecognized build method: %s' % args.build_method) | 458 raise Exception('Unrecognized build method: %s' % args.build_method) |
456 | 459 |
457 builder.download_build_install() | 460 builder.download_build_install() |
458 | 461 |
459 if __name__ == '__main__': | 462 if __name__ == '__main__': |
460 main() | 463 main() |
OLD | NEW |