| OLD | NEW |
| 1 #!/usr/bin/python | 1 #!/usr/bin/python |
| 2 # Copyright 2013 The Chromium Authors. All rights reserved. | 2 # Copyright 2013 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """Downloads, builds (with instrumentation) and installs shared libraries.""" | 6 """Downloads, builds (with instrumentation) and installs shared libraries.""" |
| 7 | 7 |
| 8 import argparse | 8 import argparse |
| 9 import os | 9 import os |
| 10 import platform | 10 import platform |
| 11 import re | 11 import re |
| 12 import shlex | 12 import shlex |
| 13 import shutil | 13 import shutil |
| 14 import subprocess | 14 import subprocess |
| 15 import sys | 15 import sys |
| 16 | 16 |
| 17 SCRIPT_ABSOLUTE_PATH = os.path.dirname(os.path.abspath(__file__)) | 17 SCRIPT_ABSOLUTE_PATH = os.path.dirname(os.path.abspath(__file__)) |
| 18 | 18 |
| 19 class ScopedChangeDirectory(object): | 19 def unescape_flags(s): |
| 20 """Changes current working directory and restores it back automatically.""" | 20 """Un-escapes build flags received from GYP. |
| 21 | 21 |
| 22 def __init__(self, path): | 22 GYP escapes build flags as if they are to be inserted directly into a command |
| 23 self.path = path | 23 line, wrapping each flag in double quotes. When flags are passed via |
| 24 self.old_path = '' | 24 CFLAGS/LDFLAGS instead, double quotes must be dropped. |
| 25 | |
| 26 def __enter__(self): | |
| 27 self.old_path = os.getcwd() | |
| 28 os.chdir(self.path) | |
| 29 return self | |
| 30 | |
| 31 def __exit__(self, exc_type, exc_value, traceback): | |
| 32 os.chdir(self.old_path) | |
| 33 | |
| 34 def get_package_build_dependencies(package): | |
| 35 command = 'apt-get -s build-dep %s | grep Inst | cut -d " " -f 2' % package | |
| 36 command_result = subprocess.Popen(command, stdout=subprocess.PIPE, | |
| 37 shell=True) | |
| 38 if command_result.wait(): | |
| 39 raise Exception('Failed to determine build dependencies for %s' % package) | |
| 40 build_dependencies = [l.strip() for l in command_result.stdout] | |
| 41 return build_dependencies | |
| 42 | |
| 43 | |
| 44 def check_package_build_dependencies(package): | |
| 45 build_dependencies = get_package_build_dependencies(package) | |
| 46 if len(build_dependencies): | |
| 47 print >> sys.stderr, 'Please, install build-dependencies for %s' % package | |
| 48 print >> sys.stderr, 'One-liner for APT:' | |
| 49 print >> sys.stderr, 'sudo apt-get -y --no-remove build-dep %s' % package | |
| 50 sys.exit(1) | |
| 51 | |
| 52 | |
| 53 def shell_call(command, verbose=False, environment=None): | |
| 54 """ Wrapper on subprocess.Popen | |
| 55 | |
| 56 Calls command with specific environment and verbosity using | |
| 57 subprocess.Popen | |
| 58 | |
| 59 Args: | |
| 60 command: Command to run in shell. | |
| 61 verbose: If False, hides all stdout and stderr in case of successful build. | |
| 62 Otherwise, always prints stdout and stderr. | |
| 63 environment: Parameter 'env' for subprocess.Popen. | |
| 64 | |
| 65 Returns: | |
| 66 None | |
| 67 | |
| 68 Raises: | |
| 69 Exception: if return code after call is not zero. | |
| 70 """ | 25 """ |
| 71 child = subprocess.Popen( | 26 return ' '.join(shlex.split(s)) |
| 72 command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, | 27 |
| 73 env=environment, shell=True) | 28 |
| 74 stdout, stderr = child.communicate() | 29 def real_path(path_relative_to_script): |
| 75 if verbose or child.returncode: | 30 """Returns the absolute path to a file. |
| 76 print stdout | 31 |
| 77 if child.returncode: | 32 GYP generates paths relative to the location of the .gyp file, which coincides |
| 78 raise Exception('Failed to run: %s' % command) | 33 with the location of this script. This function converts them to absolute |
| 79 | 34 paths. |
| 80 | 35 """ |
| 81 def run_shell_commands(commands, verbose=False, environment=None): | 36 return os.path.realpath(os.path.join(SCRIPT_ABSOLUTE_PATH, |
| 82 for command in commands: | 37 path_relative_to_script)) |
| 83 shell_call(command, verbose, environment) | 38 |
| 84 | 39 |
| 85 | 40 class InstrumentedPackageBuilder(object): |
| 86 def fix_rpaths(destdir): | 41 """Checks out and builds a single instrumented package.""" |
| 87 # TODO(earthdok): reimplement fix_rpaths.sh in Python. | 42 def __init__(self, args, clobber): |
| 88 shell_call("%s/fix_rpaths.sh %s/lib" % (SCRIPT_ABSOLUTE_PATH, destdir)) | 43 self._cc = args.cc |
| 89 | 44 self._cxx = args.cxx |
| 90 | 45 self._extra_configure_flags = args.extra_configure_flags |
| 91 def destdir_configure_make_install(parsed_arguments, environment, | 46 self._jobs = args.jobs |
| 92 install_prefix): | 47 self._libdir = args.libdir |
| 93 configure_command = './configure %s' % parsed_arguments.extra_configure_flags | 48 self._package = args.package |
| 94 configure_command += ' --libdir=/lib/' | 49 self._patch = real_path(args.patch) if args.patch else None |
| 95 # Installing to a temporary directory allows us to safely clean up the .la | 50 self._run_before_build = \ |
| 96 # files below. | 51 real_path(args.run_before_build) if args.run_before_build else None |
| 97 destdir = '%s/debian/instrumented_build' % os.getcwd() | 52 self._sanitizer = args.sanitizer |
| 98 # Some makefiles use BUILDROOT or INSTALL_ROOT instead of DESTDIR. | 53 self._verbose = args.verbose |
| 99 make_command = 'make DESTDIR=%s BUILDROOT=%s INSTALL_ROOT=%s' % (destdir, | 54 self._clobber = clobber |
| 100 destdir, | 55 self._working_dir = os.path.join( |
| 101 destdir) | 56 real_path(args.intermediate_dir), self._package, '') |
| 102 build_and_install_in_destdir = [ | 57 |
| 103 configure_command, | 58 product_dir = real_path(args.product_dir) |
| 104 '%s -j%s' % (make_command, parsed_arguments.jobs), | 59 self._destdir = os.path.join( |
| 105 # Parallel install is flaky for some packages. | 60 product_dir, 'instrumented_libraries', self._sanitizer) |
| 106 '%s install -j1' % make_command, | 61 |
| 107 # Kill the .la files. They contain absolute paths, and will cause build | 62 self._cflags = unescape_flags(args.cflags) |
| 108 # errors in dependent libraries. | 63 if args.sanitizer_blacklist: |
| 109 'rm %s/lib/*.la -f' % destdir | 64 blacklist_file = real_path(args.sanitizer_blacklist) |
| 110 ] | 65 self._cflags += ' -fsanitize-blacklist=%s' % blacklist_file |
| 111 run_shell_commands(build_and_install_in_destdir, | 66 |
| 112 parsed_arguments.verbose, environment) | 67 self._ldflags = unescape_flags(args.ldflags) |
| 113 fix_rpaths(destdir) | 68 |
| 114 run_shell_commands([ | 69 self.init_build_env() |
| 115 # Now move the contents of the temporary destdir to their final place. | 70 |
| 116 # We only care for the contents of lib/. | 71 # Initialized later. |
| 117 'mkdir -p %s/lib' % install_prefix, | 72 self._source_dir = None |
| 118 'cp %s/lib/* %s/lib/ -rdf' % (destdir, install_prefix)], | 73 |
| 119 parsed_arguments.verbose, environment) | 74 def init_build_env(self): |
| 120 | 75 self._build_env = os.environ.copy() |
| 121 | 76 |
| 122 def nss_make_and_copy(parsed_arguments, environment, install_prefix): | 77 self._build_env['CC'] = self._cc |
| 123 # NSS uses a build system that's different from configure/make/install. All | 78 self._build_env['CXX'] = self._cxx |
| 124 # flags must be passed as arguments to make. | 79 |
| 125 make_args = [] | 80 self._build_env['CFLAGS'] = self._cflags |
| 126 # Do an optimized build. | 81 self._build_env['CXXFLAGS'] = self._cflags |
| 127 make_args.append('BUILD_OPT=1') | 82 self._build_env['LDFLAGS'] = self._ldflags |
| 128 # Set USE_64=1 on x86_64 systems. | 83 |
| 129 if platform.architecture()[0] == '64bit': | 84 if self._sanitizer == 'asan': |
| 130 make_args.append('USE_64=1') | 85 # Do not report leaks during the build process. |
| 131 # Passing C(XX)FLAGS overrides the defaults, and EXTRA_C(XX)FLAGS is not | 86 self._build_env['ASAN_OPTIONS'] = \ |
| 132 # supported. Append our extra flags to CC/CXX. | 87 '%s:detect_leaks=0' % self._build_env.get('ASAN_OPTIONS', '') |
| 133 make_args.append('CC="%s %s"' % (environment['CC'], environment['CFLAGS'])) | 88 |
| 134 make_args.append('CXX="%s %s"' % | 89 # libappindicator1 needs this. |
| 135 (environment['CXX'], environment['CXXFLAGS'])) | 90 self._build_env['CSC'] = '/usr/bin/mono-csc' |
| 136 # We need to override ZDEFS_FLAG at least to prevent -Wl,-z,defs. | 91 |
| 137 # Might as well use this to pass the linker flags, since ZDEF_FLAG is always | 92 def shell_call(self, command, env=None, cwd=None): |
| 138 # added during linking on Linux. | 93 """Wrapper around subprocess.Popen(). |
| 139 make_args.append('ZDEFS_FLAG="-Wl,-z,nodefs %s"' % environment['LDFLAGS']) | 94 |
| 140 make_args.append('NSPR_INCLUDE_DIR=/usr/include/nspr') | 95 Calls command with specific environment and verbosity using |
| 141 make_args.append('NSPR_LIB_DIR=%s/lib' % install_prefix) | 96 subprocess.Popen(). |
| 142 make_args.append('NSS_ENABLE_ECC=1') | 97 """ |
| 143 # Make sure we don't override the default flags. | 98 child = subprocess.Popen( |
| 144 for variable in ['CFLAGS', 'CXXFLAGS', 'LDFLAGS']: | 99 command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, |
| 145 del environment[variable] | 100 env=env, shell=True, cwd=cwd) |
| 146 with ScopedChangeDirectory('nss') as cd_nss: | 101 stdout, stderr = child.communicate() |
| 147 # -j is not supported | 102 if self._verbose or child.returncode: |
| 148 shell_call('make %s' % ' '.join(make_args), parsed_arguments.verbose, | 103 print stdout |
| 149 environment) | 104 if child.returncode: |
| 150 fix_rpaths(os.getcwd()) | 105 raise Exception('Failed to run: %s' % command) |
| 106 |
| 107 def maybe_download_source(self): |
| 108 """Checks out the source code (if needed) and sets self._source_dir.""" |
| 109 get_fresh_source = self._clobber or not os.path.exists(self._working_dir) |
| 110 if get_fresh_source: |
| 111 self.shell_call('rm -rf %s' % self._working_dir) |
| 112 os.makedirs(self._working_dir) |
| 113 self.shell_call('apt-get source %s' % self._package, |
| 114 cwd=self._working_dir) |
| 115 |
| 116 (dirpath, dirnames, filenames) = os.walk(self._working_dir).next() |
| 117 if len(dirnames) != 1: |
| 118 raise Exception('apt-get source %s must create exactly one subdirectory.' |
| 119 % self._package) |
| 120 self._source_dir = os.path.join(dirpath, dirnames[0], '') |
| 121 |
| 122 return get_fresh_source |
| 123 |
| 124 def patch_source(self): |
| 125 if self._patch: |
| 126 self.shell_call('patch -p1 -i %s' % self._patch, cwd=self._source_dir) |
| 127 if self._run_before_build: |
| 128 self.shell_call(self._run_before_build, cwd=self._source_dir) |
| 129 |
| 130 def download_build_install(self): |
| 131 got_fresh_source = self.maybe_download_source() |
| 132 if got_fresh_source: |
| 133 self.patch_source() |
| 134 |
| 135 self.shell_call('mkdir -p %s' % self.dest_libdir()) |
| 136 |
| 137 try: |
| 138 self.build_and_install() |
| 139 except Exception as exception: |
| 140 print 'ERROR: Failed to build package %s. Have you run ' \ |
| 141 'src/third_party/instrumented_libraries/install-build-deps.sh?' % \ |
| 142 self._package |
| 143 print |
| 144 raise |
| 145 |
| 146 # Touch a text file to indicate package is installed. |
| 147 stamp_file = os.path.join(self._destdir, '%s.txt' % self._package) |
| 148 open(stamp_file, 'w').close() |
| 149 |
| 150 # Remove downloaded package and generated temporary build files. Failed |
| 151 # builds intentionally skip this step to help debug build failures. |
| 152 if self._clobber: |
| 153 self.shell_call('rm -rf %s' % self._working_dir) |
| 154 |
| 155 def fix_rpaths(self, directory): |
| 156 # TODO(earthdok): reimplement fix_rpaths.sh in Python. |
| 157 script = real_path('fix_rpaths.sh') |
| 158 self.shell_call("%s %s" % (script, directory)) |
| 159 |
| 160 def temp_dir(self): |
| 161 """Returns the directory which will be passed to `make install'.""" |
| 162 return os.path.join(self._source_dir, 'debian', 'instrumented_build') |
| 163 |
| 164 def temp_libdir(self): |
| 165 """Returns the directory under temp_dir() containing the DSOs.""" |
| 166 return os.path.join(self.temp_dir(), self._libdir) |
| 167 |
| 168 def dest_libdir(self): |
| 169 """Returns the final location of the DSOs.""" |
| 170 return os.path.join(self._destdir, self._libdir) |
| 171 |
| 172 def make(self, args, jobs=None, env=None, cwd=None): |
| 173 """Invokes `make'. |
| 174 |
| 175 Invokes `make' with the specified args, using self._build_env and |
| 176 self._source_dir by default. |
| 177 """ |
| 178 if jobs is None: |
| 179 jobs = self._jobs |
| 180 if cwd is None: |
| 181 cwd = self._source_dir |
| 182 if env is None: |
| 183 env = self._build_env |
| 184 cmd = ['make', '-j%s' % jobs] + args |
| 185 self.shell_call(' '.join(cmd), env=env, cwd=cwd) |
| 186 |
| 187 def make_install(self, args, **kwargs): |
| 188 """Invokes `make install'.""" |
| 189 self.make(['install'] + args, **kwargs) |
| 190 |
| 191 def build_and_install(self): |
| 192 """Builds and installs the DSOs. |
| 193 |
| 194 Builds the package with ./configure + make, installs it to a temporary |
| 195 location, then moves the relevant files to their permanent location. |
| 196 """ |
| 197 configure_cmd = './configure --libdir=/%s/ %s' % ( |
| 198 self._libdir, self._extra_configure_flags) |
| 199 self.shell_call(configure_cmd, env=self._build_env, cwd=self._source_dir) |
| 200 |
| 201 # Some makefiles use BUILDROOT or INSTALL_ROOT instead of DESTDIR. |
| 202 args = ['DESTDIR', 'BUILDROOT', 'INSTALL_ROOT'] |
| 203 make_args = ['%s=%s' % (name, self.temp_dir()) for name in args] |
| 204 self.make(make_args) |
| 205 |
| 206 # Some packages don't support parallel install. Use -j1 always. |
| 207 self.make_install(make_args, jobs=1) |
| 208 |
| 209 # .la files are not needed, nuke them. |
| 210 self.shell_call('rm %s/*.la -f' % self.temp_libdir()) |
| 211 |
| 212 self.fix_rpaths(self.temp_libdir()) |
| 213 |
| 214 # Now move the contents of the temporary destdir to their final place. |
| 215 # We only care for the contents of LIBDIR. |
| 216 self.shell_call('cp %s/* %s/ -rdf' % (self.temp_libdir(), |
| 217 self.dest_libdir())) |
| 218 |
| 219 |
| 220 class LibcapBuilder(InstrumentedPackageBuilder): |
| 221 def build_and_install(self): |
| 222 # libcap2 doesn't have a configure script |
| 223 build_args = ['CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS'] |
| 224 make_args = [ |
| 225 '%s="%s"' % (name, self._build_env[name]) for name in build_args |
| 226 ] |
| 227 self.make(make_args) |
| 228 |
| 229 install_args = [ |
| 230 'DESTDIR=%s' % self.temp_dir(), |
| 231 'lib=%s' % self._libdir, |
| 232 # Skip a step that requires sudo. |
| 233 'RAISE_SETFCAP=no' |
| 234 ] |
| 235 self.make_install(install_args) |
| 236 |
| 237 self.fix_rpaths(self.temp_libdir()) |
| 238 |
| 239 # Now move the contents of the temporary destdir to their final place. |
| 240 # We only care for the contents of LIBDIR. |
| 241 self.shell_call('cp %s/* %s/ -rdf' % (self.temp_libdir(), |
| 242 self.dest_libdir())) |
| 243 |
| 244 |
| 245 class Libpci3Builder(InstrumentedPackageBuilder): |
| 246 def package_version(self): |
| 247 """Guesses libpci3 version from source directory name.""" |
| 248 dir_name = os.path.split(os.path.normpath(self._source_dir))[-1] |
| 249 match = re.match('pciutils-(\d+\.\d+\.\d+)', dir_name) |
| 250 if match is None: |
| 251 raise Exception( |
| 252 'Unable to guess libpci3 version from directory name: %s' % dir_name) |
| 253 return match.group(1) |
| 254 |
| 255 def temp_libdir(self): |
| 256 # DSOs have to be picked up from <source_dir>/lib, since `make install' |
| 257 # doesn't actualy install them anywhere. |
| 258 return os.path.join(self._source_dir, 'lib') |
| 259 |
| 260 def build_and_install(self): |
| 261 # pciutils doesn't have a configure script |
| 262 # This build process follows debian/rules. |
| 263 self.shell_call('mkdir -p %s-udeb/usr/bin' % self.temp_dir()) |
| 264 |
| 265 build_args = ['CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS'] |
| 266 make_args = [ |
| 267 '%s="%s"' % (name, self._build_env[name]) for name in build_args |
| 268 ] |
| 269 make_args += [ |
| 270 'LIBDIR=/%s/' % self._libdir, |
| 271 'PREFIX=/usr', |
| 272 'SBINDIR=/usr/bin', |
| 273 'IDSDIR=/usr/share/misc', |
| 274 'SHARED=yes', |
| 275 # pciutils-3.2.1 (Trusty) fails to build due to unresolved libkmod |
| 276 # symbols. The binary package has no dependencies on libkmod, so it |
| 277 # looks like it was actually built without libkmod support. |
| 278 'LIBKMOD=no', |
| 279 ] |
| 280 self.make(make_args) |
| 281 |
| 282 # `make install' is not needed. |
| 283 self.fix_rpaths(self.temp_libdir()) |
| 284 |
| 285 # Now install the DSOs to their final place. |
| 286 self.shell_call( |
| 287 'install -m 644 %s/libpci.so* %s' % (self.temp_libdir(), |
| 288 self.dest_libdir())) |
| 289 self.shell_call( |
| 290 'ln -sf libpci.so.%s %s/libpci.so.3' % (self.package_version(), |
| 291 self.dest_libdir())) |
| 292 |
| 293 |
| 294 class NSSBuilder(InstrumentedPackageBuilder): |
| 295 def build_and_install(self): |
| 296 # NSS uses a build system that's different from configure/make/install. All |
| 297 # flags must be passed as arguments to make. |
| 298 make_args = [ |
| 299 # Do an optimized build. |
| 300 'BUILD_OPT=1', |
| 301 # CFLAGS/CXXFLAGS should not be used, as doing so overrides the flags in |
| 302 # the makefile completely. The only way to append our flags is to tack |
| 303 # them onto CC/CXX. |
| 304 'CC="%s %s"' % (self._build_env['CC'], self._build_env['CFLAGS']), |
| 305 'CXX="%s %s"' % (self._build_env['CXX'], self._build_env['CXXFLAGS']), |
| 306 # We need to override ZDEFS_FLAG at least to avoid -Wl,-z,defs, which |
| 307 # is not compatible with sanitizers. We also need some way to pass |
| 308 # LDFLAGS without overriding the defaults. Conveniently, ZDEF_FLAG is |
| 309 # always appended to link flags when building NSS on Linux, so we can |
| 310 # just add our LDFLAGS here. |
| 311 'ZDEFS_FLAG="-Wl,-z,nodefs %s"' % self._build_env['LDFLAGS'], |
| 312 'NSPR_INCLUDE_DIR=/usr/include/nspr', |
| 313 'NSPR_LIB_DIR=%s' % self.dest_libdir(), |
| 314 'NSS_ENABLE_ECC=1' |
| 315 ] |
| 316 if platform.architecture()[0] == '64bit': |
| 317 make_args.append('USE_64=1') |
| 318 |
| 319 # Make sure we don't override the default flags in the makefile. |
| 320 for variable in ['CFLAGS', 'CXXFLAGS', 'LDFLAGS']: |
| 321 del self._build_env[variable] |
| 322 |
| 323 # Hardcoded paths. |
| 324 temp_dir = os.path.join(self._source_dir, 'nss') |
| 325 temp_libdir = os.path.join(temp_dir, 'lib') |
| 326 |
| 327 # Parallel build is not supported. Also, the build happens in |
| 328 # <source_dir>/nss. |
| 329 self.make(make_args, jobs=1, cwd=temp_dir) |
| 330 |
| 331 self.fix_rpaths(temp_libdir) |
| 332 |
| 151 # 'make install' is not supported. Copy the DSOs manually. | 333 # 'make install' is not supported. Copy the DSOs manually. |
| 152 install_dir = '%s/lib/' % install_prefix | 334 for (dirpath, dirnames, filenames) in os.walk(temp_libdir): |
| 153 for (dirpath, dirnames, filenames) in os.walk('./lib/'): | |
| 154 for filename in filenames: | 335 for filename in filenames: |
| 155 if filename.endswith('.so'): | 336 if filename.endswith('.so'): |
| 156 full_path = os.path.join(dirpath, filename) | 337 full_path = os.path.join(dirpath, filename) |
| 157 if parsed_arguments.verbose: | 338 if self._verbose: |
| 158 print 'download_build_install.py: installing %s' % full_path | 339 print 'download_build_install.py: installing %s' % full_path |
| 159 shutil.copy(full_path, install_dir) | 340 shutil.copy(full_path, self.dest_libdir()) |
| 160 | 341 |
| 161 | |
| 162 def libcap2_make_install(parsed_arguments, environment, install_prefix): | |
| 163 # libcap2 doesn't come with a configure script | |
| 164 make_args = [ | |
| 165 '%s="%s"' % (name, environment[name]) | |
| 166 for name in['CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS']] | |
| 167 shell_call('make -j%s %s' % (parsed_arguments.jobs, ' '.join(make_args)), | |
| 168 parsed_arguments.verbose, environment) | |
| 169 destdir = '%s/debian/instrumented_build' % os.getcwd() | |
| 170 install_args = [ | |
| 171 'DESTDIR=%s' % destdir, | |
| 172 # Do not install in lib64/. | |
| 173 'lib=lib', | |
| 174 # Skip a step that requires sudo. | |
| 175 'RAISE_SETFCAP=no' | |
| 176 ] | |
| 177 shell_call('make -j%s install %s' % | |
| 178 (parsed_arguments.jobs, ' '.join(install_args)), | |
| 179 parsed_arguments.verbose, environment) | |
| 180 fix_rpaths(destdir) | |
| 181 run_shell_commands([ | |
| 182 # Now move the contents of the temporary destdir to their final place. | |
| 183 # We only care for the contents of lib/. | |
| 184 'mkdir -p %s/lib' % install_prefix, | |
| 185 'cp %s/lib/* %s/lib/ -rdf' % (destdir, install_prefix)], | |
| 186 parsed_arguments.verbose, environment) | |
| 187 | |
| 188 | |
| 189 def libpci3_make_install(parsed_arguments, environment, install_prefix): | |
| 190 # pciutils doesn't have a configure script | |
| 191 # This build script follows debian/rules. | |
| 192 | |
| 193 # Find out the package version. We'll use this when creating symlinks. | |
| 194 dir_name = os.path.split(os.getcwd())[-1] | |
| 195 match = re.match('pciutils-(\d+\.\d+\.\d+)', dir_name) | |
| 196 if match is None: | |
| 197 raise Exception( | |
| 198 'Unable to guess libpci3 version from directory name: %s' % dir_name) | |
| 199 version = match.group(1) | |
| 200 | |
| 201 # `make install' will create a "$(DESTDIR)-udeb" directory alongside destdir. | |
| 202 # We don't want that in our product dir, so we use an intermediate directory. | |
| 203 destdir = '%s/debian/pciutils' % os.getcwd() | |
| 204 make_args = [ | |
| 205 '%s="%s"' % (name, environment[name]) | |
| 206 for name in['CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS']] | |
| 207 make_args.append('SHARED=yes') | |
| 208 # pciutils-3.2.1 (Trusty) fails to build due to unresolved libkmod symbols. | |
| 209 # The binary package has no dependencies on libkmod, so it looks like it was | |
| 210 # actually built without libkmod support. | |
| 211 make_args.append('LIBKMOD=no') | |
| 212 paths = [ | |
| 213 'LIBDIR=/lib/', | |
| 214 'PREFIX=/usr', | |
| 215 'SBINDIR=/usr/bin', | |
| 216 'IDSDIR=/usr/share/misc', | |
| 217 ] | |
| 218 install_args = ['DESTDIR=%s' % destdir] | |
| 219 run_shell_commands([ | |
| 220 'mkdir -p %s-udeb/usr/bin' % destdir, | |
| 221 'make -j%s %s' % (parsed_arguments.jobs, ' '.join(make_args + paths)), | |
| 222 'make -j%s %s install' % ( | |
| 223 parsed_arguments.jobs, | |
| 224 ' '.join(install_args + paths))], | |
| 225 parsed_arguments.verbose, environment) | |
| 226 fix_rpaths(destdir) | |
| 227 # Now install the DSOs to their final place. | |
| 228 run_shell_commands([ | |
| 229 'mkdir -p %s/lib' % install_prefix, | |
| 230 'install -m 644 lib/libpci.so* %s/lib/' % install_prefix, | |
| 231 'ln -sf libpci.so.%s %s/lib/libpci.so.3' % (version, install_prefix)], | |
| 232 parsed_arguments.verbose, environment) | |
| 233 | |
| 234 | |
| 235 def build_and_install(parsed_arguments, environment, install_prefix): | |
| 236 if parsed_arguments.build_method == 'destdir': | |
| 237 destdir_configure_make_install( | |
| 238 parsed_arguments, environment, install_prefix) | |
| 239 elif parsed_arguments.build_method == 'custom_nss': | |
| 240 nss_make_and_copy(parsed_arguments, environment, install_prefix) | |
| 241 elif parsed_arguments.build_method == 'custom_libcap': | |
| 242 libcap2_make_install(parsed_arguments, environment, install_prefix) | |
| 243 elif parsed_arguments.build_method == 'custom_libpci3': | |
| 244 libpci3_make_install(parsed_arguments, environment, install_prefix) | |
| 245 else: | |
| 246 raise Exception('Unrecognized build method: %s' % | |
| 247 parsed_arguments.build_method) | |
| 248 | |
| 249 | |
| 250 def unescape_flags(s): | |
| 251 # GYP escapes the build flags as if they are going to be inserted directly | |
| 252 # into the command line. Since we pass them via CFLAGS/LDFLAGS, we must drop | |
| 253 # the double quotes accordingly. | |
| 254 return ' '.join(shlex.split(s)) | |
| 255 | |
| 256 | |
| 257 def build_environment(parsed_arguments, product_directory, install_prefix): | |
| 258 environment = os.environ.copy() | |
| 259 # The CC/CXX environment variables take precedence over the command line | |
| 260 # flags. | |
| 261 if 'CC' not in environment and parsed_arguments.cc: | |
| 262 environment['CC'] = parsed_arguments.cc | |
| 263 if 'CXX' not in environment and parsed_arguments.cxx: | |
| 264 environment['CXX'] = parsed_arguments.cxx | |
| 265 | |
| 266 cflags = unescape_flags(parsed_arguments.cflags) | |
| 267 if parsed_arguments.sanitizer_blacklist: | |
| 268 cflags += ' -fsanitize-blacklist=%s/%s' % ( | |
| 269 SCRIPT_ABSOLUTE_PATH, | |
| 270 parsed_arguments.sanitizer_blacklist) | |
| 271 environment['CFLAGS'] = cflags | |
| 272 environment['CXXFLAGS'] = cflags | |
| 273 | |
| 274 ldflags = unescape_flags(parsed_arguments.ldflags) | |
| 275 # Make sure the linker searches the instrumented libraries dir for | |
| 276 # library dependencies. | |
| 277 environment['LDFLAGS'] = '%s -L%s/lib' % (ldflags, install_prefix) | |
| 278 | |
| 279 if parsed_arguments.sanitizer_type == 'asan': | |
| 280 # Do not report leaks during the build process. | |
| 281 environment['ASAN_OPTIONS'] = '%s:detect_leaks=0' % \ | |
| 282 environment.get('ASAN_OPTIONS', '') | |
| 283 | |
| 284 # libappindicator1 needs this. | |
| 285 environment['CSC'] = '/usr/bin/mono-csc' | |
| 286 return environment | |
| 287 | |
| 288 | |
| 289 | |
| 290 def download_build_install(parsed_arguments): | |
| 291 product_directory = os.path.normpath('%s/%s' % ( | |
| 292 SCRIPT_ABSOLUTE_PATH, | |
| 293 parsed_arguments.product_directory)) | |
| 294 | |
| 295 install_prefix = '%s/instrumented_libraries/%s' % ( | |
| 296 product_directory, | |
| 297 parsed_arguments.sanitizer_type) | |
| 298 | |
| 299 environment = build_environment(parsed_arguments, product_directory, | |
| 300 install_prefix) | |
| 301 | |
| 302 package_directory = '%s/%s' % (parsed_arguments.intermediate_directory, | |
| 303 parsed_arguments.package) | |
| 304 | |
| 305 # Clobber by default, unless the developer wants to hack on the package's | |
| 306 # source code. | |
| 307 clobber = (environment.get('INSTRUMENTED_LIBRARIES_NO_CLOBBER', '') != '1') | |
| 308 | |
| 309 download_source = True | |
| 310 if os.path.exists(package_directory): | |
| 311 if clobber: | |
| 312 shell_call('rm -rf %s' % package_directory, parsed_arguments.verbose) | |
| 313 else: | |
| 314 download_source = False | |
| 315 if download_source: | |
| 316 os.makedirs(package_directory) | |
| 317 | |
| 318 with ScopedChangeDirectory(package_directory) as cd_package: | |
| 319 if download_source: | |
| 320 shell_call('apt-get source %s' % parsed_arguments.package, | |
| 321 parsed_arguments.verbose) | |
| 322 # There should be exactly one subdirectory after downloading a package. | |
| 323 subdirectories = [d for d in os.listdir('.') if os.path.isdir(d)] | |
| 324 if len(subdirectories) != 1: | |
| 325 raise Exception('apt-get source %s must create exactly one subdirectory.' | |
| 326 % parsed_arguments.package) | |
| 327 with ScopedChangeDirectory(subdirectories[0]): | |
| 328 # Here we are in the package directory. | |
| 329 if download_source: | |
| 330 # Patch/run_before_build steps are only done once. | |
| 331 if parsed_arguments.patch: | |
| 332 shell_call( | |
| 333 'patch -p1 -i %s/%s' % | |
| 334 (os.path.relpath(cd_package.old_path), | |
| 335 parsed_arguments.patch), | |
| 336 parsed_arguments.verbose) | |
| 337 if parsed_arguments.run_before_build: | |
| 338 shell_call( | |
| 339 '%s/%s' % | |
| 340 (os.path.relpath(cd_package.old_path), | |
| 341 parsed_arguments.run_before_build), | |
| 342 parsed_arguments.verbose) | |
| 343 try: | |
| 344 build_and_install(parsed_arguments, environment, install_prefix) | |
| 345 except Exception as exception: | |
| 346 print exception | |
| 347 print 'Failed to build package %s.' % parsed_arguments.package | |
| 348 print ('Probably, some of its dependencies are not installed: %s' % | |
| 349 ' '.join(get_package_build_dependencies(parsed_arguments.package)
)) | |
| 350 sys.exit(1) | |
| 351 | |
| 352 # Touch a txt file to indicate package is installed. | |
| 353 open('%s/%s.txt' % (install_prefix, parsed_arguments.package), 'w').close() | |
| 354 | |
| 355 # Remove downloaded package and generated temporary build files. | |
| 356 # Failed builds intentionally skip this step, in order to aid in tracking down | |
| 357 # build failures. | |
| 358 if clobber: | |
| 359 shell_call('rm -rf %s' % package_directory, parsed_arguments.verbose) | |
| 360 | 342 |
| 361 def main(): | 343 def main(): |
| 362 argument_parser = argparse.ArgumentParser( | 344 parser = argparse.ArgumentParser( |
| 363 description='Download, build and install instrumented package') | 345 description='Download, build and install an instrumented package.') |
| 364 | 346 |
| 365 argument_parser.add_argument('-j', '--jobs', type=int, default=1) | 347 parser.add_argument('-j', '--jobs', type=int, default=1) |
| 366 argument_parser.add_argument('-p', '--package', required=True) | 348 parser.add_argument('-p', '--package', required=True) |
| 367 argument_parser.add_argument( | 349 parser.add_argument( |
| 368 '-i', '--product-directory', default='.', | 350 '-i', '--product-dir', default='.', |
| 369 help='Relative path to the directory with chrome binaries') | 351 help='Relative path to the directory with chrome binaries') |
| 370 argument_parser.add_argument( | 352 parser.add_argument( |
| 371 '-m', '--intermediate-directory', default='.', | 353 '-m', '--intermediate-dir', default='.', |
| 372 help='Relative path to the directory for temporary build files') | 354 help='Relative path to the directory for temporary build files') |
| 373 argument_parser.add_argument('--extra-configure-flags', default='') | 355 parser.add_argument('--extra-configure-flags', default='') |
| 374 argument_parser.add_argument('--cflags', default='') | 356 parser.add_argument('--cflags', default='') |
| 375 argument_parser.add_argument('--ldflags', default='') | 357 parser.add_argument('--ldflags', default='') |
| 376 argument_parser.add_argument('-s', '--sanitizer-type', required=True, | 358 parser.add_argument('-s', '--sanitizer', required=True, |
| 377 choices=['asan', 'msan', 'tsan']) | 359 choices=['asan', 'msan', 'tsan']) |
| 378 argument_parser.add_argument('-v', '--verbose', action='store_true') | 360 parser.add_argument('-v', '--verbose', action='store_true') |
| 379 argument_parser.add_argument('--check-build-deps', action='store_true') | 361 parser.add_argument('--cc') |
| 380 argument_parser.add_argument('--cc') | 362 parser.add_argument('--cxx') |
| 381 argument_parser.add_argument('--cxx') | 363 parser.add_argument('--patch', default='') |
| 382 argument_parser.add_argument('--patch', default='') | |
| 383 # This should be a shell script to run before building specific libraries. | 364 # This should be a shell script to run before building specific libraries. |
| 384 # This will be run after applying the patch above. | 365 # This will be run after applying the patch above. |
| 385 argument_parser.add_argument('--run-before-build', default='') | 366 parser.add_argument('--run-before-build', default='') |
| 386 argument_parser.add_argument('--build-method', default='destdir') | 367 parser.add_argument('--build-method', default='destdir') |
| 387 argument_parser.add_argument('--sanitizer-blacklist', default='') | 368 parser.add_argument('--sanitizer-blacklist', default='') |
| 369 # The LIBDIR argument to configure/make. |
| 370 parser.add_argument('--libdir', default='lib') |
| 388 | 371 |
| 389 # Ignore all empty arguments because in several cases gyp passes them to the | 372 # Ignore all empty arguments because in several cases gyp passes them to the |
| 390 # script, but ArgumentParser treats them as positional arguments instead of | 373 # script, but ArgumentParser treats them as positional arguments instead of |
| 391 # ignoring (and doesn't have such options). | 374 # ignoring (and doesn't have such options). |
| 392 parsed_arguments = argument_parser.parse_args( | 375 args = parser.parse_args([arg for arg in sys.argv[1:] if len(arg) != 0]) |
| 393 [arg for arg in sys.argv[1:] if len(arg) != 0]) | 376 |
| 394 # Ensure current working directory is this script directory. | 377 # Clobber by default, unless the developer wants to hack on the package's |
| 395 os.chdir(SCRIPT_ABSOLUTE_PATH) | 378 # source code. |
| 396 # Ensure all build dependencies are installed. | 379 clobber = \ |
| 397 if parsed_arguments.check_build_deps: | 380 (os.environ.get('INSTRUMENTED_LIBRARIES_NO_CLOBBER', '') != '1') |
| 398 check_package_build_dependencies(parsed_arguments.package) | 381 |
| 399 | 382 if args.build_method == 'destdir': |
| 400 download_build_install(parsed_arguments) | 383 builder = InstrumentedPackageBuilder(args, clobber) |
| 401 | 384 elif args.build_method == 'custom_nss': |
| 385 builder = NSSBuilder(args, clobber) |
| 386 elif args.build_method == 'custom_libcap': |
| 387 builder = LibcapBuilder(args, clobber) |
| 388 elif args.build_method == 'custom_libpci3': |
| 389 builder = Libpci3Builder(args, clobber) |
| 390 else: |
| 391 raise Exception('Unrecognized build method: %s' % args.build_method) |
| 392 |
| 393 builder.download_build_install() |
| 402 | 394 |
| 403 if __name__ == '__main__': | 395 if __name__ == '__main__': |
| 404 main() | 396 main() |
| OLD | NEW |