OLD | NEW |
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 # Copyright 2013 The Chromium Authors. All rights reserved. | 2 # Copyright 2013 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Downloads, builds (with instrumentation) and installs shared libraries.""" | 6 """Downloads, builds (with instrumentation) and installs shared libraries.""" |
7 | 7 |
8 import argparse | 8 import argparse |
9 import os | 9 import os |
10 import platform | 10 import platform |
11 import re | 11 import re |
12 import shlex | 12 import shlex |
13 import shutil | 13 import shutil |
14 import subprocess | 14 import subprocess |
15 import sys | 15 import sys |
16 | 16 |
17 SCRIPT_ABSOLUTE_PATH = os.path.dirname(os.path.abspath(__file__)) | 17 SCRIPT_ABSOLUTE_PATH = os.path.dirname(os.path.abspath(__file__)) |
18 | 18 |
19 class ScopedChangeDirectory(object): | 19 def unescape_flags(s): |
20 """Changes current working directory and restores it back automatically.""" | 20 """Un-escapes build flags received from GYP. |
21 | 21 |
22 def __init__(self, path): | 22 GYP escapes build flags as if they are to be inserted directly into a command |
23 self.path = path | 23 line, wrapping each flag in double quotes. When flags are passed via |
24 self.old_path = '' | 24 CFLAGS/LDFLAGS instead, double quotes must be dropped. |
25 | |
26 def __enter__(self): | |
27 self.old_path = os.getcwd() | |
28 os.chdir(self.path) | |
29 return self | |
30 | |
31 def __exit__(self, exc_type, exc_value, traceback): | |
32 os.chdir(self.old_path) | |
33 | |
34 def get_package_build_dependencies(package): | |
35 command = 'apt-get -s build-dep %s | grep Inst | cut -d " " -f 2' % package | |
36 command_result = subprocess.Popen(command, stdout=subprocess.PIPE, | |
37 shell=True) | |
38 if command_result.wait(): | |
39 raise Exception('Failed to determine build dependencies for %s' % package) | |
40 build_dependencies = [l.strip() for l in command_result.stdout] | |
41 return build_dependencies | |
42 | |
43 | |
44 def check_package_build_dependencies(package): | |
45 build_dependencies = get_package_build_dependencies(package) | |
46 if len(build_dependencies): | |
47 print >> sys.stderr, 'Please, install build-dependencies for %s' % package | |
48 print >> sys.stderr, 'One-liner for APT:' | |
49 print >> sys.stderr, 'sudo apt-get -y --no-remove build-dep %s' % package | |
50 sys.exit(1) | |
51 | |
52 | |
53 def shell_call(command, verbose=False, environment=None): | |
54 """ Wrapper on subprocess.Popen | |
55 | |
56 Calls command with specific environment and verbosity using | |
57 subprocess.Popen | |
58 | |
59 Args: | |
60 command: Command to run in shell. | |
61 verbose: If False, hides all stdout and stderr in case of successful build. | |
62 Otherwise, always prints stdout and stderr. | |
63 environment: Parameter 'env' for subprocess.Popen. | |
64 | |
65 Returns: | |
66 None | |
67 | |
68 Raises: | |
69 Exception: if return code after call is not zero. | |
70 """ | 25 """ |
71 child = subprocess.Popen( | 26 return ' '.join(shlex.split(s)) |
72 command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, | 27 |
73 env=environment, shell=True) | 28 |
74 stdout, stderr = child.communicate() | 29 def real_path(path_relative_to_script): |
75 if verbose or child.returncode: | 30 """Returns the absolute path to a file. |
76 print stdout | 31 |
77 if child.returncode: | 32 GYP generates paths relative to the location of the .gyp file, which coincides |
78 raise Exception('Failed to run: %s' % command) | 33 with the location of this script. This function converts them to absolute |
79 | 34 paths. |
80 | 35 """ |
81 def run_shell_commands(commands, verbose=False, environment=None): | 36 return os.path.realpath(os.path.join(SCRIPT_ABSOLUTE_PATH, |
82 for command in commands: | 37 path_relative_to_script)) |
83 shell_call(command, verbose, environment) | 38 |
84 | 39 |
85 | 40 class InstrumentedPackageBuilder(object): |
86 def fix_rpaths(destdir): | 41 """Checks out and builds a single instrumented package.""" |
87 # TODO(earthdok): reimplement fix_rpaths.sh in Python. | 42 def __init__(self, args, clobber): |
88 shell_call("%s/fix_rpaths.sh %s/lib" % (SCRIPT_ABSOLUTE_PATH, destdir)) | 43 self._cc = args.cc |
89 | 44 self._cxx = args.cxx |
90 | 45 self._extra_configure_flags = args.extra_configure_flags |
91 def destdir_configure_make_install(parsed_arguments, environment, | 46 self._jobs = args.jobs |
92 install_prefix): | 47 self._libdir = args.libdir |
93 configure_command = './configure %s' % parsed_arguments.extra_configure_flags | 48 self._package = args.package |
94 configure_command += ' --libdir=/lib/' | 49 self._patch = real_path(args.patch) if args.patch else None |
95 # Installing to a temporary directory allows us to safely clean up the .la | 50 self._run_before_build = \ |
96 # files below. | 51 real_path(args.run_before_build) if args.run_before_build else None |
97 destdir = '%s/debian/instrumented_build' % os.getcwd() | 52 self._sanitizer = args.sanitizer |
98 # Some makefiles use BUILDROOT or INSTALL_ROOT instead of DESTDIR. | 53 self._verbose = args.verbose |
99 make_command = 'make DESTDIR=%s BUILDROOT=%s INSTALL_ROOT=%s' % (destdir, | 54 self._clobber = clobber |
100 destdir, | 55 self._working_dir = os.path.join( |
101 destdir) | 56 real_path(args.intermediate_dir), self._package, '') |
102 build_and_install_in_destdir = [ | 57 |
103 configure_command, | 58 product_dir = real_path(args.product_dir) |
104 '%s -j%s' % (make_command, parsed_arguments.jobs), | 59 self._destdir = os.path.join( |
105 # Parallel install is flaky for some packages. | 60 product_dir, 'instrumented_libraries', self._sanitizer) |
106 '%s install -j1' % make_command, | 61 |
107 # Kill the .la files. They contain absolute paths, and will cause build | 62 self._cflags = unescape_flags(args.cflags) |
108 # errors in dependent libraries. | 63 if args.sanitizer_blacklist: |
109 'rm %s/lib/*.la -f' % destdir | 64 blacklist_file = real_path(args.sanitizer_blacklist) |
110 ] | 65 self._cflags += ' -fsanitize-blacklist=%s' % blacklist_file |
111 run_shell_commands(build_and_install_in_destdir, | 66 |
112 parsed_arguments.verbose, environment) | 67 self._ldflags = unescape_flags(args.ldflags) |
113 fix_rpaths(destdir) | 68 |
114 run_shell_commands([ | 69 self.init_build_env() |
115 # Now move the contents of the temporary destdir to their final place. | 70 |
116 # We only care for the contents of lib/. | 71 # Initialized later. |
117 'mkdir -p %s/lib' % install_prefix, | 72 self._source_dir = None |
118 'cp %s/lib/* %s/lib/ -rdf' % (destdir, install_prefix)], | 73 |
119 parsed_arguments.verbose, environment) | 74 def init_build_env(self): |
120 | 75 self._build_env = os.environ.copy() |
121 | 76 |
122 def nss_make_and_copy(parsed_arguments, environment, install_prefix): | 77 self._build_env['CC'] = self._cc |
123 # NSS uses a build system that's different from configure/make/install. All | 78 self._build_env['CXX'] = self._cxx |
124 # flags must be passed as arguments to make. | 79 |
125 make_args = [] | 80 self._build_env['CFLAGS'] = self._cflags |
126 # Do an optimized build. | 81 self._build_env['CXXFLAGS'] = self._cflags |
127 make_args.append('BUILD_OPT=1') | 82 self._build_env['LDFLAGS'] = self._ldflags |
128 # Set USE_64=1 on x86_64 systems. | 83 |
129 if platform.architecture()[0] == '64bit': | 84 if self._sanitizer == 'asan': |
130 make_args.append('USE_64=1') | 85 # Do not report leaks during the build process. |
131 # Passing C(XX)FLAGS overrides the defaults, and EXTRA_C(XX)FLAGS is not | 86 self._build_env['ASAN_OPTIONS'] = \ |
132 # supported. Append our extra flags to CC/CXX. | 87 '%s:detect_leaks=0' % self._build_env.get('ASAN_OPTIONS', '') |
133 make_args.append('CC="%s %s"' % (environment['CC'], environment['CFLAGS'])) | 88 |
134 make_args.append('CXX="%s %s"' % | 89 # libappindicator1 needs this. |
135 (environment['CXX'], environment['CXXFLAGS'])) | 90 self._build_env['CSC'] = '/usr/bin/mono-csc' |
136 # We need to override ZDEFS_FLAG at least to prevent -Wl,-z,defs. | 91 |
137 # Might as well use this to pass the linker flags, since ZDEF_FLAG is always | 92 def shell_call(self, command, env=None, cwd=None): |
138 # added during linking on Linux. | 93 """Wrapper around subprocess.Popen(). |
139 make_args.append('ZDEFS_FLAG="-Wl,-z,nodefs %s"' % environment['LDFLAGS']) | 94 |
140 make_args.append('NSPR_INCLUDE_DIR=/usr/include/nspr') | 95 Calls command with specific environment and verbosity using |
141 make_args.append('NSPR_LIB_DIR=%s/lib' % install_prefix) | 96 subprocess.Popen(). |
142 make_args.append('NSS_ENABLE_ECC=1') | 97 """ |
143 # Make sure we don't override the default flags. | 98 child = subprocess.Popen( |
144 for variable in ['CFLAGS', 'CXXFLAGS', 'LDFLAGS']: | 99 command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, |
145 del environment[variable] | 100 env=env, shell=True, cwd=cwd) |
146 with ScopedChangeDirectory('nss') as cd_nss: | 101 stdout, stderr = child.communicate() |
147 # -j is not supported | 102 if self._verbose or child.returncode: |
148 shell_call('make %s' % ' '.join(make_args), parsed_arguments.verbose, | 103 print stdout |
149 environment) | 104 if child.returncode: |
150 fix_rpaths(os.getcwd()) | 105 raise Exception('Failed to run: %s' % command) |
| 106 |
| 107 def maybe_download_source(self): |
| 108 """Checks out the source code (if needed) and sets self._source_dir.""" |
| 109 get_fresh_source = self._clobber or not os.path.exists(self._working_dir) |
| 110 if get_fresh_source: |
| 111 self.shell_call('rm -rf %s' % self._working_dir) |
| 112 os.makedirs(self._working_dir) |
| 113 self.shell_call('apt-get source %s' % self._package, |
| 114 cwd=self._working_dir) |
| 115 |
| 116 (dirpath, dirnames, filenames) = os.walk(self._working_dir).next() |
| 117 if len(dirnames) != 1: |
| 118 raise Exception('apt-get source %s must create exactly one subdirectory.' |
| 119 % self._package) |
| 120 self._source_dir = os.path.join(dirpath, dirnames[0], '') |
| 121 |
| 122 return get_fresh_source |
| 123 |
| 124 def patch_source(self): |
| 125 if self._patch: |
| 126 self.shell_call('patch -p1 -i %s' % self._patch, cwd=self._source_dir) |
| 127 if self._run_before_build: |
| 128 self.shell_call(self._run_before_build, cwd=self._source_dir) |
| 129 |
| 130 def download_build_install(self): |
| 131 got_fresh_source = self.maybe_download_source() |
| 132 if got_fresh_source: |
| 133 self.patch_source() |
| 134 |
| 135 self.shell_call('mkdir -p %s' % self.dest_libdir()) |
| 136 |
| 137 try: |
| 138 self.build_and_install() |
| 139 except Exception as exception: |
| 140 print 'ERROR: Failed to build package %s. Have you run ' \ |
| 141 'src/third_party/instrumented_libraries/install-build-deps.sh?' % \ |
| 142 self._package |
| 143 print |
| 144 raise |
| 145 |
| 146 # Touch a text file to indicate package is installed. |
| 147 stamp_file = os.path.join(self._destdir, '%s.txt' % self._package) |
| 148 open(stamp_file, 'w').close() |
| 149 |
| 150 # Remove downloaded package and generated temporary build files. Failed |
| 151 # builds intentionally skip this step to help debug build failures. |
| 152 if self._clobber: |
| 153 self.shell_call('rm -rf %s' % self._working_dir) |
| 154 |
| 155 def fix_rpaths(self, directory): |
| 156 # TODO(earthdok): reimplement fix_rpaths.sh in Python. |
| 157 script = real_path('fix_rpaths.sh') |
| 158 self.shell_call("%s %s" % (script, directory)) |
| 159 |
| 160 def temp_dir(self): |
| 161 """Returns the directory which will be passed to `make install'.""" |
| 162 return os.path.join(self._source_dir, 'debian', 'instrumented_build') |
| 163 |
| 164 def temp_libdir(self): |
| 165 """Returns the directory under temp_dir() containing the DSOs.""" |
| 166 return os.path.join(self.temp_dir(), self._libdir) |
| 167 |
| 168 def dest_libdir(self): |
| 169 """Returns the final location of the DSOs.""" |
| 170 return os.path.join(self._destdir, self._libdir) |
| 171 |
| 172 def make(self, args, jobs=None, env=None, cwd=None): |
| 173 """Invokes `make'. |
| 174 |
| 175 Invokes `make' with the specified args, using self._build_env and |
| 176 self._source_dir by default. |
| 177 """ |
| 178 if jobs is None: |
| 179 jobs = self._jobs |
| 180 if cwd is None: |
| 181 cwd = self._source_dir |
| 182 if env is None: |
| 183 env = self._build_env |
| 184 cmd = ['make', '-j%s' % jobs] + args |
| 185 self.shell_call(' '.join(cmd), env=env, cwd=cwd) |
| 186 |
| 187 def make_install(self, args, **kwargs): |
| 188 """Invokes `make install'.""" |
| 189 self.make(['install'] + args, **kwargs) |
| 190 |
| 191 def build_and_install(self): |
| 192 """Builds and installs the DSOs. |
| 193 |
| 194 Builds the package with ./configure + make, installs it to a temporary |
| 195 location, then moves the relevant files to their permanent location. |
| 196 """ |
| 197 configure_cmd = './configure --libdir=/%s/ %s' % ( |
| 198 self._libdir, self._extra_configure_flags) |
| 199 self.shell_call(configure_cmd, env=self._build_env, cwd=self._source_dir) |
| 200 |
| 201 # Some makefiles use BUILDROOT or INSTALL_ROOT instead of DESTDIR. |
| 202 args = ['DESTDIR', 'BUILDROOT', 'INSTALL_ROOT'] |
| 203 make_args = ['%s=%s' % (name, self.temp_dir()) for name in args] |
| 204 self.make(make_args) |
| 205 |
| 206 # Some packages don't support parallel install. Use -j1 always. |
| 207 self.make_install(make_args, jobs=1) |
| 208 |
| 209 # .la files are not needed, nuke them. |
| 210 self.shell_call('rm %s/*.la -f' % self.temp_libdir()) |
| 211 |
| 212 self.fix_rpaths(self.temp_libdir()) |
| 213 |
| 214 # Now move the contents of the temporary destdir to their final place. |
| 215 # We only care for the contents of LIBDIR. |
| 216 self.shell_call('cp %s/* %s/ -rdf' % (self.temp_libdir(), |
| 217 self.dest_libdir())) |
| 218 |
| 219 |
| 220 class LibcapBuilder(InstrumentedPackageBuilder): |
| 221 def build_and_install(self): |
| 222 # libcap2 doesn't have a configure script |
| 223 build_args = ['CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS'] |
| 224 make_args = [ |
| 225 '%s="%s"' % (name, self._build_env[name]) for name in build_args |
| 226 ] |
| 227 self.make(make_args) |
| 228 |
| 229 install_args = [ |
| 230 'DESTDIR=%s' % self.temp_dir(), |
| 231 'lib=%s' % self._libdir, |
| 232 # Skip a step that requires sudo. |
| 233 'RAISE_SETFCAP=no' |
| 234 ] |
| 235 self.make_install(install_args) |
| 236 |
| 237 self.fix_rpaths(self.temp_libdir()) |
| 238 |
| 239 # Now move the contents of the temporary destdir to their final place. |
| 240 # We only care for the contents of LIBDIR. |
| 241 self.shell_call('cp %s/* %s/ -rdf' % (self.temp_libdir(), |
| 242 self.dest_libdir())) |
| 243 |
| 244 |
| 245 class Libpci3Builder(InstrumentedPackageBuilder): |
| 246 def package_version(self): |
| 247 """Guesses libpci3 version from source directory name.""" |
| 248 dir_name = os.path.split(os.path.normpath(self._source_dir))[-1] |
| 249 match = re.match('pciutils-(\d+\.\d+\.\d+)', dir_name) |
| 250 if match is None: |
| 251 raise Exception( |
| 252 'Unable to guess libpci3 version from directory name: %s' % dir_name) |
| 253 return match.group(1) |
| 254 |
| 255 def temp_libdir(self): |
| 256 # DSOs have to be picked up from <source_dir>/lib, since `make install' |
| 257 # doesn't actualy install them anywhere. |
| 258 return os.path.join(self._source_dir, 'lib') |
| 259 |
| 260 def build_and_install(self): |
| 261 # pciutils doesn't have a configure script |
| 262 # This build process follows debian/rules. |
| 263 self.shell_call('mkdir -p %s-udeb/usr/bin' % self.temp_dir()) |
| 264 |
| 265 build_args = ['CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS'] |
| 266 make_args = [ |
| 267 '%s="%s"' % (name, self._build_env[name]) for name in build_args |
| 268 ] |
| 269 make_args += [ |
| 270 'LIBDIR=/%s/' % self._libdir, |
| 271 'PREFIX=/usr', |
| 272 'SBINDIR=/usr/bin', |
| 273 'IDSDIR=/usr/share/misc', |
| 274 'SHARED=yes', |
| 275 # pciutils-3.2.1 (Trusty) fails to build due to unresolved libkmod |
| 276 # symbols. The binary package has no dependencies on libkmod, so it |
| 277 # looks like it was actually built without libkmod support. |
| 278 'LIBKMOD=no', |
| 279 ] |
| 280 self.make(make_args) |
| 281 |
| 282 # `make install' is not needed. |
| 283 self.fix_rpaths(self.temp_libdir()) |
| 284 |
| 285 # Now install the DSOs to their final place. |
| 286 self.shell_call( |
| 287 'install -m 644 %s/libpci.so* %s' % (self.temp_libdir(), |
| 288 self.dest_libdir())) |
| 289 self.shell_call( |
| 290 'ln -sf libpci.so.%s %s/libpci.so.3' % (self.package_version(), |
| 291 self.dest_libdir())) |
| 292 |
| 293 |
| 294 class NSSBuilder(InstrumentedPackageBuilder): |
| 295 def build_and_install(self): |
| 296 # NSS uses a build system that's different from configure/make/install. All |
| 297 # flags must be passed as arguments to make. |
| 298 make_args = [ |
| 299 # Do an optimized build. |
| 300 'BUILD_OPT=1', |
| 301 # CFLAGS/CXXFLAGS should not be used, as doing so overrides the flags in |
| 302 # the makefile completely. The only way to append our flags is to tack |
| 303 # them onto CC/CXX. |
| 304 'CC="%s %s"' % (self._build_env['CC'], self._build_env['CFLAGS']), |
| 305 'CXX="%s %s"' % (self._build_env['CXX'], self._build_env['CXXFLAGS']), |
| 306 # We need to override ZDEFS_FLAG at least to avoid -Wl,-z,defs, which |
| 307 # is not compatible with sanitizers. We also need some way to pass |
| 308 # LDFLAGS without overriding the defaults. Conveniently, ZDEF_FLAG is |
| 309 # always appended to link flags when building NSS on Linux, so we can |
| 310 # just add our LDFLAGS here. |
| 311 'ZDEFS_FLAG="-Wl,-z,nodefs %s"' % self._build_env['LDFLAGS'], |
| 312 'NSPR_INCLUDE_DIR=/usr/include/nspr', |
| 313 'NSPR_LIB_DIR=%s' % self.dest_libdir(), |
| 314 'NSS_ENABLE_ECC=1' |
| 315 ] |
| 316 if platform.architecture()[0] == '64bit': |
| 317 make_args.append('USE_64=1') |
| 318 |
| 319 # Make sure we don't override the default flags in the makefile. |
| 320 for variable in ['CFLAGS', 'CXXFLAGS', 'LDFLAGS']: |
| 321 del self._build_env[variable] |
| 322 |
| 323 # Hardcoded paths. |
| 324 temp_dir = os.path.join(self._source_dir, 'nss') |
| 325 temp_libdir = os.path.join(temp_dir, 'lib') |
| 326 |
| 327 # Parallel build is not supported. Also, the build happens in |
| 328 # <source_dir>/nss. |
| 329 self.make(make_args, jobs=1, cwd=temp_dir) |
| 330 |
| 331 self.fix_rpaths(temp_libdir) |
| 332 |
151 # 'make install' is not supported. Copy the DSOs manually. | 333 # 'make install' is not supported. Copy the DSOs manually. |
152 install_dir = '%s/lib/' % install_prefix | 334 for (dirpath, dirnames, filenames) in os.walk(temp_libdir): |
153 for (dirpath, dirnames, filenames) in os.walk('./lib/'): | |
154 for filename in filenames: | 335 for filename in filenames: |
155 if filename.endswith('.so'): | 336 if filename.endswith('.so'): |
156 full_path = os.path.join(dirpath, filename) | 337 full_path = os.path.join(dirpath, filename) |
157 if parsed_arguments.verbose: | 338 if self._verbose: |
158 print 'download_build_install.py: installing %s' % full_path | 339 print 'download_build_install.py: installing %s' % full_path |
159 shutil.copy(full_path, install_dir) | 340 shutil.copy(full_path, self.dest_libdir()) |
160 | 341 |
161 | |
162 def libcap2_make_install(parsed_arguments, environment, install_prefix): | |
163 # libcap2 doesn't come with a configure script | |
164 make_args = [ | |
165 '%s="%s"' % (name, environment[name]) | |
166 for name in['CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS']] | |
167 shell_call('make -j%s %s' % (parsed_arguments.jobs, ' '.join(make_args)), | |
168 parsed_arguments.verbose, environment) | |
169 destdir = '%s/debian/instrumented_build' % os.getcwd() | |
170 install_args = [ | |
171 'DESTDIR=%s' % destdir, | |
172 # Do not install in lib64/. | |
173 'lib=lib', | |
174 # Skip a step that requires sudo. | |
175 'RAISE_SETFCAP=no' | |
176 ] | |
177 shell_call('make -j%s install %s' % | |
178 (parsed_arguments.jobs, ' '.join(install_args)), | |
179 parsed_arguments.verbose, environment) | |
180 fix_rpaths(destdir) | |
181 run_shell_commands([ | |
182 # Now move the contents of the temporary destdir to their final place. | |
183 # We only care for the contents of lib/. | |
184 'mkdir -p %s/lib' % install_prefix, | |
185 'cp %s/lib/* %s/lib/ -rdf' % (destdir, install_prefix)], | |
186 parsed_arguments.verbose, environment) | |
187 | |
188 | |
189 def libpci3_make_install(parsed_arguments, environment, install_prefix): | |
190 # pciutils doesn't have a configure script | |
191 # This build script follows debian/rules. | |
192 | |
193 # Find out the package version. We'll use this when creating symlinks. | |
194 dir_name = os.path.split(os.getcwd())[-1] | |
195 match = re.match('pciutils-(\d+\.\d+\.\d+)', dir_name) | |
196 if match is None: | |
197 raise Exception( | |
198 'Unable to guess libpci3 version from directory name: %s' % dir_name) | |
199 version = match.group(1) | |
200 | |
201 # `make install' will create a "$(DESTDIR)-udeb" directory alongside destdir. | |
202 # We don't want that in our product dir, so we use an intermediate directory. | |
203 destdir = '%s/debian/pciutils' % os.getcwd() | |
204 make_args = [ | |
205 '%s="%s"' % (name, environment[name]) | |
206 for name in['CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS']] | |
207 make_args.append('SHARED=yes') | |
208 # pciutils-3.2.1 (Trusty) fails to build due to unresolved libkmod symbols. | |
209 # The binary package has no dependencies on libkmod, so it looks like it was | |
210 # actually built without libkmod support. | |
211 make_args.append('LIBKMOD=no') | |
212 paths = [ | |
213 'LIBDIR=/lib/', | |
214 'PREFIX=/usr', | |
215 'SBINDIR=/usr/bin', | |
216 'IDSDIR=/usr/share/misc', | |
217 ] | |
218 install_args = ['DESTDIR=%s' % destdir] | |
219 run_shell_commands([ | |
220 'mkdir -p %s-udeb/usr/bin' % destdir, | |
221 'make -j%s %s' % (parsed_arguments.jobs, ' '.join(make_args + paths)), | |
222 'make -j%s %s install' % ( | |
223 parsed_arguments.jobs, | |
224 ' '.join(install_args + paths))], | |
225 parsed_arguments.verbose, environment) | |
226 fix_rpaths(destdir) | |
227 # Now install the DSOs to their final place. | |
228 run_shell_commands([ | |
229 'mkdir -p %s/lib' % install_prefix, | |
230 'install -m 644 lib/libpci.so* %s/lib/' % install_prefix, | |
231 'ln -sf libpci.so.%s %s/lib/libpci.so.3' % (version, install_prefix)], | |
232 parsed_arguments.verbose, environment) | |
233 | |
234 | |
235 def build_and_install(parsed_arguments, environment, install_prefix): | |
236 if parsed_arguments.build_method == 'destdir': | |
237 destdir_configure_make_install( | |
238 parsed_arguments, environment, install_prefix) | |
239 elif parsed_arguments.build_method == 'custom_nss': | |
240 nss_make_and_copy(parsed_arguments, environment, install_prefix) | |
241 elif parsed_arguments.build_method == 'custom_libcap': | |
242 libcap2_make_install(parsed_arguments, environment, install_prefix) | |
243 elif parsed_arguments.build_method == 'custom_libpci3': | |
244 libpci3_make_install(parsed_arguments, environment, install_prefix) | |
245 else: | |
246 raise Exception('Unrecognized build method: %s' % | |
247 parsed_arguments.build_method) | |
248 | |
249 | |
250 def unescape_flags(s): | |
251 # GYP escapes the build flags as if they are going to be inserted directly | |
252 # into the command line. Since we pass them via CFLAGS/LDFLAGS, we must drop | |
253 # the double quotes accordingly. | |
254 return ' '.join(shlex.split(s)) | |
255 | |
256 | |
257 def build_environment(parsed_arguments, product_directory, install_prefix): | |
258 environment = os.environ.copy() | |
259 # The CC/CXX environment variables take precedence over the command line | |
260 # flags. | |
261 if 'CC' not in environment and parsed_arguments.cc: | |
262 environment['CC'] = parsed_arguments.cc | |
263 if 'CXX' not in environment and parsed_arguments.cxx: | |
264 environment['CXX'] = parsed_arguments.cxx | |
265 | |
266 cflags = unescape_flags(parsed_arguments.cflags) | |
267 if parsed_arguments.sanitizer_blacklist: | |
268 cflags += ' -fsanitize-blacklist=%s/%s' % ( | |
269 SCRIPT_ABSOLUTE_PATH, | |
270 parsed_arguments.sanitizer_blacklist) | |
271 environment['CFLAGS'] = cflags | |
272 environment['CXXFLAGS'] = cflags | |
273 | |
274 ldflags = unescape_flags(parsed_arguments.ldflags) | |
275 # Make sure the linker searches the instrumented libraries dir for | |
276 # library dependencies. | |
277 environment['LDFLAGS'] = '%s -L%s/lib' % (ldflags, install_prefix) | |
278 | |
279 if parsed_arguments.sanitizer_type == 'asan': | |
280 # Do not report leaks during the build process. | |
281 environment['ASAN_OPTIONS'] = '%s:detect_leaks=0' % \ | |
282 environment.get('ASAN_OPTIONS', '') | |
283 | |
284 # libappindicator1 needs this. | |
285 environment['CSC'] = '/usr/bin/mono-csc' | |
286 return environment | |
287 | |
288 | |
289 | |
290 def download_build_install(parsed_arguments): | |
291 product_directory = os.path.normpath('%s/%s' % ( | |
292 SCRIPT_ABSOLUTE_PATH, | |
293 parsed_arguments.product_directory)) | |
294 | |
295 install_prefix = '%s/instrumented_libraries/%s' % ( | |
296 product_directory, | |
297 parsed_arguments.sanitizer_type) | |
298 | |
299 environment = build_environment(parsed_arguments, product_directory, | |
300 install_prefix) | |
301 | |
302 package_directory = '%s/%s' % (parsed_arguments.intermediate_directory, | |
303 parsed_arguments.package) | |
304 | |
305 # Clobber by default, unless the developer wants to hack on the package's | |
306 # source code. | |
307 clobber = (environment.get('INSTRUMENTED_LIBRARIES_NO_CLOBBER', '') != '1') | |
308 | |
309 download_source = True | |
310 if os.path.exists(package_directory): | |
311 if clobber: | |
312 shell_call('rm -rf %s' % package_directory, parsed_arguments.verbose) | |
313 else: | |
314 download_source = False | |
315 if download_source: | |
316 os.makedirs(package_directory) | |
317 | |
318 with ScopedChangeDirectory(package_directory) as cd_package: | |
319 if download_source: | |
320 shell_call('apt-get source %s' % parsed_arguments.package, | |
321 parsed_arguments.verbose) | |
322 # There should be exactly one subdirectory after downloading a package. | |
323 subdirectories = [d for d in os.listdir('.') if os.path.isdir(d)] | |
324 if len(subdirectories) != 1: | |
325 raise Exception('apt-get source %s must create exactly one subdirectory.' | |
326 % parsed_arguments.package) | |
327 with ScopedChangeDirectory(subdirectories[0]): | |
328 # Here we are in the package directory. | |
329 if download_source: | |
330 # Patch/run_before_build steps are only done once. | |
331 if parsed_arguments.patch: | |
332 shell_call( | |
333 'patch -p1 -i %s/%s' % | |
334 (os.path.relpath(cd_package.old_path), | |
335 parsed_arguments.patch), | |
336 parsed_arguments.verbose) | |
337 if parsed_arguments.run_before_build: | |
338 shell_call( | |
339 '%s/%s' % | |
340 (os.path.relpath(cd_package.old_path), | |
341 parsed_arguments.run_before_build), | |
342 parsed_arguments.verbose) | |
343 try: | |
344 build_and_install(parsed_arguments, environment, install_prefix) | |
345 except Exception as exception: | |
346 print exception | |
347 print 'Failed to build package %s.' % parsed_arguments.package | |
348 print ('Probably, some of its dependencies are not installed: %s' % | |
349 ' '.join(get_package_build_dependencies(parsed_arguments.package)
)) | |
350 sys.exit(1) | |
351 | |
352 # Touch a txt file to indicate package is installed. | |
353 open('%s/%s.txt' % (install_prefix, parsed_arguments.package), 'w').close() | |
354 | |
355 # Remove downloaded package and generated temporary build files. | |
356 # Failed builds intentionally skip this step, in order to aid in tracking down | |
357 # build failures. | |
358 if clobber: | |
359 shell_call('rm -rf %s' % package_directory, parsed_arguments.verbose) | |
360 | 342 |
361 def main(): | 343 def main(): |
362 argument_parser = argparse.ArgumentParser( | 344 parser = argparse.ArgumentParser( |
363 description='Download, build and install instrumented package') | 345 description='Download, build and install an instrumented package.') |
364 | 346 |
365 argument_parser.add_argument('-j', '--jobs', type=int, default=1) | 347 parser.add_argument('-j', '--jobs', type=int, default=1) |
366 argument_parser.add_argument('-p', '--package', required=True) | 348 parser.add_argument('-p', '--package', required=True) |
367 argument_parser.add_argument( | 349 parser.add_argument( |
368 '-i', '--product-directory', default='.', | 350 '-i', '--product-dir', default='.', |
369 help='Relative path to the directory with chrome binaries') | 351 help='Relative path to the directory with chrome binaries') |
370 argument_parser.add_argument( | 352 parser.add_argument( |
371 '-m', '--intermediate-directory', default='.', | 353 '-m', '--intermediate-dir', default='.', |
372 help='Relative path to the directory for temporary build files') | 354 help='Relative path to the directory for temporary build files') |
373 argument_parser.add_argument('--extra-configure-flags', default='') | 355 parser.add_argument('--extra-configure-flags', default='') |
374 argument_parser.add_argument('--cflags', default='') | 356 parser.add_argument('--cflags', default='') |
375 argument_parser.add_argument('--ldflags', default='') | 357 parser.add_argument('--ldflags', default='') |
376 argument_parser.add_argument('-s', '--sanitizer-type', required=True, | 358 parser.add_argument('-s', '--sanitizer', required=True, |
377 choices=['asan', 'msan', 'tsan']) | 359 choices=['asan', 'msan', 'tsan']) |
378 argument_parser.add_argument('-v', '--verbose', action='store_true') | 360 parser.add_argument('-v', '--verbose', action='store_true') |
379 argument_parser.add_argument('--check-build-deps', action='store_true') | 361 parser.add_argument('--cc') |
380 argument_parser.add_argument('--cc') | 362 parser.add_argument('--cxx') |
381 argument_parser.add_argument('--cxx') | 363 parser.add_argument('--patch', default='') |
382 argument_parser.add_argument('--patch', default='') | |
383 # This should be a shell script to run before building specific libraries. | 364 # This should be a shell script to run before building specific libraries. |
384 # This will be run after applying the patch above. | 365 # This will be run after applying the patch above. |
385 argument_parser.add_argument('--run-before-build', default='') | 366 parser.add_argument('--run-before-build', default='') |
386 argument_parser.add_argument('--build-method', default='destdir') | 367 parser.add_argument('--build-method', default='destdir') |
387 argument_parser.add_argument('--sanitizer-blacklist', default='') | 368 parser.add_argument('--sanitizer-blacklist', default='') |
| 369 # The LIBDIR argument to configure/make. |
| 370 parser.add_argument('--libdir', default='lib') |
388 | 371 |
389 # Ignore all empty arguments because in several cases gyp passes them to the | 372 # Ignore all empty arguments because in several cases gyp passes them to the |
390 # script, but ArgumentParser treats them as positional arguments instead of | 373 # script, but ArgumentParser treats them as positional arguments instead of |
391 # ignoring (and doesn't have such options). | 374 # ignoring (and doesn't have such options). |
392 parsed_arguments = argument_parser.parse_args( | 375 args = parser.parse_args([arg for arg in sys.argv[1:] if len(arg) != 0]) |
393 [arg for arg in sys.argv[1:] if len(arg) != 0]) | 376 |
394 # Ensure current working directory is this script directory. | 377 # Clobber by default, unless the developer wants to hack on the package's |
395 os.chdir(SCRIPT_ABSOLUTE_PATH) | 378 # source code. |
396 # Ensure all build dependencies are installed. | 379 clobber = \ |
397 if parsed_arguments.check_build_deps: | 380 (os.environ.get('INSTRUMENTED_LIBRARIES_NO_CLOBBER', '') != '1') |
398 check_package_build_dependencies(parsed_arguments.package) | 381 |
399 | 382 if args.build_method == 'destdir': |
400 download_build_install(parsed_arguments) | 383 builder = InstrumentedPackageBuilder(args, clobber) |
401 | 384 elif args.build_method == 'custom_nss': |
| 385 builder = NSSBuilder(args, clobber) |
| 386 elif args.build_method == 'custom_libcap': |
| 387 builder = LibcapBuilder(args, clobber) |
| 388 elif args.build_method == 'custom_libpci3': |
| 389 builder = Libpci3Builder(args, clobber) |
| 390 else: |
| 391 raise Exception('Unrecognized build method: %s' % args.build_method) |
| 392 |
| 393 builder.download_build_install() |
402 | 394 |
403 if __name__ == '__main__': | 395 if __name__ == '__main__': |
404 main() | 396 main() |
OLD | NEW |