OLD | NEW |
| (Empty) |
1 #!/usr/bin/python | |
2 # Copyright (c) 2012 The Native Client Authors. All rights reserved. | |
3 # Use of this source code is governed by a BSD-style license that can be | |
4 # found in the LICENSE file. | |
5 | |
6 """NEXE building script | |
7 | |
8 This module will take a set of source files, include paths, library paths, and | |
9 additional arguments, and use them to build. | |
10 """ | |
11 | |
12 import hashlib | |
13 import json | |
14 import multiprocessing | |
15 from optparse import OptionParser | |
16 import os | |
17 import re | |
18 import Queue | |
19 import shutil | |
20 import StringIO | |
21 import subprocess | |
22 import sys | |
23 import tempfile | |
24 import time | |
25 import traceback | |
26 import urllib2 | |
27 | |
28 from build_nexe_tools import (CommandRunner, Error, FixPath, | |
29 IsFile, MakeDir, RemoveFile) | |
30 sys.path.append(os.path.join(os.path.dirname(__file__), '..')) | |
31 import pynacl.platform | |
32 | |
33 | |
34 # When a header file defining NACL_BUILD_SUBARCH is introduced, | |
35 # we can simply remove this map. | |
36 # cf) https://code.google.com/p/chromium/issues/detail?id=440012. | |
37 NACL_BUILD_ARCH_MAP = { | |
38 'x86-32': ['NACL_BUILD_ARCH=x86', 'NACL_BUILD_SUBARCH=32'], | |
39 'x86-32-nonsfi': ['NACL_BUILD_ARCH=x86', 'NACL_BUILD_SUBARCH=32'], | |
40 'x86-64': ['NACL_BUILD_ARCH=x86', 'NACL_BUILD_SUBARCH=64'], | |
41 'arm': ['NACL_BUILD_ARCH=arm', 'NACL_BUILD_SUBARCH=32'], | |
42 'arm-nonsfi': ['NACL_BUILD_ARCH=arm', 'NACL_BUILD_SUBARCH=32'], | |
43 'mips': ['NACL_BUILD_ARCH=mips', 'NACL_BUILD_SUBARCH=32'], | |
44 'pnacl': ['NACL_BUILD_ARCH=pnacl'], | |
45 } | |
46 | |
47 | |
48 def RemoveQuotes(opt): | |
49 if opt and opt[0] == '"': | |
50 assert opt[-1] == '"', opt | |
51 return opt[1:-1].replace('\\"', '"') | |
52 return opt | |
53 | |
54 | |
55 def ArgToList(opt): | |
56 outlist = [] | |
57 if opt is None: | |
58 return outlist | |
59 optlist = opt.split(' ') | |
60 for optitem in optlist: | |
61 optitem = RemoveQuotes(optitem) | |
62 if optitem: | |
63 outlist.append(optitem) | |
64 return outlist | |
65 | |
66 | |
67 def GetMTime(filepath): | |
68 """GetMTime returns the last modification time of the file or None.""" | |
69 try: | |
70 return os.path.getmtime(FixPath(filepath)) | |
71 except OSError: | |
72 return None | |
73 | |
74 | |
75 def IsStale(out_ts, src_ts, rebuilt=False): | |
76 # If either source or output timestamp was not available, assume stale. | |
77 if not out_ts or not src_ts: | |
78 return True | |
79 | |
80 # If just rebuilt timestamps may be equal due to time granularity. | |
81 if rebuilt: | |
82 return out_ts < src_ts | |
83 # If about to build, be conservative and rebuilt just in case. | |
84 return out_ts <= src_ts | |
85 | |
86 | |
87 def IsEnvFlagTrue(flag_name, default=False): | |
88 """Return true when the given flag is true. | |
89 | |
90 Note: | |
91 Any values that do not match the true pattern are False. | |
92 | |
93 Args: | |
94 flag_name: a string name of a flag. | |
95 default: default return value if the flag is not set. | |
96 | |
97 Returns: | |
98 True if the flag is in the true pattern. Otherwise False. | |
99 """ | |
100 flag_value = os.environ.get(flag_name) | |
101 if flag_value is None: | |
102 return default | |
103 return bool(re.search(r'^([tTyY]|1:?)', flag_value)) | |
104 | |
105 | |
106 def GetIntegerEnv(flag_name, default=0): | |
107 """Parses and returns integer environment variable. | |
108 | |
109 Args: | |
110 flag_name: a string name of a flag. | |
111 default: default return value if the flag is not set. | |
112 | |
113 Returns: | |
114 Integer value of the flag. | |
115 """ | |
116 flag_value = os.environ.get(flag_name) | |
117 if flag_value is None: | |
118 return default | |
119 try: | |
120 return int(flag_value) | |
121 except ValueError: | |
122 raise Error('Invalid ' + flag_name + ': ' + flag_value) | |
123 | |
124 | |
125 class Builder(CommandRunner): | |
126 """Builder object maintains options and generates build command-lines. | |
127 | |
128 The Builder object takes a set of script command-line options, and generates | |
129 a set of paths, and command-line options for the NaCl toolchain. | |
130 """ | |
131 def __init__(self, options): | |
132 super(Builder, self).__init__(options) | |
133 arch = options.arch | |
134 self.arch = arch | |
135 build_type = options.build.split('_') | |
136 toolname = build_type[0] | |
137 self.outtype = build_type[1] | |
138 self.osname = pynacl.platform.GetOS() | |
139 | |
140 # pnacl toolchain can be selected in three different ways | |
141 # 1. by specifying --arch=pnacl directly to generate | |
142 # pexe targets. | |
143 # 2. by specifying --build=newlib_translate to generated | |
144 # nexe via translation | |
145 # 3. by specifying --build=newlib_{nexe,nlib}_pnacl use pnacl | |
146 # toolchain in native mode (e.g. the IRT shim) | |
147 self.is_pnacl_toolchain = False | |
148 if self.outtype == 'translate': | |
149 self.is_pnacl_toolchain = True | |
150 | |
151 if len(build_type) > 2 and build_type[2] == 'pnacl': | |
152 self.is_pnacl_toolchain = True | |
153 | |
154 self.is_nacl_clang = len(build_type) > 2 and build_type[2] == 'clang' | |
155 | |
156 if arch.endswith('-nonsfi'): | |
157 arch = arch[:-len('-nonsfi')] | |
158 | |
159 if arch in ['x86-32', 'x86-64']: | |
160 mainarch = 'x86' | |
161 self.tool_prefix = 'x86_64-nacl-' | |
162 elif arch == 'arm': | |
163 self.tool_prefix = 'arm-nacl-' | |
164 mainarch = 'arm' | |
165 elif arch == 'mips': | |
166 self.tool_prefix = 'mipsel-nacl-' | |
167 mainarch = 'mipsel' | |
168 elif arch == 'pnacl': | |
169 self.is_pnacl_toolchain = True | |
170 else: | |
171 raise Error('Toolchain architecture %s not supported.' % arch) | |
172 | |
173 if toolname not in ['newlib', 'glibc']: | |
174 raise Error('Toolchain of type %s not supported.' % toolname) | |
175 | |
176 if arch == 'mips' and toolname == 'glibc': | |
177 raise Error('mips glibc not supported.') | |
178 | |
179 if arch == 'pnacl' and toolname == 'glibc': | |
180 raise Error('pnacl glibc not yet supported.') | |
181 | |
182 if self.is_pnacl_toolchain: | |
183 self.tool_prefix = 'pnacl-' | |
184 tool_subdir = 'pnacl_newlib' | |
185 elif self.is_nacl_clang: | |
186 tool_subdir = 'pnacl_newlib' | |
187 else: | |
188 tool_subdir = 'nacl_%s_%s' % (mainarch, toolname) | |
189 # The pnacl-clang, etc. tools are scripts. Note that for the CommandRunner | |
190 # so that it can know if a shell is needed or not. | |
191 self.SetCommandsAreScripts(self.is_pnacl_toolchain) | |
192 | |
193 build_arch = pynacl.platform.GetArch() | |
194 tooldir = os.path.join('%s_%s' % (self.osname, build_arch), tool_subdir) | |
195 | |
196 self.root_path = options.root | |
197 self.nacl_path = os.path.join(self.root_path, 'native_client') | |
198 | |
199 project_path, project_name = os.path.split(options.name) | |
200 self.outdir = options.objdir | |
201 | |
202 # Set the toolchain directories | |
203 self.toolchain = os.path.join(options.toolpath, tooldir) | |
204 self.toolbin = os.path.join(self.toolchain, 'bin') | |
205 self.toolstamp = os.path.join(self.toolchain, tool_subdir + '.json') | |
206 if not IsFile(self.toolstamp): | |
207 raise Error('Could not find toolchain prep stamp file: ' + self.toolstamp) | |
208 | |
209 self.inc_paths = ArgToList(options.incdirs) | |
210 self.lib_paths = ArgToList(options.libdirs) | |
211 self.define_list = ArgToList(options.defines) | |
212 | |
213 self.name = options.name | |
214 self.cmd_file = options.cmd_file | |
215 self.BuildCompileOptions( | |
216 options.compile_flags, self.define_list, options.arch) | |
217 self.BuildLinkOptions(options.link_flags) | |
218 self.BuildArchiveOptions() | |
219 self.strip = options.strip | |
220 self.empty = options.empty | |
221 self.strip_all = options.strip_all | |
222 self.strip_debug = options.strip_debug | |
223 self.tls_edit = options.tls_edit | |
224 self.finalize_pexe = options.finalize_pexe and arch == 'pnacl' | |
225 goma_config = self.GetGomaConfig(options.gomadir, arch, toolname) | |
226 self.gomacc = goma_config.get('gomacc', '') | |
227 self.goma_burst = goma_config.get('burst', False) | |
228 self.goma_processes = goma_config.get('processes', 1) | |
229 | |
230 # Define NDEBUG for Release builds. | |
231 if options.build_config.startswith('Release'): | |
232 self.compile_options.append('-DNDEBUG') | |
233 | |
234 # Use unoptimized native objects for debug IRT builds for faster compiles. | |
235 if (self.is_pnacl_toolchain | |
236 and (self.outtype == 'nlib' | |
237 or self.outtype == 'nexe') | |
238 and self.arch != 'pnacl'): | |
239 if (options.build_config is not None | |
240 and options.build_config.startswith('Debug')): | |
241 self.compile_options.extend(['--pnacl-allow-translate', | |
242 '--pnacl-allow-native', | |
243 '-arch', self.arch]) | |
244 | |
245 self.irt_linker = options.irt_linker | |
246 self.Log('Compile options: %s' % self.compile_options) | |
247 self.Log('Linker options: %s' % self.link_options) | |
248 | |
249 def IsGomaParallelBuild(self): | |
250 if self.gomacc and (self.goma_burst or self.goma_processes > 1): | |
251 return True | |
252 return False | |
253 | |
254 def GenNaClPath(self, path): | |
255 """Helper which prepends path with the native client source directory.""" | |
256 return os.path.join(self.root_path, 'native_client', path) | |
257 | |
258 def GetBinName(self, name): | |
259 """Helper which prepends executable with the toolchain bin directory.""" | |
260 return os.path.join(self.toolbin, self.tool_prefix + name) | |
261 | |
262 def GetCCompiler(self): | |
263 """Helper which returns C compiler path.""" | |
264 if self.is_pnacl_toolchain or self.is_nacl_clang: | |
265 return self.GetBinName('clang') | |
266 else: | |
267 return self.GetBinName('gcc') | |
268 | |
269 def GetCXXCompiler(self): | |
270 """Helper which returns C++ compiler path.""" | |
271 if self.is_pnacl_toolchain or self.is_nacl_clang: | |
272 return self.GetBinName('clang++') | |
273 else: | |
274 return self.GetBinName('g++') | |
275 | |
276 def GetAr(self): | |
277 """Helper which returns ar path.""" | |
278 return self.GetBinName('ar') | |
279 | |
280 def GetStrip(self): | |
281 """Helper which returns strip path.""" | |
282 return self.GetBinName('strip') | |
283 | |
284 def GetObjCopy(self): | |
285 """Helper which returns objcopy path.""" | |
286 return self.GetBinName('objcopy') | |
287 | |
288 def GetObjDump(self): | |
289 """Helper which returns objdump path.""" | |
290 return self.GetBinName('objdump') | |
291 | |
292 def GetReadElf(self): | |
293 """Helper which returns readelf path.""" | |
294 return self.GetBinName('readelf') | |
295 | |
296 def GetPnaclFinalize(self): | |
297 """Helper which returns pnacl-finalize path.""" | |
298 assert self.is_pnacl_toolchain | |
299 return self.GetBinName('finalize') | |
300 | |
301 def BuildAssembleOptions(self, options): | |
302 options = ArgToList(options) | |
303 self.assemble_options = options + ['-I' + name for name in self.inc_paths] | |
304 | |
305 def DebugName(self): | |
306 return self.name + '.debug' | |
307 | |
308 def UntaggedName(self): | |
309 return self.name + '.untagged' | |
310 | |
311 def LinkOutputName(self): | |
312 if (self.is_pnacl_toolchain and self.finalize_pexe or | |
313 self.strip_all or self.strip_debug): | |
314 return self.DebugName() | |
315 else: | |
316 return self.name | |
317 | |
318 def ArchiveOutputName(self): | |
319 if self.strip_debug: | |
320 return self.DebugName() | |
321 else: | |
322 return self.name | |
323 | |
324 def StripOutputName(self): | |
325 return self.name | |
326 | |
327 def TranslateOutputName(self): | |
328 return self.name | |
329 | |
330 def Soname(self): | |
331 return self.name | |
332 | |
333 def BuildCompileOptions(self, options, define_list, arch): | |
334 """Generates compile options, called once by __init__.""" | |
335 options = ArgToList(options) | |
336 # We want to shared gyp 'defines' with other targets, but not | |
337 # ones that are host system dependent. Filtering them out. | |
338 # This really should be better. | |
339 # See: http://code.google.com/p/nativeclient/issues/detail?id=2936 | |
340 define_list = [define for define in define_list | |
341 if not (define.startswith('NACL_WINDOWS=') or | |
342 define.startswith('NACL_OSX=') or | |
343 define.startswith('NACL_LINUX=') or | |
344 define.startswith('NACL_ANDROID=') or | |
345 define.startswith('NACL_BUILD_ARCH=') or | |
346 define.startswith('NACL_BUILD_SUBARCH=') or | |
347 define == 'COMPONENT_BUILD' or | |
348 'WIN32' in define or | |
349 'WINDOWS' in define or | |
350 'WINVER' in define)] | |
351 define_list.extend(['NACL_WINDOWS=0', | |
352 'NACL_OSX=0', | |
353 'NACL_LINUX=0', | |
354 'NACL_ANDROID=0']) | |
355 define_list.extend(NACL_BUILD_ARCH_MAP[arch]) | |
356 options += ['-D' + define for define in define_list] | |
357 self.compile_options = options + ['-I' + name for name in self.inc_paths] | |
358 | |
359 def BuildLinkOptions(self, options): | |
360 """Generates link options, called once by __init__.""" | |
361 options = ArgToList(options) | |
362 if self.outtype == 'nso': | |
363 options += ['-Wl,-rpath-link,' + name for name in self.lib_paths] | |
364 options += ['-shared'] | |
365 options += ['-Wl,-soname,' + os.path.basename(self.Soname())] | |
366 self.link_options = options + ['-L' + name for name in self.lib_paths] | |
367 | |
368 def BuildArchiveOptions(self): | |
369 """Generates link options, called once by __init__.""" | |
370 self.archive_options = [] | |
371 | |
372 def GetObjectName(self, src): | |
373 if self.strip: | |
374 src = src.replace(self.strip, '') | |
375 # Hash the full path of the source file and add 32 bits of that hash onto | |
376 # the end of the object file name. This helps disambiguate files with the | |
377 # same name, because all of the object files are placed into the same | |
378 # directory. Technically, the correct solution would be to preserve the | |
379 # directory structure of the input source files inside the object file | |
380 # directory, but doing that runs the risk of running into filename length | |
381 # issues on Windows. | |
382 h = hashlib.sha1() | |
383 h.update(src) | |
384 wart = h.hexdigest()[:8] | |
385 filename, _ = os.path.splitext(os.path.basename(src)) | |
386 return os.path.join(self.outdir, filename + '_' + wart + '.o') | |
387 | |
388 def FixWindowsPath(self, path): | |
389 # The windows version of the nacl toolchain returns badly | |
390 # formed system header paths. As we do want changes in the | |
391 # toolchain to trigger rebuilds, compensate by detecting | |
392 # malformed paths (starting with /libexec/) and assume these | |
393 # are actually toolchain relative. | |
394 # | |
395 # Additionally, in some cases the toolchains emit cygwin paths | |
396 # which don't work in a win32 python. | |
397 # Assume they are all /cygdrive/ relative and convert to a | |
398 # drive letter. | |
399 cygdrive = '/cygdrive/' | |
400 if path.startswith(cygdrive): | |
401 path = os.path.normpath( | |
402 path[len(cygdrive)] + ':' + path[len(cygdrive)+1:]) | |
403 elif path.startswith('/libexec/'): | |
404 path = os.path.normpath(os.path.join(self.toolchain, path[1:])) | |
405 return path | |
406 | |
407 def GetGomaConfig(self, gomadir, arch, toolname): | |
408 """Returns a goma config dictionary if goma is available or {}.""" | |
409 | |
410 # Goma is enabled only if gomadir is given. | |
411 # We do not use gomacc in GOMA_DIR or PATH anymore. | |
412 if not gomadir: | |
413 return {} | |
414 | |
415 # Start goma support from os/arch/toolname that have been tested. | |
416 # Set NO_NACL_GOMA=true to force to avoid using goma. | |
417 default_no_nacl_goma = True if pynacl.platform.IsWindows() else False | |
418 if (arch == 'mips' | |
419 or toolname not in ['newlib', 'glibc'] | |
420 or IsEnvFlagTrue('NO_NACL_GOMA', default=default_no_nacl_goma) | |
421 or IsEnvFlagTrue('GOMA_DISABLED')): | |
422 return {} | |
423 | |
424 gomacc_base = 'gomacc.exe' if pynacl.platform.IsWindows() else 'gomacc' | |
425 # TODO(yyanagisawa): not to set gomadir on use_goma=0. | |
426 gomacc = os.path.join(gomadir, gomacc_base) | |
427 if not os.path.exists(gomacc): | |
428 return {} | |
429 | |
430 goma_config = { | |
431 'gomacc': gomacc, | |
432 'burst': IsEnvFlagTrue('NACL_GOMA_BURST'), | |
433 } | |
434 default_processes = 100 if pynacl.platform.IsLinux() else 10 | |
435 goma_config['processes'] = GetIntegerEnv('NACL_GOMA_PROCESSES', | |
436 default=default_processes) | |
437 return goma_config | |
438 | |
439 def NeedsRebuild(self, outd, out, src, rebuilt=False): | |
440 if not IsFile(self.toolstamp): | |
441 if rebuilt: | |
442 raise Error('Could not find toolchain stamp file %s.' % self.toolstamp) | |
443 return True | |
444 if not IsFile(self.cmd_file): | |
445 if rebuilt: | |
446 raise Error('Could not find cmd file %s.' % self.cmd_file) | |
447 return True | |
448 if not IsFile(outd): | |
449 if rebuilt: | |
450 raise Error('Could not find dependency file %s.' % outd) | |
451 return True | |
452 if not IsFile(out): | |
453 if rebuilt: | |
454 raise Error('Could not find output file %s.' % out) | |
455 return True | |
456 | |
457 inputs = [__file__, self.toolstamp, src, self.cmd_file] | |
458 outputs = [out, outd] | |
459 | |
460 # Find their timestamps if any. | |
461 input_times = [(GetMTime(f), f) for f in inputs] | |
462 output_times = [(GetMTime(f), f) for f in outputs] | |
463 | |
464 # All inputs must exist. | |
465 missing_inputs = [p[1] for p in input_times if p[0] is None] | |
466 if missing_inputs: | |
467 raise Error('Missing inputs: %s' % str(missing_inputs)) | |
468 | |
469 # Rebuild if any outputs are missing. | |
470 missing_outputs = [p[1] for p in output_times if p[0] is None] | |
471 if missing_outputs: | |
472 if rebuilt: | |
473 raise Error('Outputs missing after rebuild: %s' % str(missing_outputs)) | |
474 return True | |
475 | |
476 newest_input = max(input_times) | |
477 oldest_output = min(output_times) | |
478 | |
479 if IsStale(oldest_output[0], newest_input[0], rebuilt): | |
480 if rebuilt: | |
481 raise Error('Output %s is older than toolchain stamp %s' % ( | |
482 oldest_output[1], newest_input[1])) | |
483 return True | |
484 | |
485 # Decode emitted makefile. | |
486 with open(FixPath(outd), 'r') as fh: | |
487 deps = fh.read() | |
488 # Remove line continuations | |
489 deps = deps.replace('\\\n', ' ') | |
490 deps = deps.replace('\n', '') | |
491 # The dependencies are whitespace delimited following the first ':' | |
492 # (that is not part of a windows drive letter) | |
493 deps = deps.split(':', 1) | |
494 if pynacl.platform.IsWindows() and len(deps[0]) == 1: | |
495 # The path has a drive letter, find the next ':' | |
496 deps = deps[1].split(':', 1)[1] | |
497 else: | |
498 deps = deps[1] | |
499 deps = deps.split() | |
500 if pynacl.platform.IsWindows(): | |
501 deps = [self.FixWindowsPath(d) for d in deps] | |
502 # Check if any input has changed. | |
503 for filename in deps: | |
504 file_tm = GetMTime(filename) | |
505 if IsStale(oldest_output[0], file_tm, rebuilt): | |
506 if rebuilt: | |
507 raise Error('Dependency %s is older than output %s.' % ( | |
508 filename, oldest_output[1])) | |
509 return True | |
510 return False | |
511 | |
512 def Compile(self, src): | |
513 """Compile the source with pre-determined options.""" | |
514 | |
515 compile_options = self.compile_options[:] | |
516 _, ext = os.path.splitext(src) | |
517 if ext in ['.c', '.S']: | |
518 bin_name = self.GetCCompiler() | |
519 compile_options.append('-std=gnu99') | |
520 if self.is_pnacl_toolchain and ext == '.S': | |
521 compile_options.append('-arch') | |
522 compile_options.append(self.arch) | |
523 elif ext in ['.cc', '.cpp']: | |
524 compile_options.append('-std=gnu++0x') | |
525 compile_options.append('-Wno-deprecated-register') | |
526 bin_name = self.GetCXXCompiler() | |
527 else: | |
528 if ext != '.h': | |
529 self.Log('Skipping unknown type %s for %s.' % (ext, src)) | |
530 return None | |
531 | |
532 # This option is only applicable to C, and C++ compilers warn if | |
533 # it is present, so remove it for C++ to avoid the warning. | |
534 if ext != '.c' and '-Wstrict-prototypes' in compile_options: | |
535 compile_options.remove('-Wstrict-prototypes') | |
536 | |
537 self.Log('\nCompile %s' % src) | |
538 | |
539 out = self.GetObjectName(src) | |
540 | |
541 # The pnacl and nacl-clang toolchains is not able to handle output paths | |
542 # where the PWD + filename is greater than 255, even if the normalised | |
543 # path would be < 255. This change also exists in the pnacl python driver | |
544 # but is duplicated here so we get meaning full error messages from | |
545 # nacl-clang too. | |
546 if pynacl.platform.IsWindows() and (self.is_pnacl_toolchain or | |
547 self.is_nacl_clang): | |
548 full_out = os.path.join(os.getcwd(), out) | |
549 if len(full_out) > 255: | |
550 # Try normalising the full path and see if that brings us under the | |
551 # limit. In this case we will be passing the full path of the .o file | |
552 # to the compiler, which will change the first line of the .d file. | |
553 # However, the .d file is only consumed by build_nexe itself so it | |
554 # should not have any adverse effects. | |
555 out = os.path.normpath(full_out) | |
556 if len(out) > 255: | |
557 raise Error('Output path too long (%s): %s' % (len(out), out)) | |
558 | |
559 outd = os.path.splitext(out)[0] + '.d' | |
560 | |
561 # Don't rebuild unneeded. | |
562 if not self.NeedsRebuild(outd, out, src): | |
563 return out | |
564 | |
565 MakeDir(os.path.dirname(out)) | |
566 self.CleanOutput(out) | |
567 self.CleanOutput(outd) | |
568 cmd_line = [bin_name, '-c', src, '-o', out, | |
569 '-MD', '-MF', outd] + compile_options | |
570 if self.gomacc: | |
571 cmd_line.insert(0, self.gomacc) | |
572 err = self.Run(cmd_line) | |
573 if err: | |
574 self.CleanOutput(outd) | |
575 raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line))) | |
576 else: | |
577 try: | |
578 self.NeedsRebuild(outd, out, src, True) | |
579 except Error as e: | |
580 raise Error('Failed to compile %s to %s with deps %s and cmdline:\t%s' | |
581 '\nNeedsRebuild returned error: %s' % ( | |
582 src, out, outd, ' '.join(cmd_line), e)) | |
583 return out | |
584 | |
585 def RunLink(self, cmd_line, link_out): | |
586 self.CleanOutput(link_out) | |
587 err = self.Run(cmd_line) | |
588 if err: | |
589 raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line))) | |
590 | |
591 def Link(self, srcs): | |
592 """Link these objects with predetermined options and output name.""" | |
593 out = self.LinkOutputName() | |
594 self.Log('\nLink %s' % out) | |
595 bin_name = self.GetCXXCompiler() | |
596 srcs_flags = [] | |
597 if not self.empty: | |
598 srcs_flags += srcs | |
599 srcs_flags += self.link_options | |
600 # Handle an IRT link specially, using a separate script. | |
601 if self.irt_linker: | |
602 if self.tls_edit is None: | |
603 raise Error('Linking the IRT requires tls_edit') | |
604 irt_link_cmd = [sys.executable, self.irt_linker, | |
605 '--output=' + out, | |
606 '--tls-edit=' + self.tls_edit, | |
607 '--link-cmd=' + bin_name, | |
608 '--readelf-cmd=' + self.GetReadElf()] | |
609 if self.commands_are_scripts: | |
610 irt_link_cmd += ['--commands-are-scripts'] | |
611 if self.arch == 'x86-64': | |
612 irt_link_cmd += ['--sandbox-base-hiding-check', | |
613 '--objdump-cmd=' + self.GetObjDump()] | |
614 irt_link_cmd += srcs_flags | |
615 err = self.Run(irt_link_cmd, normalize_slashes=False) | |
616 if err: | |
617 raise Error('FAILED with %d: %s' % (err, ' '.join(irt_link_cmd))) | |
618 return out | |
619 | |
620 MakeDir(os.path.dirname(out)) | |
621 cmd_line = [bin_name, '-o', out, '-Wl,--as-needed'] | |
622 cmd_line += srcs_flags | |
623 | |
624 self.RunLink(cmd_line, out) | |
625 return out | |
626 | |
627 # For now, only support translating a pexe, and not .o file(s) | |
628 def Translate(self, src): | |
629 """Translate a pexe to a nexe.""" | |
630 out = self.TranslateOutputName() | |
631 self.Log('\nTranslate %s' % out) | |
632 bin_name = self.GetBinName('translate') | |
633 cmd_line = [bin_name, '-arch', self.arch, src, '-o', out] | |
634 cmd_line += self.link_options | |
635 | |
636 err = self.Run(cmd_line) | |
637 if err: | |
638 raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line))) | |
639 return out | |
640 | |
641 def ListInvalidObjectsInArchive(self, archive_file, verbose=False): | |
642 """Check the object size from the result of 'ar tv foo.a'. | |
643 | |
644 'ar tv foo.a' shows information like the following: | |
645 rw-r--r-- 0/0 1024 Jan 1 09:00 1970 something1.o | |
646 rw-r--r-- 0/0 12023 Jan 1 09:00 1970 something2.o | |
647 rw-r--r-- 0/0 1124 Jan 1 09:00 1970 something3.o | |
648 | |
649 the third column is the size of object file. We parse it, and verify | |
650 the object size is not 0. | |
651 | |
652 Args: | |
653 archive_file: a path to archive file to be verified. | |
654 verbose: print information if True. | |
655 | |
656 Returns: | |
657 list of 0 byte files. | |
658 """ | |
659 | |
660 cmd_line = [self.GetAr(), 'tv', archive_file] | |
661 output = self.Run(cmd_line, get_output=True) | |
662 | |
663 if verbose: | |
664 print output | |
665 | |
666 result = [] | |
667 for line in output.splitlines(): | |
668 xs = line.split() | |
669 if len(xs) < 3: | |
670 raise Error('Unexpected string: %s' % line) | |
671 | |
672 object_size = xs[2] | |
673 if object_size == '0': | |
674 result.append(xs[-1]) | |
675 | |
676 return result | |
677 | |
678 def Archive(self, srcs, obj_to_src=None): | |
679 """Archive these objects with predetermined options and output name.""" | |
680 out = self.ArchiveOutputName() | |
681 self.Log('\nArchive %s' % out) | |
682 | |
683 needs_verify = False | |
684 if '-r' in self.link_options: | |
685 bin_name = self.GetCXXCompiler() | |
686 cmd_line = [bin_name, '-o', out, '-Wl,--as-needed'] | |
687 if not self.empty: | |
688 cmd_line += srcs | |
689 cmd_line += self.link_options | |
690 else: | |
691 bin_name = self.GetAr() | |
692 cmd_line = [bin_name, '-rc', out] | |
693 if not self.empty: | |
694 cmd_line += srcs | |
695 if self.IsGomaParallelBuild() and pynacl.platform.IsWindows(): | |
696 needs_verify = True | |
697 | |
698 MakeDir(os.path.dirname(out)) | |
699 | |
700 def RunArchive(): | |
701 self.CleanOutput(out) | |
702 err = self.Run(cmd_line) | |
703 if err: | |
704 raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line))) | |
705 | |
706 RunArchive() | |
707 | |
708 # HACK(shinyak): Verifies archive file on Windows if goma is used. | |
709 # When using goma, archive sometimes contains 0 byte object on Windows, | |
710 # though object file itself is not 0 byte on disk. So, we'd like to verify | |
711 # the content of the archive. If the archive contains 0 byte object, | |
712 # we'd like to retry archiving. I'm not sure this fixes the problem, | |
713 # however, this might give us some hints. | |
714 # See also: http://crbug.com/390764 | |
715 if needs_verify: | |
716 ok = False | |
717 for retry in xrange(3): | |
718 invalid_obj_names = self.ListInvalidObjectsInArchive(out) | |
719 if not invalid_obj_names: | |
720 ok = True | |
721 break | |
722 | |
723 print ('WARNING: found 0 byte objects in %s. ' | |
724 'Recompile them without goma (try=%d)' | |
725 % (out, retry + 1)) | |
726 | |
727 time.sleep(1) | |
728 if obj_to_src: | |
729 for invalid_obj_name in invalid_obj_names: | |
730 src = obj_to_src.get(invalid_obj_name) | |
731 | |
732 if not src: | |
733 print ('Couldn\'t find the corresponding src for %s' % | |
734 invalid_obj_name) | |
735 raise Error('ERROR archive is corrupted: %s' % out) | |
736 | |
737 print 'Recompile without goma:', src | |
738 self.gomacc = None | |
739 self.Compile(src) | |
740 | |
741 RunArchive() | |
742 | |
743 if not ok: | |
744 # Show the contents of archive if not ok. | |
745 self.ListInvalidObjectsInArchive(out, verbose=True) | |
746 raise Error('ERROR: archive is corrupted: %s' % out) | |
747 | |
748 return out | |
749 | |
750 def Strip(self, src): | |
751 """Strip the NEXE""" | |
752 self.Log('\nStrip %s' % src) | |
753 | |
754 out = self.StripOutputName() | |
755 pre_debug_tagging = self.UntaggedName() | |
756 self.CleanOutput(out) | |
757 self.CleanOutput(pre_debug_tagging) | |
758 | |
759 # Strip from foo.debug to foo.untagged. | |
760 strip_name = self.GetStrip() | |
761 strip_option = '--strip-all' if self.strip_all else '--strip-debug' | |
762 # pnacl does not have an objcopy so there are no way to embed a link | |
763 if self.is_pnacl_toolchain: | |
764 cmd_line = [strip_name, strip_option, src, '-o', out] | |
765 err = self.Run(cmd_line) | |
766 if err: | |
767 raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line))) | |
768 else: | |
769 cmd_line = [strip_name, strip_option, src, '-o', pre_debug_tagging] | |
770 err = self.Run(cmd_line) | |
771 if err: | |
772 raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line))) | |
773 | |
774 # Tag with a debug link to foo.debug copying from foo.untagged to foo. | |
775 objcopy_name = self.GetObjCopy() | |
776 cmd_line = [objcopy_name, '--add-gnu-debuglink', src, | |
777 pre_debug_tagging, out] | |
778 err = self.Run(cmd_line) | |
779 if err: | |
780 raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line))) | |
781 | |
782 # Drop the untagged intermediate. | |
783 self.CleanOutput(pre_debug_tagging) | |
784 | |
785 return out | |
786 | |
787 def Finalize(self, src): | |
788 """Finalize the PEXE""" | |
789 self.Log('\nFinalize %s' % src) | |
790 | |
791 out = self.StripOutputName() | |
792 self.CleanOutput(out) | |
793 bin_name = self.GetPnaclFinalize() | |
794 cmd_line = [bin_name, src, '-o', out] | |
795 err = self.Run(cmd_line) | |
796 if err: | |
797 raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line))) | |
798 return out | |
799 | |
800 def Generate(self, srcs, obj_to_src=None): | |
801 """Generate final output file. | |
802 | |
803 Link or Archive the final output file, from the compiled sources. | |
804 """ | |
805 if self.outtype in ['nexe', 'pexe', 'nso']: | |
806 out = self.Link(srcs) | |
807 if self.is_pnacl_toolchain and self.finalize_pexe: | |
808 # Note: pnacl-finalize also does stripping. | |
809 self.Finalize(out) | |
810 elif self.strip_all or self.strip_debug: | |
811 self.Strip(out) | |
812 elif self.outtype in ['nlib', 'plib']: | |
813 out = self.Archive(srcs, obj_to_src) | |
814 if self.strip_debug: | |
815 self.Strip(out) | |
816 elif self.strip_all: | |
817 raise Error('FAILED: --strip-all on libs will result in unusable libs.') | |
818 else: | |
819 raise Error('FAILED: Unknown outtype: %s' % (self.outtype)) | |
820 | |
821 | |
822 def UpdateBuildArgs(args, filename): | |
823 new_cmd = json.dumps(args) | |
824 | |
825 try: | |
826 with open(filename, 'r') as fileobj: | |
827 old_cmd = fileobj.read() | |
828 except: | |
829 old_cmd = None | |
830 | |
831 if old_cmd == new_cmd: | |
832 return False | |
833 | |
834 with open(filename, 'w') as fileobj: | |
835 fileobj.write(new_cmd) | |
836 return True | |
837 | |
838 | |
839 def CompileProcess(build, input_queue, output_queue): | |
840 try: | |
841 while True: | |
842 filename = input_queue.get() | |
843 if filename is None: | |
844 return | |
845 output_queue.put((filename, build.Compile(filename))) | |
846 except Exception: | |
847 # Put current exception info to the queue. | |
848 # Since the exception info contains traceback information, it cannot | |
849 # be pickled, so it cannot be sent to the parent process via queue. | |
850 # We stringify the traceback information to send. | |
851 exctype, value = sys.exc_info()[:2] | |
852 traceback_str = "".join(traceback.format_exception(*sys.exc_info())) | |
853 output_queue.put((exctype, value, traceback_str)) | |
854 | |
855 | |
856 def SenderProcess(files, num_processes, input_queue): | |
857 # Push all files into the inputs queue. | |
858 # None is also pushed as a terminator. When worker process read None, | |
859 # the worker process will terminate. | |
860 for filename in files: | |
861 input_queue.put(filename) | |
862 for _ in xrange(num_processes): | |
863 input_queue.put(None) | |
864 | |
865 | |
866 def CheckObjectSize(path): | |
867 # Here, object file should exist. However, we're seeing the case that | |
868 # we cannot read the object file on Windows. | |
869 # When some error happens, we raise an error. However, we'd like to know | |
870 # that the problem is solved or not after some time passed, so we continue | |
871 # checking object file size. | |
872 retry = 0 | |
873 error_messages = [] | |
874 | |
875 path = FixPath(path) | |
876 | |
877 while retry < 5: | |
878 try: | |
879 st = os.stat(path) | |
880 if st.st_size != 0: | |
881 break | |
882 error_messages.append( | |
883 'file size of object %s is 0 (try=%d)' % (path, retry)) | |
884 except Exception as e: | |
885 error_messages.append( | |
886 'failed to stat() for %s (try=%d): %s' % (path, retry, e)) | |
887 | |
888 time.sleep(1) | |
889 retry += 1 | |
890 | |
891 if error_messages: | |
892 raise Error('\n'.join(error_messages)) | |
893 | |
894 def Main(argv): | |
895 parser = OptionParser() | |
896 parser.add_option('--empty', dest='empty', default=False, | |
897 help='Do not pass sources to library.', action='store_true') | |
898 parser.add_option('--no-suffix', dest='suffix', default=True, | |
899 help='Do not append arch suffix.', action='store_false') | |
900 parser.add_option('--strip-debug', dest='strip_debug', default=False, | |
901 help='Strip the NEXE for debugging', action='store_true') | |
902 parser.add_option('--strip-all', dest='strip_all', default=False, | |
903 help='Strip the NEXE for production', action='store_true') | |
904 parser.add_option('--strip', dest='strip', default='', | |
905 help='Strip the filename') | |
906 parser.add_option('--nonstable-pnacl', dest='finalize_pexe', default=True, | |
907 help='Do not finalize pnacl bitcode for ABI stability', | |
908 action='store_false') | |
909 parser.add_option('--source-list', dest='source_list', | |
910 help='Filename to load a source list from') | |
911 parser.add_option('--tls-edit', dest='tls_edit', default=None, | |
912 help='tls_edit location if TLS should be modified for IRT') | |
913 parser.add_option('--irt-linker', dest='irt_linker', default=None, | |
914 help='linker tool to use if linking the IRT') | |
915 parser.add_option('-a', '--arch', dest='arch', | |
916 help='Set target architecture') | |
917 parser.add_option('-c', '--compile', dest='compile_only', default=False, | |
918 help='Compile only.', action='store_true') | |
919 parser.add_option('-i', '--include-dirs', dest='incdirs', | |
920 help='Set include directories.') | |
921 parser.add_option('-l', '--lib-dirs', dest='libdirs', | |
922 help='Set library directories.') | |
923 parser.add_option('-n', '--name', dest='name', | |
924 help='Base path and name of the nexe.') | |
925 parser.add_option('-o', '--objdir', dest='objdir', | |
926 help='Base path of the object output dir.') | |
927 parser.add_option('-r', '--root', dest='root', | |
928 help='Set the root directory of the sources') | |
929 parser.add_option('--product-directory', dest='product_directory', | |
930 help='Set the root directory of the build') | |
931 parser.add_option('-b', '--build', dest='build', | |
932 help='Set build type (<toolchain>_<outtype>, ' + | |
933 'where toolchain is newlib or glibc and outtype is ' + | |
934 'one of nexe, nlib, nso, pexe, or translate)') | |
935 parser.add_option('--compile_flags', dest='compile_flags', | |
936 help='Set compile flags.') | |
937 parser.add_option('--defines', dest='defines', | |
938 help='Set defines') | |
939 parser.add_option('--link_flags', dest='link_flags', | |
940 help='Set link flags.') | |
941 parser.add_option('-v', '--verbose', dest='verbose', default=False, | |
942 help='Enable verbosity', action='store_true') | |
943 parser.add_option('-t', '--toolpath', dest='toolpath', | |
944 help='Set the path for of the toolchains.') | |
945 parser.add_option('--config-name', dest='build_config', | |
946 help='GYP build configuration name (Release/Debug)') | |
947 parser.add_option('--gomadir', dest='gomadir', | |
948 help='Path of the goma directory.') | |
949 options, files = parser.parse_args(argv[1:]) | |
950 | |
951 if options.name is None: | |
952 parser.error('--name is required!') | |
953 if options.build_config is None: | |
954 parser.error('--config-name is required!') | |
955 if options.root is None: | |
956 parser.error('--root is required!') | |
957 if options.arch is None: | |
958 parser.error('--arch is required!') | |
959 if options.build is None: | |
960 parser.error('--build is required!') | |
961 | |
962 if not argv: | |
963 parser.print_help() | |
964 return 1 | |
965 | |
966 # Compare command-line options to last run, and force a rebuild if they | |
967 # have changed. | |
968 options.cmd_file = options.name + '.cmd' | |
969 UpdateBuildArgs(argv, options.cmd_file) | |
970 | |
971 if options.product_directory is None: | |
972 parser.error('--product-dir is required') | |
973 product_dir = options.product_directory | |
974 # Normalize to forward slashes because re.sub interprets backslashes | |
975 # as escape characters. This also simplifies the subsequent regexes. | |
976 product_dir = product_dir.replace('\\', '/') | |
977 # Remove fake child that may be apended to the path. | |
978 # See untrusted.gypi. | |
979 product_dir = re.sub(r'/+xyz$', '', product_dir) | |
980 | |
981 build = None | |
982 try: | |
983 if options.source_list: | |
984 source_list_handle = open(options.source_list, 'r') | |
985 source_list = source_list_handle.read().splitlines() | |
986 source_list_handle.close() | |
987 | |
988 for file_name in source_list: | |
989 file_name = RemoveQuotes(file_name) | |
990 if "$" in file_name: | |
991 # The "make" backend can have an "obj" interpolation variable. | |
992 file_name = re.sub(r'\$!?[({]?obj[)}]?', product_dir + '/obj', | |
993 file_name) | |
994 # Expected patterns: | |
995 # $!PRODUCT_DIR in ninja. | |
996 # $(builddir) in make. | |
997 # $(OutDir) in MSVC. | |
998 # $(BUILT_PRODUCTS_DIR) in xcode. | |
999 # Also strip off and re-add the trailing directory seperator because | |
1000 # different platforms are inconsistent on if it's there or not. | |
1001 # HACK assume only the product directory is the only var left. | |
1002 file_name = re.sub(r'\$!?[({]?\w+[)}]?/?', product_dir + '/', | |
1003 file_name) | |
1004 assert "$" not in file_name, file_name | |
1005 files.append(file_name) | |
1006 | |
1007 # Use set instead of list not to compile the same file twice. | |
1008 # To keep in mind that the order of files may differ from the .gypcmd file, | |
1009 # the set is not converted to a list. | |
1010 # Having duplicated files can cause race condition of compiling during | |
1011 # parallel build using goma. | |
1012 # TODO(sbc): remove the duplication and turn it into an error. | |
1013 files = set(files) | |
1014 | |
1015 # Fix slash style to insulate invoked toolchains. | |
1016 options.toolpath = os.path.normpath(options.toolpath) | |
1017 | |
1018 build = Builder(options) | |
1019 objs = [] | |
1020 | |
1021 if build.outtype == 'translate': | |
1022 # Just translate a pexe to a nexe | |
1023 if len(files) != 1: | |
1024 parser.error('Pexe translation requires exactly one input file.') | |
1025 build.Translate(list(files)[0]) | |
1026 return 0 | |
1027 | |
1028 obj_to_src = {} | |
1029 if build.IsGomaParallelBuild(): | |
1030 inputs = multiprocessing.Queue() | |
1031 returns = multiprocessing.Queue() | |
1032 | |
1033 # Don't limit number of processess in the burst mode. | |
1034 if build.goma_burst: | |
1035 num_processes = len(files) | |
1036 else: | |
1037 num_processes = min(build.goma_processes, len(files)) | |
1038 | |
1039 # Start parallel build. | |
1040 build_processes = [] | |
1041 for _ in xrange(num_processes): | |
1042 process = multiprocessing.Process(target=CompileProcess, | |
1043 args=(build, inputs, returns)) | |
1044 process.start() | |
1045 build_processes.append(process) | |
1046 | |
1047 # Start sender process. We cannot send tasks from here, because | |
1048 # if the input queue is stuck, no one can receive output. | |
1049 sender_process = multiprocessing.Process( | |
1050 target=SenderProcess, | |
1051 args=(files, num_processes, inputs)) | |
1052 sender_process.start() | |
1053 | |
1054 # Wait for results. | |
1055 src_to_obj = {} | |
1056 for _ in files: | |
1057 out = returns.get() | |
1058 # An exception raised in the process may come through the queue. | |
1059 # Raise it again here. | |
1060 if (isinstance(out, tuple) and len(out) == 3 and | |
1061 isinstance(out[1], Exception)): | |
1062 # TODO(shinyak): out[2] contains stringified traceback. It's just | |
1063 # a string, so we cannot pass it to raise. So, we just log it here, | |
1064 # and pass None as traceback. | |
1065 build.Log(out[2]) | |
1066 raise out[0], out[1], None | |
1067 elif out and len(out) == 2: | |
1068 src_to_obj[out[0]] = out[1] | |
1069 # Sometimes out[1] is None. | |
1070 if out[1]: | |
1071 basename = os.path.basename(out[1]) | |
1072 if basename in obj_to_src: | |
1073 raise Error('multiple same name objects detected: %s' % basename) | |
1074 obj_to_src[basename] = out[0] | |
1075 else: | |
1076 raise Error('Unexpected element in CompileProcess output_queue %s' % | |
1077 out) | |
1078 | |
1079 # Keep the input files ordering consistent for link phase to ensure | |
1080 # determinism. | |
1081 for filename in files: | |
1082 # If input file to build.Compile is something it cannot handle, it | |
1083 # returns None. | |
1084 | |
1085 if src_to_obj[filename]: | |
1086 obj_name = src_to_obj[filename] | |
1087 objs.append(obj_name) | |
1088 # TODO(shinyak): In goma environement, it turned out archive file | |
1089 # might contain 0 byte size object, however, the object file itself | |
1090 # is not 0 byte. There might be several possibilities: | |
1091 # (1) archiver failed to read object file. | |
1092 # (2) object file was written after archiver opened it. | |
1093 # I don't know what is happening, however, let me check the object | |
1094 # file size here. | |
1095 CheckObjectSize(obj_name) | |
1096 | |
1097 # Wait until all processes have stopped and verify that there are no more | |
1098 # results. | |
1099 for process in build_processes: | |
1100 process.join() | |
1101 sender_process.join() | |
1102 | |
1103 assert inputs.empty() | |
1104 assert returns.empty() | |
1105 | |
1106 else: # slow path. | |
1107 for filename in files: | |
1108 out = build.Compile(filename) | |
1109 if out: | |
1110 basename = os.path.basename(out) | |
1111 if basename in obj_to_src: | |
1112 raise Error('multiple same name objects detected: %s' % basename) | |
1113 obj_to_src[basename] = out | |
1114 objs.append(out) | |
1115 | |
1116 # Do not link if building an object. However we still want the output file | |
1117 # to be what was specified in options.name | |
1118 if options.compile_only: | |
1119 if len(objs) > 1: | |
1120 raise Error('--compile mode cannot be used with multiple sources') | |
1121 shutil.copy(objs[0], options.name) | |
1122 else: | |
1123 build.Generate(objs, obj_to_src) | |
1124 return 0 | |
1125 except Error as e: | |
1126 sys.stderr.write('%s\n' % e) | |
1127 if build is not None: | |
1128 build.EmitDeferredLog() | |
1129 return 1 | |
1130 except: | |
1131 if build is not None: | |
1132 build.EmitDeferredLog() | |
1133 raise | |
1134 | |
1135 if __name__ == '__main__': | |
1136 sys.exit(Main(sys.argv)) | |
OLD | NEW |