build/build_nexe.py - Issue 2455783004: Remove old tools used by gyp build

Unified Diff: build/build_nexe.py

Issue 2455783004: Remove old tools used by gyp build (Closed)

Patch Set: . Created 4 years, 2 months ago

Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.

Jump to:

View side-by-side diff with in-line comments

Index: build/build_nexe.py

diff --git a/build/build_nexe.py b/build/build_nexe.py

deleted file mode 100644

index e19917a906ef3a3b70341bbc5ce4faa995a68a01..0000000000000000000000000000000000000000

--- a/build/build_nexe.py

+++ /dev/null

@@ -1,1136 +0,0 @@

-#!/usr/bin/python

-# Use of this source code is governed by a BSD-style license that can be

-# found in the LICENSE file.

-"""NEXE building script

-This module will take a set of source files, include paths, library paths, and

-additional arguments, and use them to build.

-"""

-import hashlib

-import json

-import multiprocessing

-from optparse import OptionParser

-import os

-import re

-import Queue

-import shutil

-import StringIO

-import subprocess

-import sys

-import tempfile

-import time

-import traceback

-import urllib2

-from build_nexe_tools import (CommandRunner, Error, FixPath,

- IsFile, MakeDir, RemoveFile)

-sys.path.append(os.path.join(os.path.dirname(__file__), '..'))

-import pynacl.platform

-# When a header file defining NACL_BUILD_SUBARCH is introduced,

-# we can simply remove this map.

-# cf) https://code.google.com/p/chromium/issues/detail?id=440012.

-NACL_BUILD_ARCH_MAP = {

- 'x86-32': ['NACL_BUILD_ARCH=x86', 'NACL_BUILD_SUBARCH=32'],

- 'x86-32-nonsfi': ['NACL_BUILD_ARCH=x86', 'NACL_BUILD_SUBARCH=32'],

- 'x86-64': ['NACL_BUILD_ARCH=x86', 'NACL_BUILD_SUBARCH=64'],

- 'arm': ['NACL_BUILD_ARCH=arm', 'NACL_BUILD_SUBARCH=32'],

- 'arm-nonsfi': ['NACL_BUILD_ARCH=arm', 'NACL_BUILD_SUBARCH=32'],

- 'mips': ['NACL_BUILD_ARCH=mips', 'NACL_BUILD_SUBARCH=32'],

- 'pnacl': ['NACL_BUILD_ARCH=pnacl'],

-def RemoveQuotes(opt):

- if opt and opt[0] == '"':

- assert opt[-1] == '"', opt

- return opt[1:-1].replace('\\"', '"')

- return opt

-def ArgToList(opt):

- outlist = []

- if opt is None:

- return outlist

- optlist = opt.split(' ')

- for optitem in optlist:

- optitem = RemoveQuotes(optitem)

- if optitem:

- outlist.append(optitem)

- return outlist

-def GetMTime(filepath):

- """GetMTime returns the last modification time of the file or None."""

- try:

- return os.path.getmtime(FixPath(filepath))

- except OSError:

- return None

-def IsStale(out_ts, src_ts, rebuilt=False):

- # If either source or output timestamp was not available, assume stale.

- if not out_ts or not src_ts:

- return True

- # If just rebuilt timestamps may be equal due to time granularity.

- if rebuilt:

- return out_ts < src_ts

- # If about to build, be conservative and rebuilt just in case.

- return out_ts <= src_ts

-def IsEnvFlagTrue(flag_name, default=False):

- """Return true when the given flag is true.

- Note:

- Any values that do not match the true pattern are False.

- Args:

- flag_name: a string name of a flag.

- default: default return value if the flag is not set.

- Returns:

- True if the flag is in the true pattern. Otherwise False.

- """

- flag_value = os.environ.get(flag_name)

- if flag_value is None:

- return default

- return bool(re.search(r'^([tTyY]|1:?)', flag_value))

-def GetIntegerEnv(flag_name, default=0):

- """Parses and returns integer environment variable.

- Args:

- flag_name: a string name of a flag.

- default: default return value if the flag is not set.

- Returns:

- Integer value of the flag.

- """

- flag_value = os.environ.get(flag_name)

- if flag_value is None:

- return default

- try:

- return int(flag_value)

- except ValueError:

- raise Error('Invalid ' + flag_name + ': ' + flag_value)

-class Builder(CommandRunner):

- """Builder object maintains options and generates build command-lines.

- The Builder object takes a set of script command-line options, and generates

- a set of paths, and command-line options for the NaCl toolchain.

- """

- def __init__(self, options):

- super(Builder, self).__init__(options)

- arch = options.arch

- self.arch = arch

- build_type = options.build.split('_')

- toolname = build_type[0]

- self.outtype = build_type[1]

- self.osname = pynacl.platform.GetOS()

- # pnacl toolchain can be selected in three different ways

- # 1. by specifying --arch=pnacl directly to generate

- # pexe targets.

- # 2. by specifying --build=newlib_translate to generated

- # nexe via translation

- # 3. by specifying --build=newlib_{nexe,nlib}_pnacl use pnacl

- # toolchain in native mode (e.g. the IRT shim)

- self.is_pnacl_toolchain = False

- if self.outtype == 'translate':

- self.is_pnacl_toolchain = True

- if len(build_type) > 2 and build_type[2] == 'pnacl':

- self.is_pnacl_toolchain = True

- self.is_nacl_clang = len(build_type) > 2 and build_type[2] == 'clang'

- if arch.endswith('-nonsfi'):

- arch = arch[:-len('-nonsfi')]

- if arch in ['x86-32', 'x86-64']:

- mainarch = 'x86'

- self.tool_prefix = 'x86_64-nacl-'

- elif arch == 'arm':

- self.tool_prefix = 'arm-nacl-'

- mainarch = 'arm'

- elif arch == 'mips':

- self.tool_prefix = 'mipsel-nacl-'

- mainarch = 'mipsel'

- elif arch == 'pnacl':

- self.is_pnacl_toolchain = True

- else:

- raise Error('Toolchain architecture %s not supported.' % arch)

- if toolname not in ['newlib', 'glibc']:

- raise Error('Toolchain of type %s not supported.' % toolname)

- if arch == 'mips' and toolname == 'glibc':

- raise Error('mips glibc not supported.')

- if arch == 'pnacl' and toolname == 'glibc':

- raise Error('pnacl glibc not yet supported.')

- if self.is_pnacl_toolchain:

- self.tool_prefix = 'pnacl-'

- tool_subdir = 'pnacl_newlib'

- elif self.is_nacl_clang:

- tool_subdir = 'pnacl_newlib'

- else:

- tool_subdir = 'nacl_%s_%s' % (mainarch, toolname)

- # The pnacl-clang, etc. tools are scripts. Note that for the CommandRunner

- # so that it can know if a shell is needed or not.

- self.SetCommandsAreScripts(self.is_pnacl_toolchain)

- build_arch = pynacl.platform.GetArch()

- tooldir = os.path.join('%s_%s' % (self.osname, build_arch), tool_subdir)

- self.root_path = options.root

- self.nacl_path = os.path.join(self.root_path, 'native_client')

- project_path, project_name = os.path.split(options.name)

- self.outdir = options.objdir

- # Set the toolchain directories

- self.toolchain = os.path.join(options.toolpath, tooldir)

- self.toolbin = os.path.join(self.toolchain, 'bin')

- self.toolstamp = os.path.join(self.toolchain, tool_subdir + '.json')

- if not IsFile(self.toolstamp):

- raise Error('Could not find toolchain prep stamp file: ' + self.toolstamp)

- self.inc_paths = ArgToList(options.incdirs)

- self.lib_paths = ArgToList(options.libdirs)

- self.define_list = ArgToList(options.defines)

- self.name = options.name

- self.cmd_file = options.cmd_file

- self.BuildCompileOptions(

- options.compile_flags, self.define_list, options.arch)

- self.BuildLinkOptions(options.link_flags)

- self.BuildArchiveOptions()

- self.strip = options.strip

- self.empty = options.empty

- self.strip_all = options.strip_all

- self.strip_debug = options.strip_debug

- self.tls_edit = options.tls_edit

- self.finalize_pexe = options.finalize_pexe and arch == 'pnacl'

- goma_config = self.GetGomaConfig(options.gomadir, arch, toolname)

- self.gomacc = goma_config.get('gomacc', '')

- self.goma_burst = goma_config.get('burst', False)

- self.goma_processes = goma_config.get('processes', 1)

- # Define NDEBUG for Release builds.

- if options.build_config.startswith('Release'):

- self.compile_options.append('-DNDEBUG')

- # Use unoptimized native objects for debug IRT builds for faster compiles.

- if (self.is_pnacl_toolchain

- and (self.outtype == 'nlib'

- or self.outtype == 'nexe')

- and self.arch != 'pnacl'):

- if (options.build_config is not None

- and options.build_config.startswith('Debug')):

- self.compile_options.extend(['--pnacl-allow-translate',

- '--pnacl-allow-native',

- '-arch', self.arch])

- self.irt_linker = options.irt_linker

- self.Log('Compile options: %s' % self.compile_options)

- self.Log('Linker options: %s' % self.link_options)

- def IsGomaParallelBuild(self):

- if self.gomacc and (self.goma_burst or self.goma_processes > 1):

- return True

- return False

- def GenNaClPath(self, path):

- """Helper which prepends path with the native client source directory."""

- return os.path.join(self.root_path, 'native_client', path)

- def GetBinName(self, name):

- """Helper which prepends executable with the toolchain bin directory."""

- return os.path.join(self.toolbin, self.tool_prefix + name)

- def GetCCompiler(self):

- """Helper which returns C compiler path."""

- if self.is_pnacl_toolchain or self.is_nacl_clang:

- return self.GetBinName('clang')

- else:

- return self.GetBinName('gcc')

- def GetCXXCompiler(self):

- """Helper which returns C++ compiler path."""

- if self.is_pnacl_toolchain or self.is_nacl_clang:

- return self.GetBinName('clang++')

- else:

- return self.GetBinName('g++')

- def GetAr(self):

- """Helper which returns ar path."""

- return self.GetBinName('ar')

- def GetStrip(self):

- """Helper which returns strip path."""

- return self.GetBinName('strip')

- def GetObjCopy(self):

- """Helper which returns objcopy path."""

- return self.GetBinName('objcopy')

- def GetObjDump(self):

- """Helper which returns objdump path."""

- return self.GetBinName('objdump')

- def GetReadElf(self):

- """Helper which returns readelf path."""

- return self.GetBinName('readelf')

- def GetPnaclFinalize(self):

- """Helper which returns pnacl-finalize path."""

- assert self.is_pnacl_toolchain

- return self.GetBinName('finalize')

- def BuildAssembleOptions(self, options):

- options = ArgToList(options)

- self.assemble_options = options + ['-I' + name for name in self.inc_paths]

- def DebugName(self):

- return self.name + '.debug'

- def UntaggedName(self):

- return self.name + '.untagged'

- def LinkOutputName(self):

- if (self.is_pnacl_toolchain and self.finalize_pexe or

- self.strip_all or self.strip_debug):

- return self.DebugName()

- else:

- return self.name

- def ArchiveOutputName(self):

- if self.strip_debug:

- return self.DebugName()

- else:

- return self.name

- def StripOutputName(self):

- return self.name

- def TranslateOutputName(self):

- return self.name

- def Soname(self):

- return self.name

- def BuildCompileOptions(self, options, define_list, arch):

- """Generates compile options, called once by __init__."""

- options = ArgToList(options)

- # We want to shared gyp 'defines' with other targets, but not

- # ones that are host system dependent. Filtering them out.

- # This really should be better.

- # See: http://code.google.com/p/nativeclient/issues/detail?id=2936

- define_list = [define for define in define_list

- if not (define.startswith('NACL_WINDOWS=') or

- define.startswith('NACL_OSX=') or

- define.startswith('NACL_LINUX=') or

- define.startswith('NACL_ANDROID=') or

- define.startswith('NACL_BUILD_ARCH=') or

- define.startswith('NACL_BUILD_SUBARCH=') or

- define == 'COMPONENT_BUILD' or

- 'WIN32' in define or

- 'WINDOWS' in define or

- 'WINVER' in define)]

- define_list.extend(['NACL_WINDOWS=0',

- 'NACL_OSX=0',

- 'NACL_LINUX=0',

- 'NACL_ANDROID=0'])

- define_list.extend(NACL_BUILD_ARCH_MAP[arch])

- options += ['-D' + define for define in define_list]

- self.compile_options = options + ['-I' + name for name in self.inc_paths]

- def BuildLinkOptions(self, options):

- """Generates link options, called once by __init__."""

- options = ArgToList(options)

- if self.outtype == 'nso':

- options += ['-Wl,-rpath-link,' + name for name in self.lib_paths]

- options += ['-shared']

- options += ['-Wl,-soname,' + os.path.basename(self.Soname())]

- self.link_options = options + ['-L' + name for name in self.lib_paths]

- def BuildArchiveOptions(self):

- """Generates link options, called once by __init__."""

- self.archive_options = []

- def GetObjectName(self, src):

- if self.strip:

- src = src.replace(self.strip, '')

- # Hash the full path of the source file and add 32 bits of that hash onto

- # the end of the object file name. This helps disambiguate files with the

- # same name, because all of the object files are placed into the same

- # directory. Technically, the correct solution would be to preserve the

- # directory structure of the input source files inside the object file

- # directory, but doing that runs the risk of running into filename length

- # issues on Windows.

- h = hashlib.sha1()

- h.update(src)

- wart = h.hexdigest()[:8]

- filename, _ = os.path.splitext(os.path.basename(src))

- return os.path.join(self.outdir, filename + '_' + wart + '.o')

- def FixWindowsPath(self, path):

- # The windows version of the nacl toolchain returns badly

- # formed system header paths. As we do want changes in the

- # toolchain to trigger rebuilds, compensate by detecting

- # malformed paths (starting with /libexec/) and assume these

- # are actually toolchain relative.

- #

- # Additionally, in some cases the toolchains emit cygwin paths

- # which don't work in a win32 python.

- # Assume they are all /cygdrive/ relative and convert to a

- # drive letter.

- cygdrive = '/cygdrive/'

- if path.startswith(cygdrive):

- path = os.path.normpath(

- path[len(cygdrive)] + ':' + path[len(cygdrive)+1:])

- elif path.startswith('/libexec/'):

- path = os.path.normpath(os.path.join(self.toolchain, path[1:]))

- return path

- def GetGomaConfig(self, gomadir, arch, toolname):

- """Returns a goma config dictionary if goma is available or {}."""

- # Goma is enabled only if gomadir is given.

- # We do not use gomacc in GOMA_DIR or PATH anymore.

- if not gomadir:

- return {}

- # Start goma support from os/arch/toolname that have been tested.

- # Set NO_NACL_GOMA=true to force to avoid using goma.

- default_no_nacl_goma = True if pynacl.platform.IsWindows() else False

- if (arch == 'mips'

- or toolname not in ['newlib', 'glibc']

- or IsEnvFlagTrue('NO_NACL_GOMA', default=default_no_nacl_goma)

- or IsEnvFlagTrue('GOMA_DISABLED')):

- return {}

- gomacc_base = 'gomacc.exe' if pynacl.platform.IsWindows() else 'gomacc'

- # TODO(yyanagisawa): not to set gomadir on use_goma=0.

- gomacc = os.path.join(gomadir, gomacc_base)

- if not os.path.exists(gomacc):

- return {}

- goma_config = {

- 'gomacc': gomacc,

- 'burst': IsEnvFlagTrue('NACL_GOMA_BURST'),

- }

- default_processes = 100 if pynacl.platform.IsLinux() else 10

- goma_config['processes'] = GetIntegerEnv('NACL_GOMA_PROCESSES',

- default=default_processes)

- return goma_config

- def NeedsRebuild(self, outd, out, src, rebuilt=False):

- if not IsFile(self.toolstamp):

- if rebuilt:

- raise Error('Could not find toolchain stamp file %s.' % self.toolstamp)

- return True

- if not IsFile(self.cmd_file):

- if rebuilt:

- raise Error('Could not find cmd file %s.' % self.cmd_file)

- return True

- if not IsFile(outd):

- if rebuilt:

- raise Error('Could not find dependency file %s.' % outd)

- return True

- if not IsFile(out):

- if rebuilt:

- raise Error('Could not find output file %s.' % out)

- return True

- inputs = [__file__, self.toolstamp, src, self.cmd_file]

- outputs = [out, outd]

- # Find their timestamps if any.

- input_times = [(GetMTime(f), f) for f in inputs]

- output_times = [(GetMTime(f), f) for f in outputs]

- # All inputs must exist.

- missing_inputs = [p[1] for p in input_times if p[0] is None]

- if missing_inputs:

- raise Error('Missing inputs: %s' % str(missing_inputs))

- # Rebuild if any outputs are missing.

- missing_outputs = [p[1] for p in output_times if p[0] is None]

- if missing_outputs:

- if rebuilt:

- raise Error('Outputs missing after rebuild: %s' % str(missing_outputs))

- return True

- newest_input = max(input_times)

- oldest_output = min(output_times)

- if IsStale(oldest_output[0], newest_input[0], rebuilt):

- if rebuilt:

- raise Error('Output %s is older than toolchain stamp %s' % (

- oldest_output[1], newest_input[1]))

- return True

- # Decode emitted makefile.

- with open(FixPath(outd), 'r') as fh:

- deps = fh.read()

- # Remove line continuations

- deps = deps.replace('\\\n', ' ')

- deps = deps.replace('\n', '')

- # The dependencies are whitespace delimited following the first ':'

- # (that is not part of a windows drive letter)

- deps = deps.split(':', 1)

- if pynacl.platform.IsWindows() and len(deps[0]) == 1:

- # The path has a drive letter, find the next ':'

- deps = deps[1].split(':', 1)[1]

- else:

- deps = deps[1]

- deps = deps.split()

- if pynacl.platform.IsWindows():

- deps = [self.FixWindowsPath(d) for d in deps]

- # Check if any input has changed.

- for filename in deps:

- file_tm = GetMTime(filename)

- if IsStale(oldest_output[0], file_tm, rebuilt):

- if rebuilt:

- raise Error('Dependency %s is older than output %s.' % (

- filename, oldest_output[1]))

- return True

- return False

- def Compile(self, src):

- """Compile the source with pre-determined options."""

- compile_options = self.compile_options[:]

- _, ext = os.path.splitext(src)

- if ext in ['.c', '.S']:

- bin_name = self.GetCCompiler()

- compile_options.append('-std=gnu99')

- if self.is_pnacl_toolchain and ext == '.S':

- compile_options.append('-arch')

- compile_options.append(self.arch)

- elif ext in ['.cc', '.cpp']:

- compile_options.append('-std=gnu++0x')

- compile_options.append('-Wno-deprecated-register')

- bin_name = self.GetCXXCompiler()

- else:

- if ext != '.h':

- self.Log('Skipping unknown type %s for %s.' % (ext, src))

- return None

- # This option is only applicable to C, and C++ compilers warn if

- # it is present, so remove it for C++ to avoid the warning.

- if ext != '.c' and '-Wstrict-prototypes' in compile_options:

- compile_options.remove('-Wstrict-prototypes')

- self.Log('\nCompile %s' % src)

- out = self.GetObjectName(src)

- # The pnacl and nacl-clang toolchains is not able to handle output paths

- # where the PWD + filename is greater than 255, even if the normalised

- # path would be < 255. This change also exists in the pnacl python driver

- # but is duplicated here so we get meaning full error messages from

- # nacl-clang too.

- if pynacl.platform.IsWindows() and (self.is_pnacl_toolchain or

- self.is_nacl_clang):

- full_out = os.path.join(os.getcwd(), out)

- if len(full_out) > 255:

- # Try normalising the full path and see if that brings us under the

- # limit. In this case we will be passing the full path of the .o file

- # to the compiler, which will change the first line of the .d file.

- # However, the .d file is only consumed by build_nexe itself so it

- # should not have any adverse effects.

- out = os.path.normpath(full_out)

- if len(out) > 255:

- raise Error('Output path too long (%s): %s' % (len(out), out))

- outd = os.path.splitext(out)[0] + '.d'

- # Don't rebuild unneeded.

- if not self.NeedsRebuild(outd, out, src):

- return out

- MakeDir(os.path.dirname(out))

- self.CleanOutput(out)

- self.CleanOutput(outd)

- cmd_line = [bin_name, '-c', src, '-o', out,

- '-MD', '-MF', outd] + compile_options

- if self.gomacc:

- cmd_line.insert(0, self.gomacc)

- err = self.Run(cmd_line)

- if err:

- self.CleanOutput(outd)

- raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line)))

- else:

- try:

- self.NeedsRebuild(outd, out, src, True)

- except Error as e:

- raise Error('Failed to compile %s to %s with deps %s and cmdline:\t%s'

- '\nNeedsRebuild returned error: %s' % (

- src, out, outd, ' '.join(cmd_line), e))

- return out

- def RunLink(self, cmd_line, link_out):

- self.CleanOutput(link_out)

- err = self.Run(cmd_line)

- if err:

- raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line)))

- def Link(self, srcs):

- """Link these objects with predetermined options and output name."""

- out = self.LinkOutputName()

- self.Log('\nLink %s' % out)

- bin_name = self.GetCXXCompiler()

- srcs_flags = []

- if not self.empty:

- srcs_flags += srcs

- srcs_flags += self.link_options

- # Handle an IRT link specially, using a separate script.

- if self.irt_linker:

- if self.tls_edit is None:

- raise Error('Linking the IRT requires tls_edit')

- irt_link_cmd = [sys.executable, self.irt_linker,

- '--output=' + out,

- '--tls-edit=' + self.tls_edit,

- '--link-cmd=' + bin_name,

- '--readelf-cmd=' + self.GetReadElf()]

- if self.commands_are_scripts:

- irt_link_cmd += ['--commands-are-scripts']

- if self.arch == 'x86-64':

- irt_link_cmd += ['--sandbox-base-hiding-check',

- '--objdump-cmd=' + self.GetObjDump()]

- irt_link_cmd += srcs_flags

- err = self.Run(irt_link_cmd, normalize_slashes=False)

- if err:

- raise Error('FAILED with %d: %s' % (err, ' '.join(irt_link_cmd)))

- return out

- MakeDir(os.path.dirname(out))

- cmd_line = [bin_name, '-o', out, '-Wl,--as-needed']

- cmd_line += srcs_flags

- self.RunLink(cmd_line, out)

- return out

- # For now, only support translating a pexe, and not .o file(s)

- def Translate(self, src):

- """Translate a pexe to a nexe."""

- out = self.TranslateOutputName()

- self.Log('\nTranslate %s' % out)

- bin_name = self.GetBinName('translate')

- cmd_line = [bin_name, '-arch', self.arch, src, '-o', out]

- cmd_line += self.link_options

- err = self.Run(cmd_line)

- if err:

- raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line)))

- return out

- def ListInvalidObjectsInArchive(self, archive_file, verbose=False):

- """Check the object size from the result of 'ar tv foo.a'.

- 'ar tv foo.a' shows information like the following:

- rw-r--r-- 0/0 1024 Jan 1 09:00 1970 something1.o

- rw-r--r-- 0/0 12023 Jan 1 09:00 1970 something2.o

- rw-r--r-- 0/0 1124 Jan 1 09:00 1970 something3.o

- the third column is the size of object file. We parse it, and verify

- the object size is not 0.

- Args:

- archive_file: a path to archive file to be verified.

- verbose: print information if True.

- Returns:

- list of 0 byte files.

- """

- cmd_line = [self.GetAr(), 'tv', archive_file]

- output = self.Run(cmd_line, get_output=True)

- if verbose:

- print output

- result = []

- for line in output.splitlines():

- xs = line.split()

- if len(xs) < 3:

- raise Error('Unexpected string: %s' % line)

- object_size = xs[2]

- if object_size == '0':

- result.append(xs[-1])

- return result

- def Archive(self, srcs, obj_to_src=None):

- """Archive these objects with predetermined options and output name."""

- out = self.ArchiveOutputName()

- self.Log('\nArchive %s' % out)

- needs_verify = False

- if '-r' in self.link_options:

- bin_name = self.GetCXXCompiler()

- cmd_line = [bin_name, '-o', out, '-Wl,--as-needed']

- if not self.empty:

- cmd_line += srcs

- cmd_line += self.link_options

- else:

- bin_name = self.GetAr()

- cmd_line = [bin_name, '-rc', out]

- if not self.empty:

- cmd_line += srcs

- if self.IsGomaParallelBuild() and pynacl.platform.IsWindows():

- needs_verify = True

- MakeDir(os.path.dirname(out))

- def RunArchive():

- self.CleanOutput(out)

- err = self.Run(cmd_line)

- if err:

- raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line)))

- RunArchive()

- # HACK(shinyak): Verifies archive file on Windows if goma is used.

- # When using goma, archive sometimes contains 0 byte object on Windows,

- # though object file itself is not 0 byte on disk. So, we'd like to verify

- # the content of the archive. If the archive contains 0 byte object,

- # we'd like to retry archiving. I'm not sure this fixes the problem,

- # however, this might give us some hints.

- # See also: http://crbug.com/390764

- if needs_verify:

- ok = False

- for retry in xrange(3):

- invalid_obj_names = self.ListInvalidObjectsInArchive(out)

- if not invalid_obj_names:

- ok = True

- break

- print ('WARNING: found 0 byte objects in %s. '

- 'Recompile them without goma (try=%d)'

- % (out, retry + 1))

- time.sleep(1)

- if obj_to_src:

- for invalid_obj_name in invalid_obj_names:

- src = obj_to_src.get(invalid_obj_name)

- if not src:

- print ('Couldn\'t find the corresponding src for %s' %

- invalid_obj_name)

- raise Error('ERROR archive is corrupted: %s' % out)

- print 'Recompile without goma:', src

- self.gomacc = None

- self.Compile(src)

- RunArchive()

- if not ok:

- # Show the contents of archive if not ok.

- self.ListInvalidObjectsInArchive(out, verbose=True)

- raise Error('ERROR: archive is corrupted: %s' % out)

- return out

- def Strip(self, src):

- """Strip the NEXE"""

- self.Log('\nStrip %s' % src)

- out = self.StripOutputName()

- pre_debug_tagging = self.UntaggedName()

- self.CleanOutput(out)

- self.CleanOutput(pre_debug_tagging)

- # Strip from foo.debug to foo.untagged.

- strip_name = self.GetStrip()

- strip_option = '--strip-all' if self.strip_all else '--strip-debug'

- # pnacl does not have an objcopy so there are no way to embed a link

- if self.is_pnacl_toolchain:

- cmd_line = [strip_name, strip_option, src, '-o', out]

- err = self.Run(cmd_line)

- if err:

- raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line)))

- else:

- cmd_line = [strip_name, strip_option, src, '-o', pre_debug_tagging]

- err = self.Run(cmd_line)

- if err:

- raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line)))

- # Tag with a debug link to foo.debug copying from foo.untagged to foo.

- objcopy_name = self.GetObjCopy()

- cmd_line = [objcopy_name, '--add-gnu-debuglink', src,

- pre_debug_tagging, out]

- err = self.Run(cmd_line)

- if err:

- raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line)))

- # Drop the untagged intermediate.

- self.CleanOutput(pre_debug_tagging)

- return out

- def Finalize(self, src):

- """Finalize the PEXE"""

- self.Log('\nFinalize %s' % src)

- out = self.StripOutputName()

- self.CleanOutput(out)

- bin_name = self.GetPnaclFinalize()

- cmd_line = [bin_name, src, '-o', out]

- err = self.Run(cmd_line)

- if err:

- raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line)))

- return out

- def Generate(self, srcs, obj_to_src=None):

- """Generate final output file.

- Link or Archive the final output file, from the compiled sources.

- """

- if self.outtype in ['nexe', 'pexe', 'nso']:

- out = self.Link(srcs)

- if self.is_pnacl_toolchain and self.finalize_pexe:

- # Note: pnacl-finalize also does stripping.

- self.Finalize(out)

- elif self.strip_all or self.strip_debug:

- self.Strip(out)

- elif self.outtype in ['nlib', 'plib']:

- out = self.Archive(srcs, obj_to_src)

- if self.strip_debug:

- self.Strip(out)

- elif self.strip_all:

- raise Error('FAILED: --strip-all on libs will result in unusable libs.')

- else:

- raise Error('FAILED: Unknown outtype: %s' % (self.outtype))

-def UpdateBuildArgs(args, filename):

- new_cmd = json.dumps(args)

- try:

- with open(filename, 'r') as fileobj:

- old_cmd = fileobj.read()

- except:

- old_cmd = None

- if old_cmd == new_cmd:

- return False

- with open(filename, 'w') as fileobj:

- fileobj.write(new_cmd)

- return True

-def CompileProcess(build, input_queue, output_queue):

- try:

- while True:

- filename = input_queue.get()

- if filename is None:

- return

- output_queue.put((filename, build.Compile(filename)))

- except Exception:

- # Put current exception info to the queue.

- # Since the exception info contains traceback information, it cannot

- # be pickled, so it cannot be sent to the parent process via queue.

- # We stringify the traceback information to send.

- exctype, value = sys.exc_info()[:2]

- traceback_str = "".join(traceback.format_exception(*sys.exc_info()))

- output_queue.put((exctype, value, traceback_str))

-def SenderProcess(files, num_processes, input_queue):

- # Push all files into the inputs queue.

- # None is also pushed as a terminator. When worker process read None,

- # the worker process will terminate.

- for filename in files:

- input_queue.put(filename)

- for _ in xrange(num_processes):

- input_queue.put(None)

-def CheckObjectSize(path):

- # Here, object file should exist. However, we're seeing the case that

- # we cannot read the object file on Windows.

- # When some error happens, we raise an error. However, we'd like to know

- # that the problem is solved or not after some time passed, so we continue

- # checking object file size.

- retry = 0

- error_messages = []

- path = FixPath(path)

- while retry < 5:

- try:

- st = os.stat(path)

- if st.st_size != 0:

- break

- error_messages.append(

- 'file size of object %s is 0 (try=%d)' % (path, retry))

- except Exception as e:

- error_messages.append(

- 'failed to stat() for %s (try=%d): %s' % (path, retry, e))

- time.sleep(1)

- retry += 1

- if error_messages:

- raise Error('\n'.join(error_messages))

-def Main(argv):

- parser = OptionParser()

- parser.add_option('--empty', dest='empty', default=False,

- help='Do not pass sources to library.', action='store_true')

- parser.add_option('--no-suffix', dest='suffix', default=True,

- help='Do not append arch suffix.', action='store_false')

- parser.add_option('--strip-debug', dest='strip_debug', default=False,

- help='Strip the NEXE for debugging', action='store_true')

- parser.add_option('--strip-all', dest='strip_all', default=False,

- help='Strip the NEXE for production', action='store_true')

- parser.add_option('--strip', dest='strip', default='',

- help='Strip the filename')

- parser.add_option('--nonstable-pnacl', dest='finalize_pexe', default=True,

- help='Do not finalize pnacl bitcode for ABI stability',

- action='store_false')

- parser.add_option('--source-list', dest='source_list',

- help='Filename to load a source list from')

- parser.add_option('--tls-edit', dest='tls_edit', default=None,

- help='tls_edit location if TLS should be modified for IRT')

- parser.add_option('--irt-linker', dest='irt_linker', default=None,

- help='linker tool to use if linking the IRT')

- parser.add_option('-a', '--arch', dest='arch',

- help='Set target architecture')

- parser.add_option('-c', '--compile', dest='compile_only', default=False,

- help='Compile only.', action='store_true')

- parser.add_option('-i', '--include-dirs', dest='incdirs',

- help='Set include directories.')

- parser.add_option('-l', '--lib-dirs', dest='libdirs',

- help='Set library directories.')

- parser.add_option('-n', '--name', dest='name',

- help='Base path and name of the nexe.')

- parser.add_option('-o', '--objdir', dest='objdir',

- help='Base path of the object output dir.')

- parser.add_option('-r', '--root', dest='root',

- help='Set the root directory of the sources')

- parser.add_option('--product-directory', dest='product_directory',

- help='Set the root directory of the build')

- parser.add_option('-b', '--build', dest='build',

- help='Set build type (<toolchain>_<outtype>, ' +

- 'where toolchain is newlib or glibc and outtype is ' +

- 'one of nexe, nlib, nso, pexe, or translate)')

- parser.add_option('--compile_flags', dest='compile_flags',

- help='Set compile flags.')

- parser.add_option('--defines', dest='defines',

- help='Set defines')

- parser.add_option('--link_flags', dest='link_flags',

- help='Set link flags.')

- parser.add_option('-v', '--verbose', dest='verbose', default=False,

- help='Enable verbosity', action='store_true')

- parser.add_option('-t', '--toolpath', dest='toolpath',

- help='Set the path for of the toolchains.')

- parser.add_option('--config-name', dest='build_config',

- help='GYP build configuration name (Release/Debug)')

- parser.add_option('--gomadir', dest='gomadir',

- help='Path of the goma directory.')

- options, files = parser.parse_args(argv[1:])

- if options.name is None:

- parser.error('--name is required!')

- if options.build_config is None:

- parser.error('--config-name is required!')

- if options.root is None:

- parser.error('--root is required!')

- if options.arch is None:

- parser.error('--arch is required!')

- if options.build is None:

- parser.error('--build is required!')

- if not argv:

- parser.print_help()

- return 1

- # Compare command-line options to last run, and force a rebuild if they

- # have changed.

- options.cmd_file = options.name + '.cmd'

- UpdateBuildArgs(argv, options.cmd_file)

- if options.product_directory is None:

- parser.error('--product-dir is required')

- product_dir = options.product_directory

- # Normalize to forward slashes because re.sub interprets backslashes

- # as escape characters. This also simplifies the subsequent regexes.

- product_dir = product_dir.replace('\\', '/')

- # Remove fake child that may be apended to the path.

- # See untrusted.gypi.

- product_dir = re.sub(r'/+xyz$', '', product_dir)

- build = None

- try:

- if options.source_list:

- source_list_handle = open(options.source_list, 'r')

- source_list = source_list_handle.read().splitlines()

- source_list_handle.close()

- for file_name in source_list:

- file_name = RemoveQuotes(file_name)

- if "$" in file_name:

- # The "make" backend can have an "obj" interpolation variable.

- file_name = re.sub(r'\$!?[({]?obj[)}]?', product_dir + '/obj',

- file_name)

- # Expected patterns:

- # $!PRODUCT_DIR in ninja.

- # $(builddir) in make.

- # $(OutDir) in MSVC.

- # $(BUILT_PRODUCTS_DIR) in xcode.

- # Also strip off and re-add the trailing directory seperator because

- # different platforms are inconsistent on if it's there or not.

- # HACK assume only the product directory is the only var left.

- file_name = re.sub(r'\$!?[({]?\w+[)}]?/?', product_dir + '/',

- file_name)

- assert "$" not in file_name, file_name

- files.append(file_name)

- # Use set instead of list not to compile the same file twice.

- # To keep in mind that the order of files may differ from the .gypcmd file,

- # the set is not converted to a list.

- # Having duplicated files can cause race condition of compiling during

- # parallel build using goma.

- # TODO(sbc): remove the duplication and turn it into an error.

- files = set(files)

- # Fix slash style to insulate invoked toolchains.

- options.toolpath = os.path.normpath(options.toolpath)

- build = Builder(options)

- objs = []

- if build.outtype == 'translate':

- # Just translate a pexe to a nexe

- if len(files) != 1:

- parser.error('Pexe translation requires exactly one input file.')

- build.Translate(list(files)[0])

- return 0

- obj_to_src = {}

- if build.IsGomaParallelBuild():

- inputs = multiprocessing.Queue()

- returns = multiprocessing.Queue()

- # Don't limit number of processess in the burst mode.

- if build.goma_burst:

- num_processes = len(files)

- else:

- num_processes = min(build.goma_processes, len(files))

- # Start parallel build.

- build_processes = []

- for _ in xrange(num_processes):

- process = multiprocessing.Process(target=CompileProcess,

- args=(build, inputs, returns))

- process.start()

- build_processes.append(process)

- # Start sender process. We cannot send tasks from here, because

- # if the input queue is stuck, no one can receive output.

- sender_process = multiprocessing.Process(

- target=SenderProcess,

- args=(files, num_processes, inputs))

- sender_process.start()

- # Wait for results.

- src_to_obj = {}

- for _ in files:

- out = returns.get()

- # An exception raised in the process may come through the queue.

- # Raise it again here.

- if (isinstance(out, tuple) and len(out) == 3 and

- isinstance(out[1], Exception)):

- # TODO(shinyak): out[2] contains stringified traceback. It's just

- # a string, so we cannot pass it to raise. So, we just log it here,

- # and pass None as traceback.

- build.Log(out[2])

- raise out[0], out[1], None

- elif out and len(out) == 2:

- src_to_obj[out[0]] = out[1]

- # Sometimes out[1] is None.

- if out[1]:

- basename = os.path.basename(out[1])

- if basename in obj_to_src:

- raise Error('multiple same name objects detected: %s' % basename)

- obj_to_src[basename] = out[0]

- else:

- raise Error('Unexpected element in CompileProcess output_queue %s' %

- out)

- # Keep the input files ordering consistent for link phase to ensure

- # determinism.

- for filename in files:

- # If input file to build.Compile is something it cannot handle, it

- # returns None.

- if src_to_obj[filename]:

- obj_name = src_to_obj[filename]

- objs.append(obj_name)

- # TODO(shinyak): In goma environement, it turned out archive file

- # might contain 0 byte size object, however, the object file itself

- # is not 0 byte. There might be several possibilities:

- # (1) archiver failed to read object file.

- # (2) object file was written after archiver opened it.

- # I don't know what is happening, however, let me check the object

- # file size here.

- CheckObjectSize(obj_name)

- # Wait until all processes have stopped and verify that there are no more

- # results.

- for process in build_processes:

- process.join()

- sender_process.join()

- assert inputs.empty()

- assert returns.empty()

- else: # slow path.

- for filename in files:

- out = build.Compile(filename)

- if out:

- basename = os.path.basename(out)

- if basename in obj_to_src:

- raise Error('multiple same name objects detected: %s' % basename)

- obj_to_src[basename] = out

- objs.append(out)

- # Do not link if building an object. However we still want the output file

- # to be what was specified in options.name

- if options.compile_only:

- if len(objs) > 1:

- raise Error('--compile mode cannot be used with multiple sources')

- shutil.copy(objs[0], options.name)

- else:

- build.Generate(objs, obj_to_src)

- return 0

- except Error as e:

- sys.stderr.write('%s\n' % e)

- if build is not None:

- build.EmitDeferredLog()

- return 1

- except:

- if build is not None:

- build.EmitDeferredLog()

- raise

-if __name__ == '__main__':

- sys.exit(Main(sys.argv))

« no previous file with comments | « no previous file | build/compiler_version.py » ('j') | no next file with comments »