Fix PNaCl toolchain build on Mac

toolchain_build/toolchain_build_pnacl.py

 #!/usr/bin/python
 # Copyright (c) 2013 The Native Client Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Recipes for PNaCl toolchain packages.
 
    Recipes consist of specially-structured dictionaries, with keys for package
    name, type, commands to execute, etc. The structure is documented in the
    PackageBuilder docstring in toolchain_main.py.
 
    The real entry plumbing and CLI flags are also in toolchain_main.py.
 """
 
 import fnmatch
 import logging
 import os
 import shutil
+import subprocess
 import sys
 import zipfile
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 import pynacl.file_tools
 import pynacl.gsd_storage
 import pynacl.platform
 import pynacl.repo_tools
 
 import command
@@ skipping 57 matching lines @@
 # Path to the mingw cross-compiler libs on Ubuntu
 CROSS_MINGW_LIBPATH = '/usr/lib/gcc/i686-w64-mingw32/4.6'
 # Path and version of the native mingw compiler to be installed on Windows hosts
 MINGW_PATH = os.path.join(NACL_DIR, 'mingw32')
 MINGW_VERSION = 'i686-w64-mingw32-4.8.1'
 
 CHROME_CLANG = os.path.join(os.path.dirname(NACL_DIR), 'third_party',
                             'llvm-build', 'Release+Asserts', 'bin', 'clang')
 CHROME_CLANGXX = CHROME_CLANG + '++'
 
+# Required SDK version and target version for Mac builds.
+# See MAC_SDK_FLAGS, below.
+MAC_SDK_MIN = '10.10'
+MAC_DEPLOYMENT_TARGET = '10.6'
+
 # Redirectors are small shims acting like sym links with optional arguments.
 # For mac/linux we simply use a shell script which create small redirector
 # shell scripts. For windows we compile an executable which redirects to
 # the target using a compiled in table.
 REDIRECTOR_SCRIPT = os.path.join(NACL_TOOLS_DIR, 'create_redirector.sh')
 REDIRECTOR_WIN32_SRC = os.path.join(NACL_TOOLS_DIR, 'redirector')
 
 TOOL_X64_I686_REDIRECTS = [
     #Toolname, Tool Args
     ('as',     '--32'),
@@ skipping 37 matching lines @@
 
 def ProgramPath(program):
   """Returns the path for the given program, or None if it doesn't exist."""
   try:
     return pynacl.file_tools.Which(program)
   except pynacl.file_tools.ExecutableNotFound:
     pass
   return None
 
 
+# Determine the extra compiler flags necessary for Mac.  Do this once at
+# top level, rather than every time in CompilersForHost, because running
+# the external script is costly.
+def MacSdkFlags():
+  if not pynacl.platform.IsMac():
+    return []
+  mac_sdk_sysroot, mac_sdk_version = subprocess.check_output([
+      sys.executable,
+      os.path.join(os.path.dirname(NACL_DIR), 'build', 'mac', 'find_sdk.py'),
+      '--print_sdk_path',
+      MAC_SDK_MIN,
+      ]).splitlines()
+  return ['-isysroot', mac_sdk_sysroot,
+          '-mmacosx-version-min=' + MAC_DEPLOYMENT_TARGET]
+
+MAC_SDK_FLAGS = MacSdkFlags()
+
+
 def InputsForCommands(commands):
   """Returns a dict of extra 'inputs' items for command names.
 Each command name that is an absolute path gets an item named for
 its basename.  The logic here is that commands named by absolute
 paths do not come from the system installation and hence those
 binaries themselves should be considered inputs for memoization.
 """
   inputs = {}
   for command in commands:
     if os.path.isabs(command):
@@ skipping 59 matching lines @@
   joined_name = GSDJoin(component_name, host)
   if HostIsDebug(options):
     joined_name= joined_name + '_debug'
   return joined_name
 
 
 def HostArchToolFlags(host, extra_cflags, opts):
   """Return the appropriate CFLAGS, CXXFLAGS, and LDFLAGS based on host
   and opts. Does not attempt to determine flags that are attached
   to CC and CXX directly.
+
+  Returns the tuple (flags, deps) where 'flags' is a dictionary mapping
+  'CFLAGS' et al to a list of arguments, and 'deps' is a list of extra
+  dependencies for a component using these flags.
   """
   extra_cc_flags = list(extra_cflags)
   result = { 'LDFLAGS' : [],
              'CFLAGS' : [],
              'CXXFLAGS' : []}
+  deps = []
   if TripleIsWindows(host):
     result['LDFLAGS'] += ['-L%(abs_libdl)s', '-ldl']
     result['CFLAGS'] += ['-isystem','%(abs_libdl)s']
     result['CXXFLAGS'] += ['-isystem', '%(abs_libdl)s']
+    deps.append('libdl')
   else:
     if TripleIsLinux(host) and not TripleIsX8664(host):
       # Chrome clang defaults to 64-bit builds, even when run on 32-bit Linux.
       extra_cc_flags += ['-m32']
-    elif TripleIsMac(host):
-      # This is required for building with recent libc++ against OSX 10.6
-      extra_cc_flags += ['-U__STRICT_ANSI__']
     if opts.gcc or host == 'le32-nacl':
       result['CFLAGS']  += extra_cc_flags
       result['CXXFLAGS'] += extra_cc_flags
     else:
       result['CFLAGS'] += extra_cc_flags
       result['LDFLAGS'] += ['-L%(' + FlavoredName('abs_libcxx',
                                                   host, opts) + ')s/lib']
       result['CXXFLAGS'] += ([
         '-stdlib=libc++',
         '-I%(' + FlavoredName('abs_libcxx', host, opts) + ')s/include/c++/v1'] +
         extra_cc_flags)
-  return result
+      deps.append(FlavoredName('libcxx', host, opts))
+  return result, deps
 
 
 def ConfigureHostArchFlags(host, extra_cflags, options, extra_configure=None,
                            use_afl_fuzz=False):
   """Return flags passed to LLVM and binutils configure for compilers and
   compile flags.
 
-  Returns the tuple (flags, inputs) where 'flags' is a list of arguments to
-  configure and 'inputs' is a dict of extra inputs to be hashed.
+  Returns the tuple (flags, inputs, deps) where 'flags' is a list of
+  arguments to configure, 'inputs' is a dict of extra inputs to be hashed,
+  and 'deps' is a list of extra dependencies for a component using these flags.
   """
   configure_args = []
   extra_cc_args = []
 
   configure_args += options.extra_configure_args
   if extra_configure is not None:
     configure_args += extra_configure
   if options.extra_cc_args is not None:
     extra_cc_args += [options.extra_cc_args]
 
@@ skipping 28 matching lines @@
       # ccache/clang++ interaction.  Specifically, errors about
       # "argument unused during compilation".
       os.environ['CCACHE_CPP2'] = 'yes'
       cc_list = ['ccache', cc]
       cxx_list = ['ccache', cxx]
       extra_cc_args += ['-Qunused-arguments']
       extra_cxx_args += ['-Qunused-arguments']
     else:
       cc_list = [cc]
       cxx_list = [cxx]
+
+    if TripleIsMac(host):
+      cc_list += MAC_SDK_FLAGS
+      cxx_list += MAC_SDK_FLAGS
+
     configure_args.append('CC=' + ' '.join(cc_list + extra_cc_args))
     configure_args.append('CXX=' + ' '.join(cxx_list + extra_cxx_args))
     configure_args.append('AR=' + ar)
     configure_args.append('RANLIB=' + ranlib)
 
-  tool_flags = HostArchToolFlags(host, extra_cflags, options)
+  tool_flags, tool_deps = HostArchToolFlags(host, extra_cflags, options)
   configure_args.extend(
        ['CFLAGS=' + ' '.join(tool_flags['CFLAGS']),
         'CXXFLAGS=' + ' '.join(tool_flags['CXXFLAGS']),
         'LDFLAGS=' + ' '.join(tool_flags['LDFLAGS']),
        ])
   if TripleIsWindows(host):
     # The i18n support brings in runtime dependencies on MinGW DLLs
     # that we don't want to have to distribute alongside our binaries.
     # So just disable it, and compiler messages will always be in US English.
     configure_args.append('--disable-nls')
     if is_cross:
       # LLVM's linux->mingw cross build needs this
       configure_args.append('CC_FOR_BUILD=gcc')
-  return configure_args, InputsForCommands(hashables)
+  return configure_args, InputsForCommands(hashables), tool_deps
 
 
 def LibCxxHostArchFlags(host):
   cc, cxx, _, _ = CompilersForHost(host)
   hashables = [cc, cxx]
   cmake_flags = []
   cmake_flags.extend(['-DCMAKE_C_COMPILER='+cc, '-DCMAKE_CXX_COMPILER='+cxx])
   if TripleIsLinux(host) and not TripleIsX8664(host):
     # Chrome clang defaults to 64-bit builds, even when run on 32-bit Linux
     cmake_flags.extend(['-DCMAKE_C_FLAGS=-m32',
                         '-DCMAKE_CXX_FLAGS=-m32'])
+  elif TripleIsMac(host):
+    sdk_flags = ' '.join(MAC_SDK_FLAGS)
+    cmake_flags.extend(['-DCMAKE_C_FLAGS=' + sdk_flags,
+                        '-DCMAKE_CXX_FLAGS=' + sdk_flags])
   return cmake_flags, InputsForCommands(hashables)
 
 
 def CmakeHostArchFlags(host, options):
   """Set flags passed to LLVM cmake for compilers and compile flags.
 
-  Returns the tuple (flags, inputs) where 'flags' is a list of arguments to
-  cmake and 'inputs' is a dict of extra inputs to be hashed.
+  Returns the tuple (flags, inputs, deps) where 'flags' is a list of
+  arguments to cmake, 'inputs' is a dict of extra inputs to be hashed,
+  and 'deps' is a list of extra dependencies for a component using these flags.
   """
   cmake_flags = []
   if options.afl_fuzz_dir:
     cc, cxx = AflFuzzCompilers(options.afl_fuzz_dir)
   else:
     cc, cxx, _, _ = CompilersForHost(host)
   hashables = [cc, cxx]
 
   cmake_flags.extend(['-DCMAKE_C_COMPILER='+cc, '-DCMAKE_CXX_COMPILER='+cxx])
   if ProgramPath('ccache'):
     cmake_flags.extend(['-DSYSTEM_HAS_CCACHE=ON'])
 
   # There seems to be a bug in chrome clang where it exposes the msan interface
   # (even when compiling without msan) but then does not link with an
   # msan-enabled compiler_rt, leaving references to __msan_allocated_memory
   # undefined.
   cmake_flags.append('-DHAVE_SANITIZER_MSAN_INTERFACE_H=FALSE')
-  tool_flags = HostArchToolFlags(host, [], options)
-  cmake_flags.extend(['-DCMAKE_C_FLAGS=' + ' '.join(tool_flags['CFLAGS'])])
-  cmake_flags.extend(['-DCMAKE_CXX_FLAGS=' + ' '.join(tool_flags['CXXFLAGS'])])
+  tool_flags, tool_deps = HostArchToolFlags(host, [], options)
+  cflags = tool_flags['CFLAGS']
+  cxxflags = tool_flags['CXXFLAGS']
+  if TripleIsMac(host):
+    cflags = MAC_SDK_FLAGS + cflags
+    cxxflags = MAC_SDK_FLAGS + cxxflags
+  cmake_flags.append('-DCMAKE_C_FLAGS=' + ' '.join(cflags))
+  cmake_flags.append('-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags))
   for linker_type in ['EXE', 'SHARED', 'MODULE']:
     cmake_flags.extend([('-DCMAKE_%s_LINKER_FLAGS=' % linker_type) +
                         ' '.join(tool_flags['LDFLAGS'])])
-  return cmake_flags, InputsForCommands(hashables)
+  return cmake_flags, InputsForCommands(hashables), tool_deps
 
 
-def ConfigureBinutilsCommon():
-  return ['--with-pkgversion=' + PACKAGE_NAME,
-          '--with-bugurl=' + BUG_URL,
-          '--without-zlib',
-          '--prefix=',
-          '--disable-silent-rules',
-          '--enable-deterministic-archives',
-         ]
+def ConfigureBinutilsCommon(host, options, is_pnacl):
+  # Binutils still has some warnings when building with clang
+  if not options.gcc:
+    warning_flags = ['-Wno-extended-offsetof', '-Wno-absolute-value',
+                    '-Wno-unused-function', '-Wno-unused-const-variable',
+                    '-Wno-unneeded-internal-declaration',
+                    '-Wno-unused-private-field', '-Wno-format-security']
+  else:
+    warning_flags = ['-Wno-unused-function', '-Wno-unused-value']
+
+  host_arch_flags, inputs, deps = ConfigureHostArchFlags(
+      host, warning_flags, options,
+      options.binutils_pnacl_extra_configure if is_pnacl else None)
+
+  flags = [
+      '--with-pkgversion=' + PACKAGE_NAME,
+      '--with-bugurl=' + BUG_URL,
+      '--without-zlib',
+      '--prefix=',
+      '--disable-silent-rules',
+      '--enable-deterministic-archives',
+      ] + host_arch_flags
+
+  return flags, inputs, deps
 
 def LLVMConfigureAssertionsFlags(options):
   if options.enable_llvm_assertions:
     return []
   else:
     return ['--disable-debug', '--disable-assertions']
 
 
 def MakeCommand(host):
   make_command = ['make']
@@ skipping 215 matching lines @@
   # TODO(mcgrathr): With post-3.7 Clang, binutils (opcodes) build gets:
   #     error: shifting a negative signed value is undefined
   #            [-Werror,-Wshift-negative-value]
   binutils_do_werror = False
   extra_gold_deps = []
   if host == 'le32-nacl':
     # TODO(bradnelson): Fix warnings so this can go away.
     binutils_do_werror = False
     extra_gold_deps = [H('llvm')]
 
-  # Binutils still has some warnings when building with clang
-  if not options.gcc:
-    warning_flags = ['-Wno-extended-offsetof', '-Wno-absolute-value',
-                    '-Wno-unused-function', '-Wno-unused-const-variable',
-                    '-Wno-unneeded-internal-declaration',
-                    '-Wno-unused-private-field', '-Wno-format-security']
-  else:
-    warning_flags = ['-Wno-unused-function', '-Wno-unused-value']
-
   # The binutils git checkout includes all the directories in the
   # upstream binutils-gdb.git repository, but some of these
   # directories are not included in a binutils release tarball.  The
   # top-level Makefile will try to build whichever of the whole set
   # exist, but we don't want these extra directories built.  So we
   # stub them out by creating dummy <subdir>/Makefile files; having
   # these exist before the configure-<subdir> target in the
   # top-level Makefile runs prevents it from doing anything.
   binutils_dummy_dirs = ['gdb', 'libdecnumber', 'readline', 'sim']
   def DummyDirCommands(dirs):
     dummy_makefile = """\
 .DEFAULT:;@echo Ignoring $@
 """
     commands = []
     for dir in dirs:
       commands.append(command.Mkdir(dir))
       commands.append(command.WriteData(
         dummy_makefile, command.path.join(dir, 'Makefile')))
     return commands
 
-  binutils_host_arch_flags, binutils_inputs = ConfigureHostArchFlags(
-      host, warning_flags, options, options.binutils_pnacl_extra_configure)
-  binutils_inputs.update({'macros': os.path.join(
-      NACL_DIR, 'pnacl', 'support', 'clang_direct', 'nacl-arm-macros.s')})
+  binutils_flags, binutils_inputs, binutils_deps = ConfigureBinutilsCommon(
+      host, options, True)
+  binutils_inputs['macros'] = os.path.join(
+      NACL_DIR, 'pnacl', 'support', 'clang_direct', 'nacl-arm-macros.s')
   tools = {
       # The binutils_pnacl package is used both for bitcode linking (gold) and
       # for its conventional use with arm-nacl-clang.
       H('binutils_pnacl'): {
-          'dependencies': ['binutils_pnacl_src'] + extra_gold_deps,
+          'dependencies': (['binutils_pnacl_src'] +
+                           extra_gold_deps + binutils_deps),
           'type': 'build',
           'inputs' : binutils_inputs,
           'commands': [
               command.SkipForIncrementalCommand([
                   'sh',
-                  '%(binutils_pnacl_src)s/configure'] +
-                  ConfigureBinutilsCommon() + binutils_host_arch_flags +
+                  '%(binutils_pnacl_src)s/configure'] + binutils_flags +
                   [
                   '--enable-gold=yes',
                   '--enable-plugins',
                   '--enable-shared=no',
                   '--enable-targets=arm-nacl,i686-nacl,x86_64-nacl,mipsel-nacl',
                   '--enable-werror=' + ('yes' if binutils_do_werror else 'no'),
                   '--program-prefix=le32-nacl-',
                   '--target=arm-nacl',
                   '--with-sysroot=/le32-nacl',
                   '--without-gas'
@@ skipping 59 matching lines @@
   # TODO(mcgrathr): The latest Clang gets -Wredunant-move errors under
   # -Werror, and llvm's configure makes it difficult to get -Wno-... options
   # in the place they need to be, so just disable -Werror for now.
   # After the next LLVM merge, the LLVM sources will probably be compatible
   # with the latest Clang and -Werror.
   llvm_do_werror = False and not (TripleIsWindows(host) or options.gcc)
 
   # Older CMake ignore CMAKE_*_LINKER_FLAGS during config step.
   # https://public.kitware.com/Bug/view.php?id=14066
   # The workaround is to set LDFLAGS in the environment.
-  llvm_cmake_config_env = {'LDFLAGS': ' '.join(
-      HostArchToolFlags(host, [], options)['LDFLAGS'])}
+  tool_flags, tool_deps = HostArchToolFlags(host, [], options)
+  llvm_cmake_config_env = {'LDFLAGS': ' '.join(tool_flags['LDFLAGS'])}
   llvm_cmake_config_env.update(AflFuzzEnvMap(host, options))
 
-  llvm_host_arch_flags, llvm_inputs = CmakeHostArchFlags(host, options)
-  llvm_inputs.update({'test_xfails': os.path.join(NACL_DIR,
-                                                  'pnacl', 'scripts')})
+  llvm_host_arch_flags, llvm_inputs, llvm_deps = CmakeHostArchFlags(
+      host, options)
+  llvm_deps = list(set(tool_deps + llvm_deps))
+  llvm_inputs['test_xfails'] = os.path.join(NACL_DIR, 'pnacl', 'scripts')
   llvm_cmake = {
       H('llvm'): {
           'dependencies': ['clang_src', 'llvm_src', 'binutils_pnacl_src',
-                           'subzero_src'],
+                           'subzero_src'] + llvm_deps,
           'inputs': llvm_inputs,
           'type': 'build',
           'commands': [
               command.SkipForIncrementalCommand([
                   'cmake', '-G', 'Ninja'] +
                   llvm_host_arch_flags + asan_flags +
                   [
                   '-DBUILD_SHARED_LIBS=ON',
                   '-DCMAKE_BUILD_TYPE=' + ('Debug' if HostIsDebug(options)
                                            else 'Release'),
@@ skipping 21 matching lines @@
       },
   }
   cleanup_static_libs = []
   shared = []
   if host != 'le32-nacl':
     shared = ['--enable-shared']
     cleanup_static_libs = [
         command.Remove(*[os.path.join('%(output)s', 'lib', f) for f
                          in '*.a', '*Hello.*', 'BugpointPasses.*']),
     ]
-  llvm_host_arch_flags, llvm_inputs = ConfigureHostArchFlags(
+  llvm_host_arch_flags, llvm_inputs, llvm_deps = ConfigureHostArchFlags(
       host, [], options, use_afl_fuzz=options.afl_fuzz_dir)
   llvm_inputs.update({'test_xfails': os.path.join(NACL_DIR,
                                                   'pnacl', 'scripts')})
   llvm_autoconf = {
       H('llvm'): {
           'dependencies': ['clang_src', 'llvm_src', 'binutils_pnacl_src',
-                           'subzero_src'],
+                           'subzero_src'] + llvm_deps,
           'inputs': llvm_inputs,
           'type': 'build',
           'commands': [
               command.SkipForIncrementalCommand([
                   'sh',
                   '%(llvm_src)s/configure'] +
                   AflFuzzEnvList(host, options) +
                   llvm_host_arch_flags +
                   LLVMConfigureAssertionsFlags(options) +
                   [
@@ skipping 37 matching lines @@
                                Exe('c-index-test'), Exe('clang-tblgen'),
                                Exe('llvm-tblgen')])] +
               CreateSymLinksToDirectToNaClTools(host) +
               CopyWindowsHostLibs(host),
       },
   }
   if options.cmake:
     tools.update(llvm_cmake)
   else:
     tools.update(llvm_autoconf)
-  if TripleIsWindows(host):
-    tools[H('binutils_pnacl')]['dependencies'].append('libdl')
-    tools[H('llvm')]['dependencies'].append('libdl')
-  elif not options.gcc and host != 'le32-nacl':
-    tools[H('binutils_pnacl')]['dependencies'].append(H('libcxx'))
-    tools[H('llvm')]['dependencies'].append(H('libcxx'))
   return tools
 
 
 def TargetLibCompiler(host, options):
   def H(component_name):
     return FlavoredName(component_name, host, options)
   compiler = {
       # Because target_lib_compiler is not a memoized target, its name doesn't
       # need to have the host appended to it (it can be different on different
       # hosts), which means that target library build rules don't have to care
@@ skipping 88 matching lines @@
     redirect_deps = []
     redirect_inputs = { 'redirector_script': REDIRECTOR_SCRIPT }
     redirect_cmds = [
         command.Command([
             '%(abs_redirector_script)s',
             command.path.join('%(output)s', 'bin', 'i686-nacl-' + tool),
             'x86_64-nacl-' + tool,
             args])
         for tool, args in TOOL_X64_I686_REDIRECTS]
 
+  binutils_flags, binutils_inputs, binutils_deps = ConfigureBinutilsCommon(
+      host, options, False)
+  redirect_inputs.update(binutils_inputs)
   tools.update({
       H('binutils_x86'): {
           'type': 'build',
-          'dependencies': ['binutils_x86_src'] + redirect_deps,
+          'dependencies': ['binutils_x86_src'] + redirect_deps + binutils_deps,
           'inputs': redirect_inputs,
           'commands': [
               command.SkipForIncrementalCommand(
-                  ['sh', '%(binutils_x86_src)s/configure'] +
-                  ConfigureBinutilsCommon() +
+                  ['sh', '%(binutils_x86_src)s/configure'] + binutils_flags +
                   ['--target=x86_64-nacl',
-                   '--enable-gold',
+                   # TODO(mcgrathr): Enable gold if we rebase to 2.25.
+                   # The 2.24 gold sources are not compatible with the libc++
+                   # version we use to build.
+                   '--disable-gold',
                    '--enable-targets=x86_64-nacl,i686-nacl',
                    '--disable-werror']),
               command.Command(MakeCommand(host)),
               command.Command(MAKE_DESTDIR_CMD + ['install-strip'])] +
               # Remove the share dir from this binutils build and leave the one
               # from the newer version used for bitcode linking. Always remove
               # the lib dirs, which have unneeded host libs.
               [command.RemoveDirectory(os.path.join('%(output)s', dir))
                for dir in ('lib', 'lib32', 'lib64', 'share')] +
               # Create the set of directories for target libs and includes, for
@@ skipping 290 matching lines @@
       SANDBOXED_TRANSLATOR_ARCHES))
 
   tb = toolchain_main.PackageBuilder(packages,
                                      upload_packages,
                                      leftover_args)
   return tb.Main()
 
 
 if __name__ == '__main__':
   sys.exit(main())