Adding le32-nacl build of pnacl tools.

toolchain_build/toolchain_build_pnacl.py

 #!/usr/bin/python
 # Copyright (c) 2013 The Native Client Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Recipes for PNaCl toolchain packages.
 
    Recipes consist of specially-structured dictionaries, with keys for package
    name, type, commands to execute, etc. The structure is documented in the
    PackageBuilder docstring in toolchain_main.py.
@@ skipping 124 matching lines @@
 def TripleIsX8664(t):
   return fnmatch.fnmatch(t, 'x86_64*')
 
 
 # Return a tuple (C compiler, C++ compiler) of the compilers to compile the host
 # toolchains
 def CompilersForHost(host):
   compiler = {
       # For now we only do native builds for linux and mac
       # treat 32-bit linux like a native build
-      'i686-linux': (CHROME_CLANG, CHROME_CLANGXX),
-      'x86_64-linux': (CHROME_CLANG, CHROME_CLANGXX),
-      'x86_64-apple-darwin': (CHROME_CLANG, CHROME_CLANGXX),
+      'i686-linux': (CHROME_CLANG, CHROME_CLANGXX, 'ar', 'ranlib'),
+      'x86_64-linux': (CHROME_CLANG, CHROME_CLANGXX, 'ar', 'ranlib'),
+      'x86_64-apple-darwin': (CHROME_CLANG, CHROME_CLANGXX, 'ar', 'ranlib'),
       # Windows build should work for native and cross
-      'i686-w64-mingw32': ('i686-w64-mingw32-gcc', 'i686-w64-mingw32-g++'),
+      'i686-w64-mingw32': (
+          'i686-w64-mingw32-gcc', 'i686-w64-mingw32-g++', 'ar', 'ranlib'),
       # TODO: add arm-hosted support
-      'i686-pc-cygwin': ('gcc', 'g++'),
+      'i686-pc-cygwin': ('gcc', 'g++', 'ar', 'ranlib'),
   }
+  if host == 'le32-nacl':
+    nacl_sdk = os.environ.get('NACL_SDK_ROOT')
+    assert nacl_sdk, 'NACL_SDK_ROOT not set'
+    pnacl_bin_dir = os.path.join(nacl_sdk, 'toolchain/linux_pnacl/bin')
+    glibc_bin_dir = os.path.join(nacl_sdk, 'toolchain/linux_x86_glibc/bin')
+    compiler.update({
+        'le32-nacl': (os.path.join(pnacl_bin_dir, 'pnacl-clang'),
+                      os.path.join(pnacl_bin_dir, 'pnacl-clang++'),
+                      os.path.join(pnacl_bin_dir, 'pnacl-ar'),
+                      os.path.join(pnacl_bin_dir, 'pnacl-ranlib')),
+    })
   return compiler[host]
 
 
 def GSDJoin(*args):
   return '_'.join([pynacl.gsd_storage.LegalizeName(arg) for arg in args])
 
 
-def ConfigureHostArchFlags(host, extra_cflags, options):
+def ConfigureHostArchFlags(host, extra_cflags, options, extra_configure=None):
   """ Return flags passed to LLVM and binutils configure for compilers and
   compile flags. """
   configure_args = []
   extra_cc_args = []
 
+  configure_args += options.extra_configure_args
+  if extra_configure is not None:
+    configure_args += extra_configure
+  if options.extra_cc_args is not None:
+    extra_cc_args += [options.extra_cc_args]

[Derek Schuff, 2015/01/26 17:13:54]

should options.extra_cc_args already be a list?

[bradn, 2015/01/26 19:32:20]

So I decided it was easier to treat this one as a string coming in, otherwise
there'd be the issue of spliting it on space and then combining it back
together.
The configure args need to be separate so that they can individually have spaces
for things like: 'LIBS=-lfoo -lbar', which actually needs to be passes in a
separte argv for configure.

+
   native = pynacl.platform.PlatformTriple()
   is_cross = host != native
   if is_cross:
     if (pynacl.platform.IsLinux64() and
         fnmatch.fnmatch(host, '*-linux*')):
       # 64 bit linux can build 32 bit linux binaries while still being a native
       # build for our purposes. But it's not what config.guess will yield, so
       # use --build to force it and make sure things build correctly.
       configure_args.append('--build=' + host)
     else:
       configure_args.append('--host=' + host)
   if TripleIsLinux(host) and not TripleIsX8664(host):
     # Chrome clang defaults to 64-bit builds, even when run on 32-bit Linux.
     extra_cc_args = ['-m32']
 
   extra_cxx_args = list(extra_cc_args)
 
   if not options.gcc:
-    cc, cxx = CompilersForHost(host)
+    cc, cxx, ar, ranlib = CompilersForHost(host)
 
     configure_args.append('CC=' + ' '.join([cc] + extra_cc_args))
     configure_args.append('CXX=' + ' '.join([cxx] + extra_cxx_args))
+    configure_args.append('AR=' + ar)
+    configure_args.append('RANLIB=' + ranlib)
 
   if TripleIsWindows(host):
     # The i18n support brings in runtime dependencies on MinGW DLLs
     # that we don't want to have to distribute alongside our binaries.
     # So just disable it, and compiler messages will always be in US English.
     configure_args.append('--disable-nls')
     configure_args.extend(['LDFLAGS=-L%(abs_libdl)s -ldl',
                            'CFLAGS=-isystem %(abs_libdl)s',
                            'CXXFLAGS=-isystem %(abs_libdl)s'])
     if is_cross:
       # LLVM's linux->mingw cross build needs this
       configure_args.append('CC_FOR_BUILD=gcc')
   else:
     if TripleIsMac(host):
       # This is required for building with recent libc++ against OSX 10.6
       extra_cflags.append('-U__STRICT_ANSI__')
-    if options.gcc:
+    if options.gcc or host == 'le32-nacl':
       configure_args.extend(['CFLAGS=' + ' '.join(extra_cflags),
                              'CXXFLAGS=' + ' '.join(extra_cflags)])
     else:
       configure_args.extend(
        ['CFLAGS=' + ' '.join(extra_cflags),
         'LDFLAGS=-L%(' + GSDJoin('abs_libcxx', host) + ')s/lib',
         'CXXFLAGS=-stdlib=libc++ -I%(' + GSDJoin('abs_libcxx', host) +
         ')s/include/c++/v1 ' + ' '.join(extra_cflags)])
 
   return configure_args
 
 
 def LibCxxHostArchFlags(host):
-  cc, cxx = CompilersForHost(host)
+  cc, cxx, _, _ = CompilersForHost(host)
   cmake_flags = []
   cmake_flags.extend(['-DCMAKE_C_COMPILER='+cc, '-DCMAKE_CXX_COMPILER='+cxx])
   if TripleIsLinux(host) and not TripleIsX8664(host):
     # Chrome clang defaults to 64-bit builds, even when run on 32-bit Linux
     cmake_flags.extend(['-DCMAKE_C_FLAGS=-m32',
                         '-DCMAKE_CXX_FLAGS=-m32'])
   return cmake_flags
 
 def CmakeHostArchFlags(host, options):
   """ Set flags passed to LLVM cmake for compilers and compile flags. """
   cmake_flags = []
-  cc, cxx = CompilersForHost(host)
+  cc, cxx, _, _ = CompilersForHost(host)
 
   cmake_flags.extend(['-DCMAKE_C_COMPILER='+cc, '-DCMAKE_CXX_COMPILER='+cxx])
 
   # There seems to be a bug in chrome clang where it exposes the msan interface
   # (even when compiling without msan) but then does not link with an
   # msan-enabled compiler_rt, leaving references to __msan_allocated_memory
   # undefined.
   cmake_flags.append('-DHAVE_SANITIZER_MSAN_INTERFACE_H=FALSE')
 
   if options.sanitize:
@@ skipping 143 matching lines @@
   if TripleIsLinux(host):
     libname = 'libc++.so.1'
   elif TripleIsMac(host):
     libname = 'libc++.1.dylib'
   else:
     return []
   return [command.Mkdir(dest, parents=True),
           command.Copy('%(' + GSDJoin('abs_libcxx', host) +')s/lib/' + libname,
                        os.path.join(dest, libname))]
 
-def CreateSymLinksToDirectToNaClTools():
+def CreateSymLinksToDirectToNaClTools(host):
+  if host == 'le32-nacl':
+    return []
   return (
       [command.Command(['ln', '-f',
                         command.path.join('%(output)s', 'bin','clang'),
                         command.path.join('%(output)s', 'bin',
                                           arch + '-nacl-clang')])
        for arch in DIRECT_TO_NACL_ARCHES] +
       [command.Command(['ln', '-f',
                         command.path.join('%(output)s', 'bin','clang'),
                         command.path.join('%(output)s', 'bin',
                                           arch + '-nacl-clang++')])
@@ skipping 58 matching lines @@
 def HostTools(host, options):
   def H(component_name):
     # Return a package name for a component name with a host triple.
     return GSDJoin(component_name, host)
   # Return the file name with the appropriate suffix for an executable file.
   def Exe(file):
     if TripleIsWindows(host):
       return file + '.exe'
     else:
       return file
+
+  werror = []
+  extra_gold_deps = []
+  if host == 'le32-nacl':
+    # TODO(bradnelson): Fix warnings so this can go away.
+    werror = ['--enable-werror=no']
+    extra_gold_deps = [H('llvm')]
+
   # Binutils still has some warnings when building with clang
   if not options.gcc:
     warning_flags = ['-Wno-extended-offsetof', '-Wno-absolute-value',
                     '-Wno-unused-function', '-Wno-unused-const-variable',
                     '-Wno-unneeded-internal-declaration',
                     '-Wno-unused-private-field', '-Wno-format-security']
   else:
     warning_flags = ['-Wno-unused-function', '-Wno-unused-value']
+
   tools = {
       # The binutils_pnacl package is used both for bitcode linking (gold) and
       # for its conventional use with arm-nacl-clang.
       H('binutils_pnacl'): {
-          'dependencies': ['binutils_pnacl_src'],
+          'dependencies': ['binutils_pnacl_src'] + extra_gold_deps,
           'type': 'build',
           'inputs' : { 'macros': os.path.join(NACL_DIR,
               'pnacl', 'support', 'clang_direct', 'nacl-arm-macros.s')},
           'commands': [
               command.SkipForIncrementalCommand([
                   'sh',
                   '%(binutils_pnacl_src)s/configure'] +
                   ConfigureBinutilsCommon() +
-                  ConfigureHostArchFlags(host, warning_flags, options) +
+                  ConfigureHostArchFlags(
+                    host, warning_flags, options,
+                    options.binutils_pnacl_extra_configure) +
                   ['--target=arm-nacl',
                   '--program-prefix=le32-nacl-',
                   '--enable-targets=arm-nacl,i686-nacl,x86_64-nacl,' +
                   'mipsel-nacl',
                   '--enable-shared=no',
                   '--enable-gold=default',
                   '--enable-plugins',
                   '--without-gas',
-                  '--with-sysroot=/le32-nacl']),
+                  '--with-sysroot=/le32-nacl'] + werror),
               command.Command(MakeCommand(host)),
               command.Command(MAKE_DESTDIR_CMD + ['install-strip'])] +
               [command.RemoveDirectory(os.path.join('%(output)s', dir))
                for dir in ('lib', 'lib32')] +
               # Since it has dual use, just create links for both sets of names
               # (i.e. le32-nacl-foo and arm-nacl-foo)
               # TODO(dschuff): Use the redirector scripts here like binutils_x86
               [command.Command([
                   'ln', '-f',
                   command.path.join('%(output)s', 'bin', 'le32-nacl-' + tool),
@@ skipping 44 matching lines @@
                   '-DLLVM_ENABLE_ASSERTIONS=ON',
                   '-DLLVM_ENABLE_ZLIB=OFF',
                   '-DLLVM_BUILD_TESTS=ON',
                   '-DLLVM_APPEND_VC_REV=ON',
                   '-DLLVM_BINUTILS_INCDIR=%(abs_binutils_pnacl_src)s/include',
                   '-DLLVM_EXTERNAL_CLANG_SOURCE_DIR=%(clang_src)s',
                   '%(llvm_src)s']),
               command.Command(['ninja', '-v']),
               command.Command(['ninja', 'install']),
               ] +
-          CreateSymLinksToDirectToNaClTools()
+          CreateSymLinksToDirectToNaClTools(host)
       },
   }
+  cleanup_libs = []

[Derek Schuff, 2015/01/26 17:13:54]

maybe call this cleanup_static_libs

[bradn, 2015/01/26 19:32:20]

Done.

+  shared = []
+  if host != 'le32-nacl':
+    shared = ['--enable-shared']
+    cleanup_libs = [
+        command.Remove(*[os.path.join('%(output)s', 'lib', f) for f
+                         in '*.a', '*Hello.*', 'BugpointPasses.*']),
+    ]
   llvm_autoconf = {
       H('llvm'): {
           'dependencies': ['clang_src', 'llvm_src', 'binutils_pnacl_src',
                            'subzero_src'],
           'type': 'build',
           'commands': [
               command.SkipForIncrementalCommand([
                   'sh',
                   '%(llvm_src)s/configure'] +
                   ConfigureHostArchFlags(host, [], options) +
                   LLVMConfigureAssertionsFlags(options) +
                   ['--prefix=/',
-                   '--enable-shared',
                    '--disable-zlib',
                    '--disable-terminfo',
                    '--disable-jit',
                    '--disable-bindings', # ocaml is currently the only binding.
                    '--with-binutils-include=%(abs_binutils_pnacl_src)s/include',
                    '--enable-targets=x86,arm,mips',
                    '--program-prefix=',
                    '--enable-optimized',
-                   '--with-clang-srcdir=%(abs_clang_src)s'])] +
+                   '--with-clang-srcdir=%(abs_clang_src)s'] + shared)] +
               CopyHostLibcxxForLLVMBuild(
                   host,
                   os.path.join('Release+Asserts', 'lib'),
                   options) +
               [command.Command(MakeCommand(host) + [
                   'VERBOSE=1',
                   'NACL_SANDBOX=0',
+                  'PNACL_BROWSER_TRANSLATOR=0',
                   'SUBZERO_SRC_ROOT=%(abs_subzero_src)s',
                   'all']),
-              command.Command(MAKE_DESTDIR_CMD + ['install']),
-              command.Remove(*[os.path.join('%(output)s', 'lib', f) for f in
-                              '*.a', '*Hello.*', 'BugpointPasses.*']),
+              command.Command(MAKE_DESTDIR_CMD + ['install'])] +
+              cleanup_libs + [
               command.Remove(*[os.path.join('%(output)s', 'bin', f) for f in
                                Exe('clang-format'), Exe('clang-check'),
                                Exe('c-index-test'), Exe('clang-tblgen'),
                                Exe('llvm-tblgen')])] +
-              CreateSymLinksToDirectToNaClTools() +
+              CreateSymLinksToDirectToNaClTools(host) +
               CopyWindowsHostLibs(host),
       },
   }
   if options.cmake:
     tools.update(llvm_cmake)
   else:
     tools.update(llvm_autoconf)
   if TripleIsWindows(host):
     tools[H('binutils_pnacl')]['dependencies'].append('libdl')
     tools[H('llvm')]['dependencies'].append('libdl')
-  elif not options.gcc:
+  elif not options.gcc and host != 'le32-nacl':
     tools[H('binutils_pnacl')]['dependencies'].append(H('libcxx'))
     tools[H('llvm')]['dependencies'].append(H('libcxx'))
   return tools
 
 
 def TargetLibCompiler(host, options):
   def H(component_name):
     return GSDJoin(component_name, host)
   compiler = {
       # Because target_lib_compiler is not a memoized target, its name doesn't
@@ skipping 320 matching lines @@
   parser.add_argument('--gcc', action='store_true', default=False,
                       help="Use the default compiler 'cc' instead of clang")
   parser.add_argument('--sanitize', choices=['address', 'thread', 'memory',
                                              'undefined'],
                       help="Use a sanitizer with LLVM's clang cmake build")
   parser.add_argument('--testsuite-sync', action='store_true', default=False,
                       help=('Sync the sources for the LLVM testsuite. '
                       'Only useful if --sync/ is also enabled'))
   parser.add_argument('--build-sbtc', action='store_true', default=False,
                       help='Build the sandboxed translators')
+  parser.add_argument('--pnacl-in-pnacl', action='store_true', default=False,
+                      help='Build with a PNaCl toolchain')
+  parser.add_argument('--extra-cc-args', default=None,
+                      help='Extra arguments to pass to cc/cxx')
+  parser.add_argument('--extra-configure-arg', dest='extra_configure_args',
+                      default=[], action='append',
+                      help='Extra arguments to pass pass to host configure')
+  parser.add_argument('--binutils-pnacl-extra-configure',
+                      default=[], action='append',
+                      help='Extra binutils-pnacl arguments '
+                           'to pass pass to host configure')
   args, leftover_args = parser.parse_known_args()
   if '-h' in leftover_args or '--help' in leftover_args:
     print 'The following arguments are specific to toolchain_build_pnacl.py:'
     parser.print_help()
     print 'The rest of the arguments are generic, in toolchain_main.py'
 
   if args.sanitize and not args.cmake:
     print 'Use of sanitizers requires a cmake build'
     sys.exit(1)
 
@@ skipping 13 matching lines @@
       leftover_args.append('--sync-only')
   else:
     upload_packages = GetUploadPackageTargets()
     if pynacl.platform.IsWindows():
       InstallMinGWHostCompiler()
 
     packages.update(HostToolsSources(GetGitSyncCmdsCallback(rev)))
     if args.testsuite_sync:
       packages.update(TestsuiteSources(GetGitSyncCmdsCallback(rev)))
 
-    hosts = [pynacl.platform.PlatformTriple()]
+    if args.pnacl_in_pnacl:
+      hosts = ['le32-nacl']
+    else:
+      hosts = [pynacl.platform.PlatformTriple()]
     if pynacl.platform.IsLinux() and BUILD_CROSS_MINGW:
       hosts.append(pynacl.platform.PlatformTriple('win', 'x86-32'))
     for host in hosts:
-      packages.update(HostLibs(host, args))
       packages.update(HostTools(host, args))
-      packages.update(HostToolsDirectToNacl(host))
-    packages.update(TargetLibCompiler(pynacl.platform.PlatformTriple(), args))
+      if not args.pnacl_in_pnacl:
+        packages.update(HostLibs(host, args))
+        packages.update(HostToolsDirectToNacl(host, args))
+    if not args.pnacl_in_pnacl:
+      packages.update(TargetLibCompiler(pynacl.platform.PlatformTriple(), args))
     # Don't build the target libs on Windows because of pathname issues.
     # Only the linux64 bot is canonical (i.e. it will upload its packages).
     # The other bots will use a 'work' target instead of a 'build' target for
     # the target libs, so they will not be memoized, but can be used for tests.
     # TODO(dschuff): Even better would be if we could memoize non-canonical
     # build targets without doing things like mangling their names (and for e.g.
     # scons tests, skip running them if their dependencies haven't changed, like
     # build targets)
     is_canonical = pynacl.platform.IsLinux64()
-    if pynacl.platform.IsLinux() or pynacl.platform.IsMac():
+    if ((pynacl.platform.IsLinux() or pynacl.platform.IsMac())
+        and not args.pnacl_in_pnacl):
       packages.update(pnacl_targetlibs.TargetLibsSrc(
         GetGitSyncCmdsCallback(rev)))
       for bias in BITCODE_BIASES:
         packages.update(pnacl_targetlibs.TargetLibs(bias, is_canonical))
       for arch in DIRECT_TO_NACL_ARCHES:
         packages.update(pnacl_targetlibs.TargetLibs(arch, is_canonical))
         packages.update(pnacl_targetlibs.SDKLibs(arch, is_canonical))
       for arch in TRANSLATOR_ARCHES:
         packages.update(pnacl_targetlibs.TranslatorLibs(arch, is_canonical))
       packages.update(Metadata(rev, is_canonical))
       packages.update(pnacl_targetlibs.SDKCompiler(
                       ['le32'] + DIRECT_TO_NACL_ARCHES))
       packages.update(pnacl_targetlibs.SDKLibs('le32', is_canonical))
-    if pynacl.platform.IsLinux() or pynacl.platform.IsMac():
       unsandboxed_irt_canonical = is_canonical or pynacl.platform.IsMac()
       packages.update(pnacl_targetlibs.UnsandboxedIRT(
           'x86-32-%s' % pynacl.platform.GetOS(), unsandboxed_irt_canonical))
 
-    if args.build_sbtc:
+    if args.build_sbtc and not args.pnacl_in_pnacl:
       packages.update(pnacl_sandboxed_translator.SandboxedTranslators(
         SANDBOXED_TRANSLATOR_ARCHES))
 
   tb = toolchain_main.PackageBuilder(packages,
                                      upload_packages,
                                      leftover_args)
   tb.Main()